class PyAPNSNotification(threading.Thread, NotificationAbstract):
def __init__(self, host, app_id, cert_file, dev_mode = False, reconnect_interval=10, chunk_size=10):
super(PyAPNSNotification, self).__init__()
self.keepRunning = True
self.is_server_ready = False
self.notifications = Queue()
pyapns_client.configure({'HOST': host})
self.reconnect_interval = reconnect_interval
self.app_id = app_id
self.cert_file = cert_file
self.chunk_size = chunk_size
if dev_mode:
self.mode = 'sandbox'
else:
self.mode = 'production'
def run(self):
while self.keepRunning or not self.notifications.empty():
if not self.is_server_ready:
try:
pyapns_client.provision(self.app_id, open(self.cert_file).read(), self.mode)
self.is_server_ready = True
except Exception:
if self.keepRunning:
self.is_server_ready = False
time.sleep(self.reconnect_interval)
continue
else:
break
tokens = []
messages = []
for i in xrange(self.chunk_size):
if self.notifications.empty() and len(tokens):
break
notification = self.notifications.get()
if notification is None:
self.notifications.task_done()
break
tokens.append(notification['token'])
messages.append(notification['message'])
self.notifications.task_done()
try:
if len(tokens):
pyapns_client.notify(self.app_id, tokens, messages)
except Exception:
self.is_server_ready = False
for i in xrange(len(tokens)):
self.notifications.put({'token':tokens[i],'message':messages[i]})
def stop(self):
self.keepRunning = False
self.notifications.put(None)
def perform_notification(self, token, aps_message):
self.notifications.put({'token':token,'message':aps_message})
class EndPoint(object):
'''
Representation of a stream's endpoint.
'''
def __init__(self, channel, idx):
self._channel = channel
self._i = idx
self._queue = Queue()
def receive(self, block = True):
try:
if self._queue.empty():
# print 'Updating for end point: [%d]' % (self._i)
self._channel._update(self._i)
except StreamClosedException:
if self._queue.empty():
raise
else:
pass
# print 'Returning for end point: [%d]' % (self._i)
return self._queue.get()
def processed(self):
self._queue.task_done()
def send(self, r):
self._queue.put(r)
class QueuedWavePlayer(object):
def __init__(self):
self.queue = Queue();
freq = 44100 # audio CD quality
bitsize = -16 # unsigned 16 bit
channels = 2 # 1 is mono, 2 is stereo
buffer = 2048 # number of samples (experiment to get right sound)
pygame.mixer.init(freq, bitsize, channels, buffer)
self.channel = pygame.mixer.Channel(1)
self.logger = logging.getLogger("qplayer")
t=threading.Thread(target=self.playing_target);
t.daemon = True;
t.start()
def playing_target(self):
while True:
file_to_play=self.queue.get()
self.logger.info("Queuing %s in channel",file_to_play)
self.channel.queue(pygame.mixer.Sound(file_to_play))
self.queue.task_done()
while self.channel.get_queue():
time.sleep(1);
def queue_for_playing(self,file_to_play):
self.queue.put(file_to_play)
class IWRCBot():
def __init__(self, site, safe = True):
self.other_ns = re.compile(u'14\[\[07(' + u'|'.join(site.namespaces()) + u')')
interwiki.globalvar.autonomous = True
self.site = site
self.queue = Queue()
self.processed = []
self.safe = safe
# Start 20 threads
for i in range(20):
t = threading.Thread(target=self.worker)
t.setDaemon(True)
t.start()
def worker(self):
bot = interwiki.InterwikiBot()
while True:
# Will wait until one page is available
bot.add(self.queue.get())
bot.queryStep()
self.queue.task_done()
def addQueue(self, name):
if self.other_ns.match(name):
return
if self.safe:
if name in self.processed:
return
self.processed.append(name)
page = pywikibot.Page(self.site, name)
# the Queue has for now an unlimited size,
# it is a simple atomic append(), no need to acquire a semaphore
self.queue.put_nowait(page)
class CommandQueue(object):
"""Asynchronous command queue that can be used to communicate with blip.pl
in the background."""
def __init__(self):
self.queue = Queue()
self.worker = threading.Thread(target=self.__worker)
self.worker.setDaemon(True)
def __del__(self):
self.Finish()
def __worker(self):
while True:
item = self.queue.get(True)
item()
self.queue.task_done()
def Finish(self):
"""Finishes all commands in the queue"""
self.queue.join()
def HasPendingCommands(self):
"""Returns True if the queue is busy"""
return self.queue.qsize() > 0
def Enqueue(self, command):
"""Enqueues a command in the queue.
Command must refer to a function without parameters."""
self.queue.put(command)
class Worker:
def __init__(self):
self.q = Queue()
self.t = Thread(target=self._handle)
self.t.setDaemon(True)
self.t.start()
def _handle(self):
while True:
reset_caches()
fn = self.q.get()
try:
fn()
self.q.task_done()
except:
import traceback
print traceback.format_exc()
def do(self, fn, *a, **kw):
fn1 = lambda: fn(*a, **kw)
self.q.put(fn1)
def join(self):
self.q.join()
def test_request_retries_configurable(self):
# We guess at some ports that will be unused by Riak or
# anything else.
client = self.create_client(http_port=DUMMY_HTTP_PORT,
pb_port=DUMMY_PB_PORT)
# Change the retry count
client.retries = 10
self.assertEqual(10, client.retries)
# The retry count should be a thread local
retries = Queue()
def _target():
retries.put(client.retries)
retries.join()
th = Thread(target=_target)
th.start()
self.assertEqual(3, retries.get(block=True))
retries.task_done()
th.join()
# Modify the retries in a with statement
with client.retry_count(5):
self.assertEqual(5, client.retries)
self.assertRaises(IOError, client.ping)
def test_create_cell_recalculator_should(self, mock_recalculate):
unrecalculated_queue = Queue()
unrecalculated_queue.put(1)
unrecalculated_queue.put(1)
unrecalculated_queue.task_done = Mock()
leaf_queue = Queue()
leaf_queue.put(sentinel.one)
leaf_queue.put(sentinel.two)
leaf_queue.task_done = Mock()
target = create_cell_recalculator(leaf_queue, unrecalculated_queue, sentinel.graph, sentinel.context)
target()
self.assertTrue(unrecalculated_queue.empty())
self.assertEquals(
unrecalculated_queue.task_done.call_args_list,
[ ((), {}), ((), {}), ]
)
self.assertTrue(leaf_queue.empty())
self.assertEquals(
mock_recalculate.call_args_list,
[
((sentinel.one, leaf_queue, sentinel.graph, sentinel.context), {}),
((sentinel.two, leaf_queue, sentinel.graph, sentinel.context), {})
]
)
self.assertEquals(
leaf_queue.task_done.call_args_list,
[ ((), {}), ((), {}), ]
)
class FrameSaver( threading.Thread ): def __init__( self ): threading.Thread.__init__( self ) self.daemon = True self.name = 'FrameSaver' self.reset() def reset( self ): self.queue = Queue() def run( self ): self.reset() while 1: message = self.queue.get() if message[0] == 'Save': cmd, fileName, bib, t, frame = message #sys.stderr.write( 'save' ) PhotoFinish.SavePhoto( fileName, bib, t, frame ) self.queue.task_done() elif message[0] == 'Terminate': self.queue.task_done() self.reset() break def stop( self ): self.queue.put( ['Terminate'] ) self.join() def save( self, fileName, bib, t, frame ): self.queue.put( ['Save', fileName, bib, t, frame] )
class ThreadQueue(object):
def __init__(self):
self.q = Queue()
t = Thread(target=self._thread_worker)
t.setDaemon(True)
t.start()
def add_request(self, func, *args, **kwargs):
"""Add a request to the queue. Pass callback= and/or error= as
keyword arguments to receive return from functions or exceptions.
"""
self.q.put((func, args, kwargs))
def _thread_worker(self):
while True:
request = self.q.get()
self.do_request(request)
self.q.task_done()
def do_request(self, (func, args, kwargs)):
callback = kwargs.pop('callback', None)
error = kwargs.pop('error', None)
try:
r = func(*args, **kwargs)
if not isinstance(r, tuple): r = (r,)
if callback: self.do_callback(callback, *r)
except Exception, e:
if error: self.do_callback(error, e)
else: print "Unhandled error:", e
class FileGenerator(object):
def __init__(self):
self.q = Queue()
def read_generator(self):
running = True
while(running):
try:
data = self.q.get(block=True, timeout=1)
self.q.task_done()
if data is None:
running = False
else:
yield data
except:
running = False
def write(self, s):
self.q.put(s)
def close(self):
self.q.put(None)
def tell(self):
return 0
def flush(self):
return True
class DownloadThreadPool(object):
def __init__(self, size=3):
self.queue = Queue()
self.workers = [Thread(target=self._do_work) for _ in range(size)]
self.initialized = False
def init_threads(self):
for worker in self.workers:
worker.setDaemon(True)
worker.start()
self.initialized = True
def _do_work(self):
while True:
url, target = self.queue.get()
download(url, target, close_target=True, quiet=False)
self.queue.task_done()
def join(self):
self.queue.join()
def submit(self, url, target):
if not self.initialized:
self.init_threads()
self.queue.put((url, target))
class WorkerThread(threading.Thread):
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.input_queue = Queue()
def send(self, item):
self.input_queue.put(item)
def close(self):
self.input_queue.put(None)
self.input_queue.join()
def run(self):
while True:
item = self.input_queue.get()
if item is None:
break
# Process the item: replace with useful work
print item
self.input_queue.task_done()
# Done. Indicate that sentinel was received and return
self.input_queue.task_done()
return
class Events(threading.Thread):
def __init__(self, callback):
super(Events, self).__init__()
self.queue = Queue()
self.daemon = True
self.callback = callback
self.name = "EVENT-QUEUE"
self.stop = threading.Event()
def put(self, type):
self.queue.put(type)
def run(self):
while (not self.stop.is_set()):
try:
# get event type
type = self.queue.get(True, 1)
# perform callback if we got a event type
self.callback(type)
# event completed
self.queue.task_done()
except Empty:
type = None
# exiting thread
self.stop.clear()
# System Events
class SystemEvent(Event):
RESTART = "RESTART"
SHUTDOWN = "SHUTDOWN"
class ThreadPool():
def __init__(self):
self.q = Queue()
self.NUM = 3
self.JOBS = 10
def do_somthing_using(self,arguments):
print arguments
def working(self):
while True:
arguments = self.q.get()
self.do_somthing_using(arguments+1)
sleep(1)
self.q.task_done()
def PoolStart(self):
for i in range(self.NUM):
t = Thread(target=self.working)
t.setDaemon(True)
t.start()
for i in range(self.JOBS):
self.q.put(i)
self.q.join()
class Sceduler:
def __init__(self, config):
fs = config.get('scheduler', 'fs', 0)
dest = config.get('store', 'path', 0)
self.ioqueue = Queue()
self.iothread = Thread(target=self.ioprocess)
self.iothread.daemon = True
self.observer = Observer()
self.event_handler = IoTask(self.ioqueue, fs, dest)
self.observer.schedule(self.event_handler, fs, recursive=True)
def ioprocess(self):
while True:
t = self.ioqueue.get()
try:
t.process()
finally:
self.ioqueue.task_done()
def start(self):
self.observer.start()
self.iothread.start()
def stop(self):
self.observer.stop()
self.iothread.stop()
def join(self):
self.observer.join()
self.iothread.join()
class DecodingThread(threading.Thread):
"""Thread for concurrent simulation.
A :class:`DecodingThread` is responsible for one specific decoder. As soon as an item is
placed on the :attr:`jobQueue`, decoding starts. After finishing, the attributes
:attr:`time`, :attr:`error`, :attr:`objVal` and :attr:`mlCertificate` contain information
about the solution.
:param decoder: The :class:`.Decoder` used for this process.
:param revealSent: If decoding should reveal the sent codeword.
.. attribute:: jobQueue
On this queue, pairs (llr, sentCodeword) are put. The process will start decoding
immediately, and signal :func:`JoinableQueue.task_done` when finished."""
def __init__(self, decoder, revealSent):
threading.Thread.__init__(self)
self.decoder = decoder
self.jobQueue = Queue()
self.daemon = True
self.revealSent = revealSent
self.time = 0.0
self.start()
def run(self):
while True:
llr, sent = self.jobQueue.get()
with Timer() as timer:
if self.revealSent:
self.decoder.decode(llr, sent=sent)
else:
self.decoder.decode(llr)
self.time = timer.duration
self.jobQueue.task_done()
def test_get_batches_new_chromosome():
"""
Test to get a batch
"""
batch_queue = Queue()
variants = []
first_variant = get_variant_line()
second_variant = get_variant_line(chrom="2")
variants.append(first_variant)
variants.append(second_variant)
header = HeaderParser()
header.parse_header_line("#{0}".format(HEADER))
chromosomes = get_batches(variants=variants, batch_queue=batch_queue,
header=header)
batch_1 = batch_queue.get()
batch_queue.task_done()
batch_2 = batch_queue.get()
batch_queue.task_done()
assert chromosomes == ['1', '2']
assert len(batch_1) == 1
assert len(batch_2) == 1
class ThreadPool(object):
"""A thread pool manager."""
def __init__(self, threads=8):
"""Construct a new thread pool with :ref:`threads` threads.
:param threads: Number of threads to start in the thread pool.
"""
self._queue = Queue()
self._pool = [threading.Thread(target=self._worker)
for _ in range(threads)]
def _worker(self):
"""Waits for and executes jobs from the queue."""
while True:
message = self._queue.get()
if message is None:
self._queue.task_done()
return
job, args, kwargs = message
try:
job(*args, **kwargs)
except Exception, e:
log.error('thread pool worker failed', exc_info=e)
self._queue.task_done()
def test_get_batches_two_regions():
"""
Test to get a batch
"""
batch_queue = Queue()
variants = []
first_variant = get_variant_line()
second_variant = get_variant_line(pos="2", info="Annotation=DDD;Exonic")
variants.append(first_variant)
variants.append(second_variant)
header = HeaderParser()
header.parse_header_line("#{0}".format(HEADER))
chromosomes = get_batches(variants=variants, batch_queue=batch_queue,
header=header)
batch_1 = batch_queue.get()
batch_queue.task_done()
batch_2 = batch_queue.get()
batch_queue.task_done()
assert chromosomes == ['1']
assert len(batch_1) == 1
assert len(batch_2) == 1
def multiget(client, keys, **options):
"""
Executes a parallel-fetch across multiple threads. Returns a list
containing :class:`~riak.riak_object.RiakObject` instances, or
3-tuples of bucket, key, and the exception raised.
:param client: the client to use
:type client: :class:`~riak.client.RiakClient`
:param keys: the bucket/key pairs to fetch in parallel
:type keys: list of two-tuples -- bucket/key pairs
:rtype: list
"""
outq = Queue()
RIAK_MULTIGET_POOL.start()
for bucket, key in keys:
task = Task(client, outq, bucket, key, options)
RIAK_MULTIGET_POOL.enq(task)
results = []
for _ in range(len(keys)):
if RIAK_MULTIGET_POOL.stopped():
raise RuntimeError("Multi-get operation interrupted by pool "
"stopping!")
results.append(outq.get())
outq.task_done()
return results
def test_interrupt_ctrl_c(self):
"""
Interrupt "waagent -deprovision" by "ctrl -c"
"""
self.log.info("Interrupt \"waagent -deprovision\" by \"ctrl -c\"")
# Start 2 threads:
# session1 is for running deprovision command and getting output
# session2 is for getting pid and killing process,
def session1(q):
session = self.vm_test01.wait_for_login()
session.cmd_output("echo {0} | sudo -S sh -c ''".format(self.vm_test01.password))
session.cmd_output("sudo su -")
q.put(session.cmd_output("waagent -deprovision").rstrip('\n'))
def session2():
time.sleep(5)
pid = self.vm_test01.get_pid("deprovision")
self.vm_test01.get_output("kill -2 {0}".format(pid))
import threading
from Queue import Queue
q = Queue()
thread1 = threading.Thread(target=session1, args=(q,))
thread1.setDaemon(True)
thread1.start()
thread2 = threading.Thread(target=session2)
thread2.setDaemon(True)
thread2.start()
thread1.join()
output = q.get()
q.task_done()
self.log.info(output)
self.assertNotIn("message=Traceback", output,
"Should not raise exception.")
def test_get_batches_vep():
"""
Test to get a batch
"""
batch_queue = Queue()
variants = []
first_variant = get_variant_line(info="MQ;CSQ=G|ADK")
second_variant = get_variant_line(pos="2", info="MQ;CSQ=G|ADK")
variants.append(first_variant)
variants.append(second_variant)
header = HeaderParser()
header.parse_header_line("#{0}".format(HEADER))
header.vep_columns = ['Allele', 'SYMBOL']
chromosomes = get_batches(variants=variants, batch_queue=batch_queue,
header=header)
batch_1 = batch_queue.get()
batch_queue.task_done()
batch_2 = batch_queue.get()
batch_queue.task_done()
assert chromosomes == ['1']
assert len(batch_1) == 1
assert len(batch_2) == 1
class ThreadPool(object):
"""Pool of threads consuming tasks from a queue"""
def __init__(self, workers):
self.tasks = Queue()
self.workers = [Worker(self.tasks) for x in xrange(workers)]
self.state = ThreadPoolState.IDLE
def apply_async(self, func, args, **kargs):
"""Add a task to the queue"""
if self.state != ThreadPoolState.IDLE:
raise ThreadPoolError('ThreadPool cant accept any more tasks')
self.tasks.put((func, args, kargs))
def close(self):
self.state = ThreadPoolState.CLOSED
while not self.tasks.empty():
self.tasks.get_nowait()
self.tasks.task_done()
for worker in self.workers:
self.tasks.put((None, (), {}))
def join(self):
"""Wait for completion of all the tasks in the queue"""
self.state = ThreadPoolState.WAIT_JOIN
self.tasks.join()
class CQueueObject:
def __init__(self,sQueueName):
self.sQueueName = sQueueName
self.queue = Queue() #(maxsize = 1000) #处理队列，默认容量无限制
def GetQueueName(self):
return self.sQueueName
def GetQueueSize(self):
return self.queue.qsize()
def PutToQueue(self, oObject):
dictObj = {}
dictObj['object'] = oObject
dictObj['ins_time'] = GetCurrentTime()
self.queue.put(dictObj, block=True)
return self.queue.qsize()
def GetFmQueue(self, sThreadName):
while True:
try:
dictObj = self.queue.get(timeout=0.1) #timeout 0.1s
self.queue.task_done()
return dictObj
except Empty: #如果 Queue 是空，就会出异常
PrintAndSleep(0.1,'%s.GetFmQueue' % sThreadName,False) #避免过多日志
return None
class Listener(object):
def __init__(self, etype, subject):
self.type = etype
self.queue = Queue()
self.subject = subject
def get(self, block=True, timeout=None):
try:
event = self.queue.get(block, timeout)
self.queue.task_done()
logger.debug("%s recv event: %s", self, event)
return event
except Empty:
return None
def __repr__(self):
return "<UeventListener '%s'>" % self.type
def __str__(self):
return self.__repr__()
def __enter__(self):
logger.debug("%s ENTER", self)
return self
def __exit__(self, t, v, tb):
logger.debug("%s EXIT", self)
self.subject.detach(self)
class Events(Thread):
def __init__(self, callback):
super(Events, self).__init__()
self.queue = Queue()
self.daemon = True
self.alive = True
self.callback = callback
self.name = "EVENT-QUEUE"
def put(self, type):
self.queue.put(type)
def run(self):
while(self.alive):
try:
# get event type
type = self.queue.get(True, 1)
# perform callback if we got a event type
self.callback(type)
# event completed
self.queue.task_done()
except Empty:
type = None
# System Events
class SystemEvent(Event):
RESTART = "RESTART"
SHUTDOWN = "SHUTDOWN"
class NotifWorker(threading.Thread):
'''
NotifWorker is a simple worker thread for handling notifications.
It can be subclassed or call out to external function tasks as appropriate.
'''
def __init__(self, is_daemon = True):
threading.Thread.__init__(self)
self.name = "NotifWorker"
self.daemon = is_daemon
self.tasks = Queue()
self.start()
def add_task(self, func, *args, **kargs):
"""Add a task to the queue"""
self.tasks.put((func, args, kargs))
def run(self):
# wait for a task and process it
while True:
func, args, kargs = self.tasks.get()
# TODO: need a sentinel task to indicate shutdown
try:
func(*args, **kargs)
except Exception, e:
# TODO: log this somewhere
print "NotifWorker task raised Exception:"
print e
self.tasks.task_done()
class Sender(threading.Thread):
def __init__(self, server):
threading.Thread.__init__(self,None, None, "Sender")
self.daemon = True # ce thread est un daemon, il s'arretera quand tous les threads non daemon s'arreteront
self._server = server
self._queue = Queue()
def addMsg(self, mask_from, to, msg):
self._queue.put((mask_from,to,msg))
def run(self):
self._server.write("Sender loop start", colorConsol.OKGREEN)
while not self._server.e_shutdown.is_set():
try:
mask_from, to, msg = self._queue.get(True, 2)
except Empty:
pass
else:
self._send(mask_from, to, msg)
self._queue.task_done()
self._server.write("Sender loop stop", colorConsol.WARNING)
def _send(self, mask_from, to, msg):
self._server.write("send : '%s'"%msg)
for c in self._server.clients:
if to & (1 << c.id):
#self._server.write("to : '%s'"%c.id)
threading.Thread(None, c.send, "Sender send to %s"%c.id, (mask_from, msg)).start()
def main(argv):
ourhost = platform.node()
queue = Queue()
threads = [
Thread(target=ssh_thread, kwargs={"queue": queue, "host": argv[1]}),
Thread(target=local_thread, kwargs={"queue": queue})
]
for thread in threads:
thread.daemon = True
thread.start()
while True:
try:
while all(map(lambda t: t.is_alive(), threads)):
try:
line = queue.get(timeout=0.1).strip("\n")
if ourhost+"/L3" in line and "Scheduling" in line:
print("Restarting our services")
Thread(target=restart_services).start()
queue.task_done()
except KeyboardInterrupt:
raise
except Empty:
continue
except:
raise
except KeyboardInterrupt:
break
except Exception as e:
print("Exception caught: {}, reticulating splines...".format(e.msg))
finally:
return 0
class SequentialScheduler (IScheduler):
def __init__ (self):
self.rq = None
self.machine = None
def reschedule (self, machine):
self.machine = machine
self.rq = Queue(-1)
session = Session()
programs = session.query(Program).order_by(desc(Program.priority)).filter(Program.finished == False).all()
if not programs:
return
log.debug("rescheduling, found %d runnable programs" % len(list(programs)))
for program in programs:
self.rq.put(program)
machine.wakeup()
def next (self):
if not self.rq.empty():
return self.rq.get()
return None
def done (self, task, error=None):
if error:
log.debug("Error processing program %s." % str(task))
log.exception(error)
else:
task.finished = True
self.rq.task_done()
self.machine.wakeup()
class EnclosureWriter(Thread):
"""
Writes data to Serial port.
#. Enqueues all commands received from Mycroft enclosures
implementation
#. Process them on the received order by writing on the Serial port
E.g. Displaying a text on Mycroft's Mouth
#. ``EnclosureMouth`` sends a text command
#. ``EnclosureWriter`` captures and enqueue the command
#. ``EnclosureWriter`` removes the next command from the queue
#. ``EnclosureWriter`` writes the command to Serial port
Note: A command has to end with a line break
"""
def __init__(self, serial, client, size=16):
super(EnclosureWriter, self).__init__(target=self.flush)
self.alive = True
self.daemon = True
self.serial = serial
self.client = client
self.commands = Queue(size)
self.start()
def flush(self):
while self.alive:
try:
cmd = self.commands.get()
self.serial.write(cmd + '\n')
LOGGER.info("Writing: " + cmd)
self.commands.task_done()
except Exception as e:
LOGGER.error("Writing error: {0}".format(e))
def write(self, command):
self.commands.put(str(command))
def stop(self):
self.alive = False
self.join()
class Events(threading.Thread):
def __init__(self, callback):
super(Events, self).__init__()
self.queue = Queue()
self.daemon = True
self.callback = callback
self.name = "EVENT-QUEUE"
self.stop = threading.Event()
def put(self, type):
self.queue.put(type)
def run(self):
"""
Actually runs the thread to process events
"""
try:
while not self.stop.is_set():
try:
# get event type
type = self.queue.get(True, 1)
# perform callback if we got a event type
self.callback(type)
# event completed
self.queue.task_done()
except Empty:
type = None
# exiting thread
self.stop.clear()
except Exception as e:
logger.log(u"Exception generated in thread " + self.name + ": " + ex(e), logger.ERROR)
logger.log(repr(traceback.format_exc()), logger.DEBUG)
# System Events
class SystemEvent(Event):
RESTART = "RESTART"
SHUTDOWN = "SHUTDOWN"
def solution(A, B, M, X, Y):
my_queue = Queue()
count = 0
stops = 0
while count < len(A):
free_cap = Y
actual_floor = 0
people_count = 0
while True:
weight = A[count]
if weight <= free_cap:
my_queue.put(count)
free_cap -= weight
print("get: {} w:{} free_w:{}".format(count, weight,free_cap))
count += 1
people_count +=1
else:
break
if people_count == X or count == len(A):
break
while not my_queue.empty():
val = my_queue.get()
people_floor = B[val]
print ("leave: {} floor:{}".format(val, people_floor))
if actual_floor != people_floor:
stops += 1
actual_floor = people_floor
else:
print("same floor")
my_queue.task_done()
print("come back!")
stops += 1
print("stops: {}".format(stops))
def getPath(Raw, startBus, endBus):
# Function to generate the list of paths
NeighbourDict = getNeighbours(
Raw
) # key: any bus in the raw file, value: set of all neighbours (line and tf)
# Use CAPENeighbourDict and BFS to find path from one bus to another. Use the concept given in getNeighboursAtCertainDepthFn
PathDict = {}
explored = set()
#startBus = raw_input('Enter start bus: ')
#endBus = raw_input('Enter end bus: ')
frontier = Queue(maxsize=0)
frontier.put(startBus)
while not frontier.empty():
currentBus = frontier.get()
frontier.task_done()
if currentBus == endBus:
break
NeighBourList = list(NeighbourDict[currentBus])
explored.add(currentBus)
for neighbour in NeighBourList:
if neighbour in explored:
continue
if currentBus in PathDict.keys():
PathDict[neighbour] = PathDict[currentBus] + '->' + neighbour
else: # if currentBus is the start bus
PathDict[neighbour] = currentBus + '->' + neighbour
frontier.put(neighbour)
Path = PathDict[endBus]
return Path
def main():
"""main function"""
context = zmq.Context()
worker = context.socket(zmq.ROUTER)
worker.bind('tcp://*:5570')
print 'Worker started'
receiver = context.socket(zmq.PULL)
receiver.bind('tcp://*:5503')
poll = zmq.Poller()
poll.register(worker, zmq.POLLIN)
poll.register(receiver, zmq.POLLIN)
identifier = ""
queue = Queue(10000)
count = 0
received_count = 0
begin = time.time()
while True:
sockets = dict(poll.poll(10))
if receiver in sockets:
msg = receiver.recv()
queue.put(msg)
received_count = received_count + 1
print 'message received %d time %d' % (received_count,
time.time() - begin)
if worker in sockets:
ident, req_msg = worker.recv_multipart()
#print 'Worker received %s from %s' % (req_msg, ident)
# replies = randint(0,4)
# for i in range(replies):
# time.sleep(1. / (randint(1,10)))
if not queue.empty():
msg = queue.get()
queue.task_done()
worker.send_multipart([ident, msg])
count = count + 1
worker.close()
class WorkQueue(object):
_stopit = object()
def __init__(self, maxsize=5):
self._Q = Queue(maxsize=maxsize)
def push(self, callable):
self._Q.put_nowait(callable) # throws Queue.Full
def push_wait(self, callable):
self._Q.put(callable)
def interrupt(self):
"""Break one call to handle()
eg. Call N times to break N threads.
This call blocks if the queue is full.
"""
self._Q.put(self._stopit)
def handle(self):
"""Process queued work until interrupt() is called
"""
while True:
# TODO: Queue.get() (and anything using thread.allocate_lock
# ignores signals :( so timeout periodically to allow delivery
try:
callable = self._Q.get(True, 1.0)
except Empty:
continue # retry on timeout
try:
if callable is self._stopit:
break
callable()
except:
_log.exception("Error from WorkQueue")
finally:
self._Q.task_done()
class ThreadPool(object):
"""Pool of threads consuming tasks from a queue"""
def __init__(self, num_threads):
self.tasks = Queue()
self.results = Queue()
self.num_threads = num_threads
for _ in range(self.num_threads):
Worker(self.tasks, self.results)
def workload(self, func, *args, **kargs):
""" set current workload for threadpool """
for _ in range(self.num_threads):
self.tasks.put((func, args, kargs))
def query(self):
""" get return values from worker threads """
while True:
yield self.results.get()
self.results.task_done()
def test_queue():
queue = Queue(1)
def consumer():
print 'Consumer waiting'
queue.get() # will blocked if queue is empty
print 'Consumer done'
def producer():
print 'Producer putting'
queue.put(object()) # will blocked if queue is full
print 'Producer done'
for _ in range(10):
threading.Thread(target=producer).start()
print 'Will consume the items'
for _ in range(10):
threading.Thread(target=consumer).start()
queue.task_done()
class ThreadPool():
def __init__(self, pages, url):
self.q = Queue()
self.NUM = 2
self.JOBS = pages
self.url=url
def Clist(self, arg):
print self.url+str(arg)+".html"
req = urllib2.Request(self.url+str(arg)+".html")
html = urllib2.urlopen(req).read()
#html = html.decode("utf8")
print html
Ahref = re.findall('<a class="more".*?href="(.*?)".*?', html, re.S)
i = 0
pc = PhoneCrawl()
for item in Ahref:
name = str(arg)+"-"+str(i)+".txt"
pc.CStart(item,name)
print item+"----------finish!"
i=i+1
def working(self):
while True:
arguments = self.q.get()
self.Clist(arguments+1)
sleep(1)
self.q.task_done()
def PoolStart(self):
for i in range(self.NUM):
t = Thread(target=self.working)
t.setDaemon(True)
t.start()
for i in range(self.JOBS):
self.q.put(i)
self.q.join()
def test_iteration(self):
"""
Iteration over the pool resources, even when some are claimed,
should eventually touch all resources (excluding ones created
during iteration).
"""
for i in range(25):
started = Queue()
n = 1000
threads = []
touched = []
pool = EmptyListPool()
rand = SystemRandom()
def _run():
psleep = rand.uniform(0.05, 0.1)
with pool.transaction() as a:
started.put(1)
started.join()
a.append(rand.uniform(0, 1))
sleep(psleep)
for i in range(n):
th = Thread(target=_run)
threads.append(th)
th.start()
for i in range(n):
started.get()
started.task_done()
for resource in pool:
touched.append(resource)
for thr in threads:
thr.join()
self.assertItemsEqual(pool.resources, touched)
class Executor(object):
def __init__(self):
self.models = list()
self.queue = Queue()
def addModel(self,model):
self.models.append(model)
def doWork(self):
mdl = self.queue.get()
self.singlerun(mdl)
self.queue.task_done()
def singlerun(self,mdl):
map(lambda x: x.start(),mdl.reader)
mdl.writer.start()
mdl.observations = list()
while(self.until(mdl.cyclecount)):
time.sleep(mdl.cycletime)
backup = mdl.data.copy()
mdl.debug('Start cycle in state: ' + mdl.state)
trigger = mdl.getTrigger()
rules = filter(lambda l: (l.prev == mdl.state) and mdl.matchTriggers(trigger,l.trig),mdl.behav)
rules = filter(lambda l: mdl.applyCondition(l.cond),rules)
trname = ''
if (len(rules) > 0):
rule = random.choice(rules)
trname = rule.name
mdl.state = rule.next
mdl.applyActions(rule.acts)
mdl.debug('Trigger received: ' + rule.trig)
mdl.debug('Condition ok: ' + rule.cond)
mdl.debug('Next state: ' + rule.next)
mdl.debug('Action applied: ' + rule.acts)
mdl.record(trname)
mdl.recordStat()
mdl.checkEvents(backup)
mdl.cyclecount += 1
mdl.cleanChannels()
def test_read_thread(session):
'''
Test a situation where threads are created outside of any active
context (hence dry).
'''
cfg = {'db_uri': session, 'schema': SCHEMA}
with connect(cfg):
create_tables()
countries = View('country').read().all()
nb_cty = len(countries)
assert nb_cty > 2
read_threads = []
out_q = Queue()
in_queues = []
for i in range(NB_THREADS):
in_q = Queue(maxsize=1)
in_queues.append(in_q)
t = Thread(target=read, args=(in_q, out_q, cfg))
t.start()
read_threads.append(t)
# Launch metronome to feed input lists
metro_thread = Thread(target=metronome, args=(in_queues, nb_cty))
metro_thread.start()
# Loop on results
is_full = lambda x : len(x) == nb_cty
per_thread = defaultdict(list)
while True:
t_id, c = out_q.get()
out_q.task_done()
per_thread[t_id].append(c)
if all(map(is_full, per_thread.values())):
break
# Join everything
metro_thread.join()
for t in read_threads:
t.join()
class ThreadQueue(object):
def __init__(self):
self.q = Queue()
t = Thread(target=self._thread_worker)
t.setDaemon(True)
t.start()
def add_request(self, func, *args, **kwargs):
"""Add a request to the queue. Pass callback= and/or error= as
keyword arguments to receive return from functions or exceptions.
"""
self.q.put((func, args, kwargs))
def _thread_worker(self):
while True:
request = self.q.get()
self.do_request(request)
self.q.task_done()
def do_request(self, (func, args, kwargs)):
callback = kwargs.pop('callback', None)
error = kwargs.pop('error', None)
def run():
try:
r = func(*args, **kwargs)
if not isinstance(r, tuple):
r = (r,)
if callback:
self.do_callback(callback, *r)
except Exception, e:
if error:
tb = traceback.format_exception(type(e), e, sys.exc_traceback)
self.do_callback(error, *(e, tb))
else:
print "Unhandled error:", e
class MessageSender(Thread):
def __init__(self, s):
Thread.__init__(self)
self.daemon = True
self.s = s
self.q = Queue()
self.saved_exception = None
def sendall(self, msg):
if self.q == None:
if self.saved_exception != None:
raise self.saved_exception
else:
return
self.q.put(msg)
def clear_exception(self):
self.saved_exception = None
def exit(self):
if self.q != None:
self.q.put(SystemExit)
def run(self):
try:
while True:
msg = self.q.get()
if msg == None:
self.s.close()
self.q = None
break
if msg == SystemExit:
raise SystemExit
self.s.sendall(msg)
self.q.task_done()
except (socket.error, SystemExit) as e:
self.saved_exception = e
self.q = None # Release resources
self.s.close()
class Action(tornado.websocket.WebSocketHandler):
def __init__(self, *args, **kwargs):
self.q = Queue()
self.t = None
self.car = kwargs.pop('car')
super(Action, self).__init__(*args, **kwargs)
def check_origin(self, origin):
return True
def _execute_actions(self):
logging.debug("Action()._execute_action()")
global car # TODO: Handle in better way
while True:
message = self.q.get()
if message:
logging.debug("Message => " + message)
message = message.split(" ")
method_name = message[1]
if hasattr(self.car, method_name):
method = getattr(self.car, method_name)
method()
self.q.task_done()
else:
logging.error("Invalid method " + method_name)
else:
car.stop()
def on_message(self, message):
if not self.t:
logging.debug("Thread Create for execute action")
self.t = Thread(target=self._execute_actions)
self.t.start()
try:
logging.debug("Message recevied: " + message)
self.q.put(message)
except tornado.websocket.WebSocketClosedError:
pass
def test():
started = Queue()
n = 1000
threads = []
touched = []
pool = EmptyListPool()
rand = SystemRandom()
def _run():
psleep = rand.uniform(0.05, 0.1)
with pool.take() as a:
started.put(1)
started.join()
a.append(rand.uniform(0, 1))
if psleep > 1:
print psleep
sleep(psleep)
for i in range(n):
th = Thread(target=_run)
threads.append(th)
th.start()
for i in range(n):
started.get()
started.task_done()
for element in pool:
touched.append(element)
for thr in threads:
thr.join()
if set(pool.elements) != set(touched):
print set(pool.elements) - set(touched)
return False
else:
return True
class QueueProcessorThread(Thread):
"""Thread to get and process tasks from a queue until queue.done (custom attr) is True"""
def __init__(self,
action,
input_queue=None,
problem_list=None,
output_action=None,
**kargs):
super(QueueProcessorThread, self).__init__(**kargs)
self.action = action
self.input_queue = Queue() if input_queue is None else input_queue
self.problem_list = [] if problem_list is None else problem_list
self.output_action = output_action
def run(self):
while True:
try:
task = self.input_queue.get(block=False)
except Empty:
log.debug(
"[%s] No tasks found, done? %s" %
(self.name, getattr(self.input_queue, "done", False)))
if getattr(self.input_queue, "done", False):
break
time.sleep(.1)
continue
try:
result = self.action(task)
except:
log.exception(
"Exception on executing task {task}".format(**locals()))
self.problem_list.append(task) # list append is thread safe
else:
if self.output_action is not None:
self.output_action(result)
finally:
self.input_queue.task_done()
log.debug("[%s] Done!" % self.name)
class ThreadPool(object): def __init__(self,threadNum): #工作队列 self.workQueue = Queue() #结果队列 self.resultQueue = Queue() #线程池 self.threadPool = [] #线程数目 self.threadNum = threadNum #启动线程 def startThreads(self): for i in range(self.threadNum): self.threadPool.append(Worker(self)) #等待线程结束 def workJoin(self,*args,**kargs): self.workQueue.join() #添加工作任务 def addJob(self,func,*args,**kargs): self.workQueue.put((func,args,kargs)) #工作任务完成 def workDone(self,*args): self.workQueue.task_done() #获得结果 def getResult(self,*args,**kargs): return self.resultQueue.get(*args,**kargs) #结束线程 def stopThreads(self): for thread in self.threadPool: #thread.join() thread.stop() del self.threadPool[:]
class Threader:
"""Threader class.
Threader class is responsible for making multiple parallel requests
"""
def __init__(self, num_threads=5):
self.concurrent = num_threads
self.queue = Queue(num_threads * 2)
self.obj = None
def attach(self, obj):
self.obj = obj
def job(self):
while True:
url = self.queue.get()
if self.obj is not None:
response = self.obj.worker(url)
self.result(response)
self.queue.task_done()
def result(self, response):
try:
self.obj.result(response)
except Exception:
print "Exception occured"
def start(self):
for i in range(self.concurrent):
t = Thread(target=self.job)
t.daemon = True
t.start()
def submit(self):
try:
self.obj.prepare(self.queue)
except KeyboardInterrupt:
sys.exit(1)
class Worker:
def __init__(self):
self.q = Queue()
self.t = Thread(target=self._handle)
self.t.setDaemon(True)
self.t.start()
def _handle(self):
while True:
fn = self.q.get()
try:
fn()
self.q.task_done()
except:
import traceback
print traceback.format_exc()
def do(self, fn, *a, **kw):
fn1 = lambda: fn(*a, **kw)
self.q.put(fn1)
def join(self):
self.q.join()
class WorkerThread(Thread):
def __init__(self, *args, **kwargs):
Thread.__init__(self, *args, **kwargs)
self.input_queue = Queue()
def send(self, item):
self.input_queue.put(item)
def close(self):
self.input_queue.put(None)
self.input_queue.join()
print('close....')
def run(self):
while True:
item = self.input_queue.get()
print('.' * 50)
if item is None:
break
print(item)
self.input_queue.task_done()
self.input_queue.task_done()
return
class Parallel:
def __init__(self, ncpu, njobs):
self.running = Queue(ncpu)
self.returned = Queue()
self.njobs = njobs
def run(self, cmd, args):
wrap = Wrap(self, (cmd, args), self.returned)
thread = Thread(None, wrap)
thread.start()
def __call__(self, cmd, args):
if type(cmd) == str:
print cmd
for a in args:
cmd += " %s " % a
args = (cmd, )
cmd = commands.getstatusoutput
self.running.put((cmd, args))
ret = cmd(*args)
self.running.get()
self.running.task_done()
return ret
class on_thread_thread(BackgroundThread):
'yet another consumer thread'
def __init__(self, name, daemon=True):
BackgroundThread.__init__(self, name=name)
self.setDaemon(daemon)
self.work = Queue()
self.done = False
def run(self):
self.BeforeRun()
try:
SEHGuard(self._consumer_loop)
finally:
self.AfterRun()
def _consumer_loop(self):
while not self.done:
setattr(self, 'loopcount', getattr(self, 'loopcount', 0) + 1)
func, args, kwargs = self.work.get()
try:
func(*args, **kwargs)
except Exception:
print_exc()
self.work.task_done()
on_thread(self.name)._done()
def queue(self, func, *a, **k):
if __debug__:
import traceback
self.last_stack = traceback.format_stack()
self.work.put((func, a, k))
def join(self):
self.done = True
self.work.join()
class csvWriter:
def __init__(self, data, concurrentThreads):
self.concurrent = concurrentThreads
self.q = Queue(concurrentThreads)
self.dataset = data
self.lock = threading.Lock()
print("Outputing csv files..")
def createThreading(self,args,fileName,header):
for i in range(self.concurrent):
t = threading.Thread(target=self.writingcsv,args=(args,fileName,header))
t.daemon = True
t.start()
for data in self.dataset:
self.q.put(data)
self.q.join()
if(len(dataset)>0):
return True
return False
def writingcsv(self,args,fileName,header):
csvfile = open( fileName, 'wb')
writer = csv.writer(csvfile, delimiter=',', lineterminator='\r\n', quoting=csv.QUOTE_NONE, escapechar=' ')
writer.writerow(header)
with self.lock:
while not self.q.empty():
data = self.q.get()
writer.writerow(data)
self.q.task_done()
csvfile.close()
class SequentialScheduler(IScheduler):
def __init__(self):
self.rq = None
self.machine = None
def reschedule(self, machine):
self.machine = machine
self.rq = Queue(-1)
#programs = Program.query.all()
#log.debug("rescheduling, found %d programs." % len(programs))
# for program in programs:
# for obs in program.observations:
# for exp in obs.exposures:
# self.rq.put(exp)
exps = Exposure.query.filter_by(finished=False).all()
log.debug("rescheduling, found %d exposures." % len(exps))
for exp in exps:
self.rq.put(exp)
machine.wakeup()
def next(self):
if not self.rq.empty():
return self.rq.get()
def done(self, task):
task.finished = True
task.flush()
self.rq.task_done()
self.machine.wakeup()
def test_thread_safety(self):
"""
The pool should allocate n objects for n concurrent operations.
"""
n = 10
pool = EmptyListPool()
readyq = Queue()
finishq = Queue()
threads = []
def _run():
with pool.transaction() as resource:
readyq.put(1)
resource.append(currentThread())
finishq.get(True)
finishq.task_done()
for i in range(n):
th = Thread(target=_run)
threads.append(th)
th.start()
for i in range(n):
readyq.get()
readyq.task_done()
for i in range(n):
finishq.put(1)
for thr in threads:
thr.join()
self.assertEqual(n, len(pool.resources))
for resource in pool.resources:
self.assertFalse(resource.claimed)
self.assertEqual(1, len(resource.object))
self.assertIn(resource.object[0], threads)
class Threads(object):
def thread(self,*args):
thread_count = self.global_options['threads']
self.stopped = threading.Event()
self.q = Queue()
self.q.put(args[0])
threads = []
for i in range(thread_count):
t = threading.Thread(target=self.thread_wrapper,args=args[1:])
threads.append(t)
t.setDaemon(True)
t.start()
try:
while not self.q.empty():
time.sleep(2)
except KeyboardInterrupt:
warn2('Waiting for threads to exit...')
self.stopped.set()
for t in threads:
t.join()
raise
self.q.join()
self.stopped.set()
def thread_wrapper(self,*args):
thread_name = threading.current_thread().name
while not self.stopped.is_set():
try:
obj = self.q.get_nowait()
except Empty:
continue
try:
self.module_thread(obj,*args)
except Exception as e:
warn(e.message)
finally:
self.q.task_done()
class workerThread(threading.Thread):
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.input_queue = Queue()
def send(self, item):
self.input_queue.put_nowait(item)
def close(self):
self.input_queue.put_nowait(None)
self.input_queue.join()
def run(self):
while True:
# time.sleep(3)
item = self.input_queue.get(
) # error:if replaced with self.input_queue.get_nowait()
if item is None:
print 'thread is closed'
break
print item
self.input_queue.task_done()
self.input_queue.task_done()
return
class LocalVolatileTaskQueue(TaskQueueBase):
implements(ITaskQueue)
def __init__(self, **kwargs):
self.queue = Queue()
def __len__(self):
return self.queue.qsize()
def put(self, task):
self.queue.put(task, block=True)
def get(self, *args, **kwargs):
try:
return self.queue.get(block=False)
except Empty:
return None
def task_done(self, *args, **kwargs):
self.queue.task_done()
def reset(self):
self.queue = Queue()
