def monitor(self, workers):
finished_workers = []
failed_workers = []
for worker in workers:
exitcode = worker.exitcode
if exitcode is None: # the process is running
continue
elif exitcode >= 1: # the process failed
failed_workers.append(worker)
else: # the process has finished (0) or was interrupted (<0)
finished_workers.append(worker)
# remove finished workers
for worker in finished_workers:
log.info(' [{}] Worker stopped'.format(worker.id))
workers.remove(worker)
# restart failed workers
for worker in failed_workers:
log.info(' [{}] Worker failed'.format(worker.id))
workers.remove(worker)
new_worker = Worker(queue_name=worker.queue_name,
worker_url=worker.worker_url)
workers.append(new_worker)
log.info(' [{}] Starting worker'.format(new_worker.id))
new_worker.start()
def start_workers_for_grader_url(cls, queue_name,
destination_url, num_workers=1):
"""We need to start workers (consumers) to pull messages
from the queue and pass them to our passive grader.
`queue_name`: The name of the queue to pull messages from (string)
`destination_url`: The url to forward responses to.
`num_workers`: The number of workers to start for this queue (int)
Raises an `AssertionError` if trying to start workers before
stopping the current workers."""
if hasattr(cls, 'worker_list'):
assert(len(cls.worker_list) > 0)
else:
cls.worker_list = []
for i in range(num_workers):
worker = Worker(queue_name=queue_name, worker_url=destination_url)
# There is a bug in pika on Mac OS X
# in which using multithreading.Process with
# pika's ioloop causes an IncompatibleProtocolError
# to be raised.
# The workaround for now is to run each worker
# as a separate thread.
worker_thread = threading.Thread(target=worker.run)
worker_thread.daemon = True
worker_thread.start()
cls.worker_list.append(worker)
def start_workers_for_grader_url(cls,
queue_name,
destination_url,
num_workers=1):
"""We need to start workers (consumers) to pull messages
from the queue and pass them to our passive grader.
`queue_name`: The name of the queue to pull messages from (string)
`destination_url`: The url to forward responses to.
`num_workers`: The number of workers to start for this queue (int)
Raises an `AssertionError` if trying to start workers before
stopping the current workers."""
if hasattr(cls, 'worker_list'):
assert (len(cls.worker_list) > 0)
else:
cls.worker_list = []
for i in range(num_workers):
worker = Worker(queue_name=queue_name, worker_url=destination_url)
worker.start()
cls.worker_list.append(worker)
def handle(self, *args, **options):
log.info(' [*] Starting queue consumers...')
workers = []
queues = settings.XQUEUES.items()
# Assigned one worker for queue
for name, url in queues:
if url is not None:
worker = Worker(queue_name=name, worker_url=url)
worker.start()
workers.append(worker)
for worker in workers:
worker.join()
log.info(' [*] All workers finished. Exiting')
def handle(self, *args, **options):
log.info(' [*] Starting queue workers...')
workers = []
queues = settings.XQUEUES.items()
# Assigned one worker for queue
for name, url in queues:
if url is not None:
worker = Worker(queue_name=name, worker_url=url)
workers.append(worker)
# Start workers
for worker in workers:
log.info(' [{}] Starting worker'.format(worker.id))
worker.start()
# Monitor workers
while workers:
self.monitor(workers)
time.sleep(MONITOR_SLEEPTIME)
log.info(' [*] All workers finished. Exiting')
def setUp(self):
self.worker = Worker(queue_name=self.QUEUE_NAME,
worker_url='some_test_url')
