def monitor(self, workers):
finished_workers = []
failed_workers = []
for worker in workers:
exitcode = worker.exitcode
if exitcode is None: # the process is running
continue
elif exitcode >= 1: # the process failed
failed_workers.append(worker)
else: # the process has finished (0) or was interrupted (<0)
finished_workers.append(worker)
# remove finished workers
for worker in finished_workers:
log.info(' [{}] Worker stopped'.format(worker.queue_name))
workers.remove(worker)
# restart failed workers
for worker in failed_workers:
log.info(' [{}] Worker failed'.format(worker.queue_name))
workers.remove(worker)
new_worker = Worker(queue_name=worker.queue_name, worker_url=worker.worker_url)
workers.append(new_worker)
log.info(' [{}] Starting worker'.format(new_worker.queue_name))
new_worker.start()
def start_workers_for_grader_url(cls,
queue_name,
destination_url,
num_workers=1):
"""We need to start workers (consumers) to pull messages
from the queue and pass them to our passive grader.
`queue_name`: The name of the queue to pull messages from (string)
`destination_url`: The url to forward responses to.
`num_workers`: The number of workers to start for this queue (int)
Raises an `AssertionError` if trying to start workers before
stopping the current workers."""
if hasattr(cls, 'worker_list'):
assert (len(cls.worker_list) > 0)
else:
cls.worker_list = []
for i in range(num_workers):
worker = Worker(queue_name=queue_name, worker_url=destination_url)
worker.start()
cls.worker_list.append(worker)
def handle(self, *args, **options):
log.info(' [*] Starting queue workers...')
workers = []
queues = list(settings.XQUEUES.items())
# Assigned one worker for queue
for name, url in queues:
if url is not None:
worker = Worker(queue_name=name, worker_url=url)
workers.append(worker)
# Start workers
for worker in workers:
log.info(' [{}] Starting worker'.format(worker.queue_name))
worker.start()
# Monitor workers
while workers:
self.monitor(workers)
time.sleep(MONITOR_SLEEPTIME)
log.info(' [*] All workers finished. Exiting')