def loop(config, jobloader):
# unique name for restart
redisclient = sparqueue.redis.client(config)
queue_manager = sparqueue.queue.QueueManager(config, redisclient)
queue_manager.add_list(config['queues'])
while RUN:
while RUN:
try:
(queue, job) = queue_manager.pop()
break
except sparqueue.queue.QueueException, e:
# TODO: want probably to spawn this in
# another cron process instead
requeued = queue_manager.requeue()
if len(requeued) > 1:
print 'Requeued %s' % ','.join(requeued)
# exit before continuing on when not running
if not RUN:
break
logger.info(job)
jobid = job['metadata']['jobid']
reporter = sparqueue.reporter.Reporter(logger, queue, jobid)
logger.info('Processing jobid %s' % jobid)
if 'install' in job:
if 'pip' in job['install']:
reporter.step('installing %s' % job['install']['pip'])
try:
logger.info(sh.pip('install', job['install']['pip']))
except sh.ErrorReturnCode, e:
logger.error('problem installing %s: %s' % (
job['metadata']['jobid'], traceback.format_exc(e)))
queue.failed(e)
continue
jobid = job['metadata']['jobid']
reporter = sparqueue.reporter.Reporter(logger, queue, jobid)
logger.info('Processing jobid %s' % jobid)
if 'install' in job:
if 'pip' in job['install']:
reporter.step('installing %s' % job['install']['pip'])
try:
logger.info(sh.pip('install', job['install']['pip']))
except sh.ErrorReturnCode, e:
logger.error('problem installing %s: %s' % (
job['metadata']['jobid'], traceback.format_exc(e)))
queue.failed(e)
continue
try:
jobInstance = jobloader.getInstance(job['class'], config)
if not jobInstance:
logger.error('Invalid class name: %s' % job['class'])
job['vars']['reporter'] = reporter
output = jobInstance.perform(**job['vars'])
reporter.finish()
queue.success(output, reporter.stats())
except Exception, e:
logger.error('problem processing %s: %s' % (
job['metadata']['jobid'], traceback.format_exc(e)))
queue.failed(e)
logger.info('Exiting gracefully, no current queue jobs left unprocessed')
logger.info('Exited: %s' % queue_manager.exit())