def loop(config, jobloader): # unique name for restart redisclient = sparqueue.redis.client(config) queue_manager = sparqueue.queue.QueueManager(config, redisclient) queue_manager.add_list(config['queues']) while RUN: while RUN: try: (queue, job) = queue_manager.pop() break except sparqueue.queue.QueueException, e: # TODO: want probably to spawn this in # another cron process instead requeued = queue_manager.requeue() if len(requeued) > 1: print 'Requeued %s' % ','.join(requeued) # exit before continuing on when not running if not RUN: break logger.info(job) jobid = job['metadata']['jobid'] reporter = sparqueue.reporter.Reporter(logger, queue, jobid) logger.info('Processing jobid %s' % jobid) if 'install' in job: if 'pip' in job['install']: reporter.step('installing %s' % job['install']['pip']) try: logger.info(sh.pip('install', job['install']['pip'])) except sh.ErrorReturnCode, e: logger.error('problem installing %s: %s' % ( job['metadata']['jobid'], traceback.format_exc(e))) queue.failed(e) continue
jobid = job['metadata']['jobid'] reporter = sparqueue.reporter.Reporter(logger, queue, jobid) logger.info('Processing jobid %s' % jobid) if 'install' in job: if 'pip' in job['install']: reporter.step('installing %s' % job['install']['pip']) try: logger.info(sh.pip('install', job['install']['pip'])) except sh.ErrorReturnCode, e: logger.error('problem installing %s: %s' % ( job['metadata']['jobid'], traceback.format_exc(e))) queue.failed(e) continue try: jobInstance = jobloader.getInstance(job['class'], config) if not jobInstance: logger.error('Invalid class name: %s' % job['class']) job['vars']['reporter'] = reporter output = jobInstance.perform(**job['vars']) reporter.finish() queue.success(output, reporter.stats()) except Exception, e: logger.error('problem processing %s: %s' % ( job['metadata']['jobid'], traceback.format_exc(e))) queue.failed(e) logger.info('Exiting gracefully, no current queue jobs left unprocessed') logger.info('Exited: %s' % queue_manager.exit())