def parmake_job2(args): """ args = tuple job_id, context, queue_name, show_events Returns a dictionary with fields "user_object", "new_jobs", 'delete_jobs'. "user_object" is set to None because we do not want to load in our thread if not necessary. Sometimes it is necessary because it might contain a Promise. """ job_id, context, event_queue_name, show_output = args # @UnusedVariable check_isinstance(job_id, str) check_isinstance(event_queue_name, str) from .pmake_manager import PmakeManager event_queue = PmakeManager.queues[event_queue_name] db = context.get_compmake_db() setproctitle('compmake:%s' % job_id) class G(): nlostmessages = 0 try: # We register a handler for the events to be passed back # to the main process def handler( event): try: if not CompmakeConstants.disable_interproc_queue: event_queue.put(event, block=False) except Full: G.nlostmessages += 1 # Do not write messages here, it might create a recursive # problem. # sys.stderr.write('job %s: Queue is full, message is lost.\n' # % job_id) remove_all_handlers() if show_output: register_handler("*", handler) def proctitle(event): stat = '[%s/%s %s] (compmake)' % (event.progress, event.goal, event.job_id) setproctitle(stat) register_handler("job-progress", proctitle) publish(context, 'worker-status', job_id=job_id, status='started') # Note that this function is called after the fork. # All data is conserved, but resources need to be reopened try: db.reopen_after_fork() # @UndefinedVariable except: pass publish(context, 'worker-status', job_id=job_id, status='connected') res = make(job_id, context=context) publish(context, 'worker-status', job_id=job_id, status='ended') res['user_object'] = None result_dict_check(res) return res except KeyboardInterrupt: assert False, 'KeyboardInterrupt should be captured by make() (' \ 'inside Job.compute())' except JobInterrupted: publish(context, 'worker-status', job_id=job_id, status='interrupted') raise except JobFailed: raise except BaseException: # XXX raise except: raise finally: publish(context, 'worker-status', job_id=job_id, status='cleanup') setproctitle('compmake-worker-finished')
def parmake_job2(args): """ args = tuple job_id, context, queue_name, show_events Returns a dictionary with fields "user_object", "new_jobs", 'delete_jobs'. "user_object" is set to None because we do not want to load in our thread if not necessary. Sometimes it is necessary because it might contain a Promise. """ job_id, context, event_queue_name, show_output = args # @UnusedVariable check_isinstance(job_id, str) check_isinstance(event_queue_name, str) from .pmake_manager import PmakeManager event_queue = PmakeManager.queues[event_queue_name] db = context.get_compmake_db() setproctitle('compmake:%s' % job_id) class G(): nlostmessages = 0 try: # We register a handler for the events to be passed back # to the main process def handler(event): try: if not CompmakeConstants.disable_interproc_queue: event_queue.put(event, block=False) except Full: G.nlostmessages += 1 # Do not write messages here, it might create a recursive # problem. # sys.stderr.write('job %s: Queue is full, message is lost.\n' # % job_id) remove_all_handlers() if show_output: register_handler("*", handler) def proctitle(event): stat = '[%s/%s %s] (compmake)' % (event.progress, event.goal, event.job_id) setproctitle(stat) register_handler("job-progress", proctitle) publish(context, 'worker-status', job_id=job_id, status='started') # Note that this function is called after the fork. # All data is conserved, but resources need to be reopened try: db.reopen_after_fork() # @UndefinedVariable except: pass publish(context, 'worker-status', job_id=job_id, status='connected') res = make(job_id, context=context) publish(context, 'worker-status', job_id=job_id, status='ended') res['user_object'] = None result_dict_check(res) return res except KeyboardInterrupt: assert False, 'KeyboardInterrupt should be captured by make() (' \ 'inside Job.compute())' except JobInterrupted: publish(context, 'worker-status', job_id=job_id, status='interrupted') raise except JobFailed: raise except BaseException: # XXX raise except: raise finally: publish(context, 'worker-status', job_id=job_id, status='cleanup') setproctitle('compmake-worker-finished %s' % job_id)
def main(): setproctitle('compmake') parser = OptionParser(version=version) parser.add_option("--slave", action="store_true", default=False, dest="slave", help="[internal] Runs compmake in slave mode.") parser.add_option("--redis_events", action="store_true", default=False, dest="redis_events", help="[internal] Relays events using Redis.") config_populate_optparser(parser) (options, args) = parser.parse_args() initialize_backend() # We load plugins after we parsed the configuration from compmake import plugins #@UnusedImport if options.redis_events: if not compmake_config.db == 'redis': #@UndefinedVariable error('Cannot use redis_events without redis.') sys.exit(-2) from compmake.storage.redisdb import RedisInterface # register an handler that will capture all events def handler(event): RedisInterface.events_push(event) remove_all_handlers() register_handler("*", handler) if not options.slave: # XXX make sure this is the default set_compmake_status(compmake_status_interactive) # TODO: add command namespace # TODO: add command "load" if not args: user_error('I expect at least one parameter (module name)') sys.exit(-2) module_name = args[0] args = args[1:] if module_name.endswith('.py') or (module_name.find('/') > 0): warning('You passed a string "%s" which looks like a filename.' % module_name) module_name = module_name.replace('/', '.') module_name = module_name.replace('.py', '') warning('However, I need a module name. I will try with "%s".' % module_name) set_namespace(module_name) compmake.is_it_time = True try: __import__(module_name) except Exception as e: error('Error while trying to import module "%s": %s' % (module_name, e)) traceback.print_exc(file=sys.stderr) sys.exit(-5) # TODO: BUG: XXX: remove old jobs those in defined_this_section else: set_compmake_status(compmake_status_slave) if not args: user_error('I expect at least one parameter (namespace name)') sys.exit(-2) module_name = args.pop(0) set_namespace(module_name) if args: try: # XXX is this redudant? # compmake_config.interactive = False retcode = interpret_commands(args) # print "Exiting with retcode %s" % retcode sys.exit(retcode) except UserError as e: user_error(e) sys.exit(-6) else: retcode = interactive_console() sys.exit(retcode)