def main(): parser = argparse.ArgumentParser( description='Start Pinball master server.') parser.add_argument( '-c', '--config_file', dest='config_file', required=True, help='full path to the pinball setting configure file') parser.add_argument( '-p', '--port', dest='port', type=int, default=PinballConfig.MASTER_PORT, help='port to run on') options = parser.parse_args(sys.argv[1:]) PinballConfig.parse(options.config_file) master_port = options.port if options.port else PinballConfig.MASTER_PORT factory = Factory(master_port=master_port) # The reason why these imports are not at the top level is that some of the # imported code (db models initializing table names) depends on parameters # passed on the command line (master name). Those imports need to be delayed # until after command line parameter parsing. from pinball.persistence.store import DbStore factory.create_master(DbStore()) factory.run_master_server()
def get_workflow_jobs_from_parser(workflow): config_parser = load_parser_with_caller(PinballConfig.PARSER, PinballConfig.get_parser_params(workflow), ParserCaller.UI) tokens = config_parser.get_workflow_tokens(workflow) jobs_data = [] for token in tokens: name = Name.from_job_token_name(token.name) if name.job: assert name.workflow == workflow job = pickle.loads(token.data) jobs_data.append(JobData(workflow=workflow, instance=None, job=name.job, job_type=job.__class__.__name__, is_condition=job.IS_CONDITION, info=job.info(), inputs=job.inputs, outputs=job.outputs, emails=job.emails, max_attempts=job.max_attempts, retry_delay_sec=job.retry_delay_sec, warn_timeout_sec=job.warn_timeout_sec, abort_timeout_sec=job.abort_timeout_sec, priority=token.priority, status=Status.NEVER_RUN)) return jobs_data
def _read_tokens_from_parser_params(self,workflow): """Read archived job tokens from the PinballConfig.PARSER_PARAMS. """ config_parser = load_parser_with_caller(PinballConfig.PARSER, PinballConfig.get_parser_params(workflow), ParserCaller.ANALYZER) tokens = config_parser.get_workflow_tokens(self._workflow) self._filter_job_tokens(tokens)
def __init__( self, next_run_time=None, recurrence_seconds=None, overrun_policy=OverrunPolicy.SKIP, parser_params=None, workflow=None, emails=None, max_running_instances=None): Schedule.__init__(self, next_run_time, recurrence_seconds, overrun_policy) if parser_params is None: parser_params = PinballConfig.get_parser_params(workflow) self.parser_params = annotate_parser_caller( parser_params, ParserCaller.SCHEDULE) self.workflow = workflow self.emails = emails if emails is not None else [] self.max_running_instances = max_running_instances if max_running_instances \ else PinballConfig.DEFAULT_MAX_WORKFLOW_RUNNING_INSTANCES
def main(): _register_signal_listener() parser = argparse.ArgumentParser( description='Start Pinball master and workers.') parser.add_argument( '-c', '--config_file', dest='config_file', required=True, help='full path to the pinball setting configure file') parser.add_argument( '-m', '--mode', dest='mode', choices=['master', 'scheduler', 'workers', 'ui'], default='master', help='execution mode') options = parser.parse_args(sys.argv[1:]) PinballConfig.parse(options.config_file) if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME: master_name(PinballConfig.MASTER_NAME) _pinball_imports() if PinballConfig.UI_HOST: emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT) else: emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT) if options.mode == 'ui': hostport = '%s:%d' % (socket.gethostname(), PinballConfig.UI_PORT) cache_thread.start_cache_thread(DbStore()) if not PinballConfig.UI_HOST: hostport = 'localhost:%d' % PinballConfig.UI_PORT # Disable reloader to prevent auto refresh on file changes. The # problem with auto-refresh is that it starts multiple processes. Some # of those processes will become orphans if we kill the UI in a wrong # way. management.call_command('runserver', hostport, interactive=False, use_reloader=False) return factory = Factory(master_hostname=PinballConfig.MASTER_HOST, master_port=PinballConfig.MASTER_PORT) threads = [] if options.mode == 'master': factory.create_master(DbStore()) elif options.mode == 'scheduler': threads.append(_create_scheduler(factory, emailer)) else: assert options.mode == 'workers' if PinballConfig.UI_HOST: emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT) else: emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT) threads = _create_workers(PinballConfig.WORKERS, factory, emailer) try: if options.mode == 'master': factory.run_master_server() else: _wait_for_threads(threads) except KeyboardInterrupt: LOG.info('Exiting') sys.exit()