def main():
parser = argparse.ArgumentParser(
description='Start Pinball master server.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-p',
'--port',
dest='port',
type=int,
default=PinballConfig.MASTER_PORT,
help='port to run on')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
master_port = options.port if options.port else PinballConfig.MASTER_PORT
factory = Factory(master_port=master_port)
# The reason why these imports are not at the top level is that some of the
# imported code (db models initializing table names) depends on parameters
# passed on the command line (master name). Those imports need to be delayed
# until after command line parameter parsing.
from pinball.persistence.store import DbStore
factory.create_master(DbStore())
factory.run_master_server()
def main():
parser = argparse.ArgumentParser(
description='Start Pinball master server.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-p',
'--port',
dest='port',
type=int,
default=PinballConfig.MASTER_PORT,
help='port to run on')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
master_port = options.port if options.port else PinballConfig.MASTER_PORT
factory = Factory(master_port=master_port)
# The reason why these imports are not at the top level is that some of the
# imported code (db models initializing table names) depends on parameters
# passed on the command line (master name). Those imports need to be delayed
# until after command line parameter parsing.
from pinball.persistence.store import DbStore
factory.create_master(DbStore())
factory.run_master_server()
def get_workflow_jobs_from_parser(workflow):
config_parser = load_parser_with_caller(PinballConfig.PARSER,
PinballConfig.get_parser_params(workflow),
ParserCaller.UI)
tokens = config_parser.get_workflow_tokens(workflow)
jobs_data = []
for token in tokens:
name = Name.from_job_token_name(token.name)
if name.job:
assert name.workflow == workflow
job = pickle.loads(token.data)
jobs_data.append(JobData(workflow=workflow,
instance=None,
job=name.job,
job_type=job.__class__.__name__,
is_condition=job.IS_CONDITION,
info=job.info(),
inputs=job.inputs,
outputs=job.outputs,
emails=job.emails,
max_attempts=job.max_attempts,
retry_delay_sec=job.retry_delay_sec,
warn_timeout_sec=job.warn_timeout_sec,
abort_timeout_sec=job.abort_timeout_sec,
priority=token.priority,
status=Status.NEVER_RUN))
return jobs_data
def _read_tokens_from_parser_params(self,workflow):
"""Read archived job tokens from the PinballConfig.PARSER_PARAMS.
"""
config_parser = load_parser_with_caller(PinballConfig.PARSER,
PinballConfig.get_parser_params(workflow),
ParserCaller.ANALYZER)
tokens = config_parser.get_workflow_tokens(self._workflow)
self._filter_job_tokens(tokens)
def __init__(
self,
next_run_time=None,
recurrence_seconds=None,
overrun_policy=OverrunPolicy.SKIP,
parser_params=None,
workflow=None,
emails=None,
max_running_instances=None):
Schedule.__init__(self, next_run_time, recurrence_seconds,
overrun_policy)
if parser_params is None:
parser_params = PinballConfig.get_parser_params(workflow)
self.parser_params = annotate_parser_caller(
parser_params,
ParserCaller.SCHEDULE)
self.workflow = workflow
self.emails = emails if emails is not None else []
self.max_running_instances = max_running_instances if max_running_instances \
else PinballConfig.DEFAULT_MAX_WORKFLOW_RUNNING_INSTANCES
def main():
_register_signal_listener()
parser = argparse.ArgumentParser(
description='Start Pinball master and workers.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-m',
'--mode',
dest='mode',
choices=['master', 'scheduler', 'workers', 'ui'],
default='master',
help='execution mode')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME:
master_name(PinballConfig.MASTER_NAME)
_pinball_imports()
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
if options.mode == 'ui':
hostport = '%s:%d' % (socket.gethostname(), PinballConfig.UI_PORT)
cache_thread.start_cache_thread(DbStore())
if not PinballConfig.UI_HOST:
hostport = 'localhost:%d' % PinballConfig.UI_PORT
# Disable reloader to prevent auto refresh on file changes. The
# problem with auto-refresh is that it starts multiple processes. Some
# of those processes will become orphans if we kill the UI in a wrong
# way.
management.call_command('runserver', hostport, interactive=False,
use_reloader=False)
return
factory = Factory(master_hostname=PinballConfig.MASTER_HOST,
master_port=PinballConfig.MASTER_PORT)
threads = []
if options.mode == 'master':
factory.create_master(DbStore())
elif options.mode == 'scheduler':
threads.append(_create_scheduler(factory, emailer))
else:
assert options.mode == 'workers'
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
threads = _create_workers(PinballConfig.WORKERS, factory, emailer)
try:
if options.mode == 'master':
factory.run_master_server()
else:
_wait_for_threads(threads)
except KeyboardInterrupt:
LOG.info('Exiting')
sys.exit()
