def main():
parser = argparse.ArgumentParser(
description='Start Pinball master server.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-p',
'--port',
dest='port',
type=int,
default=PinballConfig.MASTER_PORT,
help='port to run on')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
master_port = options.port if options.port else PinballConfig.MASTER_PORT
factory = Factory(master_port=master_port)
# The reason why these imports are not at the top level is that some of the
# imported code (db models initializing table names) depends on parameters
# passed on the command line (master name). Those imports need to be delayed
# until after command line parameter parsing.
from pinball.persistence.store import DbStore
factory.create_master(DbStore())
factory.run_master_server()
def get_context_data(self, **kwargs):
context = super(TokenView, self).get_context_data(**kwargs)
token_name = self.request.GET['path']
data_builder = DataBuilder(DbStore())
token_data = data_builder.get_token(token_name)
token_format = token_data.format()
for key, value in token_format.items():
context[key] = value
return context
def get_context_data(self, **kwargs):
context = super(ScheduleView, self).get_context_data(**kwargs)
workflow = self.request.GET['workflow']
data_builder = DataBuilder(DbStore())
schedule_data = data_builder.get_schedule(workflow)
formatted_schedule = schedule_data.format()
for key, value in formatted_schedule.items():
context[key] = value
context['emails'] = ' '.join(schedule_data.emails)
return context
def schedules(_):
try:
data_builder = DataBuilder(DbStore())
schedules_data = data_builder.get_schedules()
schedules_json = _serialize(schedules_data)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(schedules_json, mimetype='application/json')
def instances(request):
try:
workflow = request.GET['workflow']
data_builder = DataBuilder(DbStore(), use_cache=True)
instances_data = data_builder.get_instances(workflow)
instances_json = _serialize(instances_data)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(instances_json, mimetype='application/json')
def token_paths(request):
try:
path = request.GET['path']
data_builder = DataBuilder(DbStore())
tokens_data = data_builder.get_token_paths(path)
tokens_json = _serialize(tokens_data)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(tokens_json, mimetype='application/json')
def jobs(request):
try:
data_builder = DataBuilder(DbStore(), use_cache=True)
workflow = request.GET['workflow']
instance = request.GET['instance']
if instance == 'latest':
instance = data_builder.get_latest_instance(workflow).instance
jobs_data = data_builder.get_jobs(workflow, instance)
jobs_json = _serialize(jobs_data)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(jobs_json, mimetype='application/json')
def file_content(request):
try:
workflow = request.GET['workflow']
instance = request.GET['instance']
job = request.GET['job']
execution = int(request.GET['execution'])
log_type = request.GET['log_type']
if execution < 0:
return HttpResponseServerError(
'execution must not be negative; got ' + execution)
data_builder = DataBuilder(DbStore())
file_data = data_builder.get_file_content(workflow, instance, job,
execution, log_type)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(file_data, mimetype='text/plain')
def executions(request):
try:
workflow = request.GET['workflow']
instance = request.GET.get('instance')
job = request.GET['job']
data_builder = DataBuilder(DbStore())
if instance:
executions_data = data_builder.get_executions(
workflow, instance, job)
else:
executions_data = data_builder.get_executions_across_instances(
workflow, job)
executions_json = _serialize(executions_data)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(executions_json, content_type='application/json')
def graph(request):
try:
data_builder = DataBuilder(DbStore(), use_cache=True)
workflow = request.GET['workflow']
if 'instance' in request.GET:
instance = request.GET['instance']
if instance == 'latest':
instance = data_builder.get_latest_instance(workflow).instance
jobs_data = data_builder.get_jobs(workflow=workflow,
instance=instance)
instance_data = data_builder.get_instance(workflow=workflow,
instance=instance)
workflow_graph = WorkflowGraph(jobs_data, instance_data)
else:
workflow_graph = WorkflowGraph.from_parser(workflow)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(workflow_graph.get_svg(), mimetype='image/svg+xml')
def status(request):
try:
workflow = request.GET.get('workflow')
instance = request.GET.get('instance')
data_builder = DataBuilder(DbStore())
status = []
if data_builder.is_signal_set(workflow, instance, Signal.EXIT):
status = ['exiting']
elif data_builder.is_signal_set(workflow, instance, Signal.ABORT):
status = ['aborting']
elif data_builder.is_signal_set(workflow, instance, Signal.DRAIN):
status = ['draining']
if not _is_master_alive():
status.append('no master at %s:%d' % (socket.gethostname(),
PinballConfig.MASTER_PORT))
status_json = json.dumps(status)
except:
LOG.exception('')
return HttpResponseServerError(traceback.format_exc())
else:
return HttpResponse(status_json, mimetype='application/json')
def get_context_data(self, **kwargs):
context = super(ExecutionView, self).get_context_data(**kwargs)
workflow = self.request.GET['workflow']
instance = self.request.GET['instance']
job = self.request.GET['job']
execution = int(self.request.GET['execution'])
data_builder = DataBuilder(DbStore())
execution_data = data_builder.get_execution(workflow, instance, job,
execution)
formatted_data = execution_data.format()
for key, value in formatted_data.items():
context[key] = value
properties = []
for key, value in execution_data.properties.items():
properties.append('%s=%s' % (key, value))
context['properties'] = ', '.join(properties)
if not execution_data.end_time:
context['end_time'] = ''
if execution_data.exit_code is None:
context['exit_code'] = ''
return context
def main():
parser = argparse.ArgumentParser(
description='Start Pinball master server.')
parser.add_argument('-w',
'--active_workflows',
dest='active_workflows',
type=int,
default=5,
help='number of active workflows to generate')
parser.add_argument('-r',
'--archived_workflows',
dest='archived_workflows',
type=int,
default=5,
help='number of archived workflows to generate')
parser.add_argument('-i',
'--instances',
dest='instances',
type=int,
default=5,
help='number of instances per workflow')
parser.add_argument('-j',
'--jobs',
dest='jobs',
type=int,
default=5,
help='number of jobs per workflow')
parser.add_argument('-e',
'--executions',
dest='executions',
type=int,
default=5,
help='number of executions per job')
options = parser.parse_args(sys.argv[1:])
generate_workflows(options.active_workflows, options.archived_workflows,
options.instances, options.jobs, options.executions,
DbStore())
def _run_scheduler(factory, emailer):
client = factory.get_client()
scheduler = Scheduler(client, DbStore(), emailer)
scheduler.run()
def main():
_register_signal_listener()
parser = argparse.ArgumentParser(
description='Start Pinball master and workers.')
parser.add_argument(
'-c',
'--config_file',
dest='config_file',
required=True,
help='full path to the pinball setting configure file')
parser.add_argument(
'-m',
'--mode',
dest='mode',
choices=['master', 'scheduler', 'workers', 'ui'],
default='master',
help='execution mode')
options = parser.parse_args(sys.argv[1:])
PinballConfig.parse(options.config_file)
if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME:
master_name(PinballConfig.MASTER_NAME)
_pinball_imports()
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
if options.mode == 'ui':
hostport = '%s:%d' % (socket.gethostname(), PinballConfig.UI_PORT)
cache_thread.start_cache_thread(DbStore())
if not PinballConfig.UI_HOST:
hostport = 'localhost:%d' % PinballConfig.UI_PORT
# Disable reloader to prevent auto refresh on file changes. The
# problem with auto-refresh is that it starts multiple processes. Some
# of those processes will become orphans if we kill the UI in a wrong
# way.
management.call_command('runserver', hostport, interactive=False,
use_reloader=False)
return
factory = Factory(master_hostname=PinballConfig.MASTER_HOST,
master_port=PinballConfig.MASTER_PORT)
threads = []
if options.mode == 'master':
factory.create_master(DbStore())
elif options.mode == 'scheduler':
threads.append(_create_scheduler(factory, emailer))
else:
assert options.mode == 'workers'
if PinballConfig.UI_HOST:
emailer = Emailer(PinballConfig.UI_HOST, PinballConfig.UI_PORT)
else:
emailer = Emailer(socket.gethostname(), PinballConfig.UI_PORT)
threads = _create_workers(PinballConfig.WORKERS, factory, emailer)
try:
if options.mode == 'master':
factory.run_master_server()
else:
_wait_for_threads(threads)
except KeyboardInterrupt:
LOG.info('Exiting')
sys.exit()
def _run_worker(factory, emailer):
client = factory.get_client()
worker = Worker(client, DbStore(), emailer)
worker.run()