def remote_run_start(args):
system_paasta_config, service, cluster, \
soa_dir, instance, instance_type = extract_args(args)
overrides_dict = {}
constraints_json = args.constraints_json
if constraints_json:
try:
constraints = json.loads(constraints_json)
except Exception as e:
paasta_print("Error while parsing constraints: %s", e)
if constraints:
overrides_dict['constraints'] = constraints
if args.cmd:
overrides_dict['cmd'] = args.cmd
if args.instances:
overrides_dict['instances'] = args.instances
run_id = args.run_id
if run_id is None:
run_id = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8))
paasta_print("Assigned random run-id: %s" % run_id)
if args.detach:
paasta_print("Running in background")
if os.fork() > 0:
return
os.setsid()
if os.fork() > 0:
return
sys.stdout = open('/dev/null', 'w')
sys.stderr = open('/dev/null', 'w')
paasta_print('Scheduling a task on Mesos')
processor = TaskProcessor()
processor.load_plugin(provider_module='task_processing.plugins.mesos')
processor.load_plugin(provider_module='task_processing.plugins.stateful')
MesosExecutor = processor.executor_cls(provider='mesos')
native_job_config = load_paasta_native_job_config(
service,
instance,
cluster,
soa_dir=soa_dir,
instance_type=instance_type,
config_overrides=overrides_dict,
load_deployments=not args.docker_image,
)
try:
task_config = MesosExecutor.TASK_CONFIG_INTERFACE(
**paasta_to_task_config_kwargs(
service=service,
instance=instance,
system_paasta_config=system_paasta_config,
native_job_config=native_job_config,
config_overrides=overrides_dict,
docker_image=args.docker_image,
offer_timeout=args.staging_timeout,
), )
except InvariantException as e:
if len(e.missing_fields) > 0:
paasta_print(
PaastaColors.red(
"Mesos task config is missing following fields: {}".format(
', '.join(e.missing_fields), ), ), )
elif len(e.invariant_errors) > 0:
paasta_print(
PaastaColors.red(
"Mesos task config is failing following checks: {}".format(
', '.join(str(ie)
for ie in e.invariant_errors), ), ), )
else:
paasta_print(PaastaColors.red(f"Mesos task config error: {e}"), )
traceback.print_exc()
emit_counter_metric('paasta.remote_run.start.failed', service,
instance)
sys.exit(1)
except PTypeError as e:
paasta_print(
PaastaColors.red(
f"Mesos task config is failing a type check: {e}", ), )
traceback.print_exc()
emit_counter_metric('paasta.remote_run.start.failed', service,
instance)
sys.exit(1)
def handle_interrupt(_signum, _frame):
paasta_print(
PaastaColors.red("Signal received, shutting down scheduler."), )
if runner is not None:
runner.stop()
if _signum == signal.SIGTERM:
sys.exit(143)
else:
sys.exit(1)
signal.signal(signal.SIGINT, handle_interrupt)
signal.signal(signal.SIGTERM, handle_interrupt)
default_role = system_paasta_config.get_remote_run_config().get(
'default_role')
assert default_role
try:
executor_stack = build_executor_stack(
processor=processor,
service=service,
instance=instance,
role=native_job_config.get_role() or default_role,
pool=native_job_config.get_pool(),
cluster=cluster,
run_id=run_id,
system_paasta_config=system_paasta_config,
framework_staging_timeout=args.staging_timeout,
)
runner = Sync(executor_stack)
terminal_event = runner.run(task_config)
runner.stop()
except (Exception, ValueError) as e:
paasta_print("Except while running executor stack: %s", e)
traceback.print_exc()
emit_counter_metric('paasta.remote_run.start.failed', service,
instance)
sys.exit(1)
if terminal_event.success:
paasta_print("Task finished successfully")
sys.exit(0)
else:
paasta_print(PaastaColors.red(f"Task failed: {terminal_event.raw}"), )
# This is not necessarily an infrastructure failure. It may just be a
# application failure.
emit_counter_metric('paasta.remote_run.start.failed', service,
instance)
sys.exit(1)
def remote_run_start(args):
system_paasta_config, service, cluster, soa_dir, instance, instance_type = extract_args(args)
overrides_dict = {}
constraints_json = args.constraints_json
if constraints_json:
try:
constraints = json.loads(constraints_json)
except Exception as e:
paasta_print("Error while parsing constraints: %s", e)
if constraints:
overrides_dict['constraints'] = constraints
if args.cmd:
overrides_dict['cmd'] = args.cmd
if args.instances:
overrides_dict['instances'] = args.instances
run_id = args.run_id
if run_id is None:
run_id = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(8)
)
paasta_print("Assigned random run-id: %s" % run_id)
if args.detach:
paasta_print("Running in background")
if os.fork() > 0:
return
os.setsid()
if os.fork() > 0:
return
sys.stdout = open('/dev/null', 'w')
sys.stderr = open('/dev/null', 'w')
paasta_print('Scheduling a task on Mesos')
processor = TaskProcessor()
processor.load_plugin(provider_module='task_processing.plugins.mesos')
processor.load_plugin(provider_module='task_processing.plugins.stateful')
MesosExecutor = processor.executor_cls(provider='mesos')
task_config = MesosExecutor.TASK_CONFIG_INTERFACE(
**paasta_to_task_config_kwargs(
service,
instance,
cluster,
system_paasta_config,
instance_type,
soa_dir=soa_dir,
config_overrides=overrides_dict,
),
)
executor_stack = build_executor_stack(
processor,
service,
instance,
cluster,
run_id,
system_paasta_config,
args.staging_timeout,
)
runner = Sync(executor_stack)
def handle_interrupt(_signum, _frame):
paasta_print(
PaastaColors.red("Signal received, shutting down scheduler."),
)
runner.stop()
if _signum == signal.SIGTERM:
sys.exit(143)
else:
sys.exit(1)
signal.signal(signal.SIGINT, handle_interrupt)
signal.signal(signal.SIGTERM, handle_interrupt)
terminal_event = runner.run(task_config)
runner.stop()
if terminal_event.success:
paasta_print("Task finished successfully")
sys.exit(0)
else:
paasta_print(
PaastaColors.red("Task failed: {}".format(terminal_event.raw)),
)
sys.exit(1)