def remote_run_start(args): system_paasta_config, service, cluster, \ soa_dir, instance, instance_type = extract_args(args) overrides_dict = {} constraints_json = args.constraints_json if constraints_json: try: constraints = json.loads(constraints_json) except Exception as e: paasta_print("Error while parsing constraints: %s", e) if constraints: overrides_dict['constraints'] = constraints if args.cmd: overrides_dict['cmd'] = args.cmd if args.instances: overrides_dict['instances'] = args.instances run_id = args.run_id if run_id is None: run_id = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) paasta_print("Assigned random run-id: %s" % run_id) if args.detach: paasta_print("Running in background") if os.fork() > 0: return os.setsid() if os.fork() > 0: return sys.stdout = open('/dev/null', 'w') sys.stderr = open('/dev/null', 'w') paasta_print('Scheduling a task on Mesos') processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') processor.load_plugin(provider_module='task_processing.plugins.stateful') MesosExecutor = processor.executor_cls(provider='mesos') native_job_config = load_paasta_native_job_config( service, instance, cluster, soa_dir=soa_dir, instance_type=instance_type, config_overrides=overrides_dict, load_deployments=not args.docker_image, ) try: task_config = MesosExecutor.TASK_CONFIG_INTERFACE( **paasta_to_task_config_kwargs( service=service, instance=instance, system_paasta_config=system_paasta_config, native_job_config=native_job_config, config_overrides=overrides_dict, docker_image=args.docker_image, offer_timeout=args.staging_timeout, ), ) except InvariantException as e: if len(e.missing_fields) > 0: paasta_print( PaastaColors.red( "Mesos task config is missing following fields: {}".format( ', '.join(e.missing_fields), ), ), ) elif len(e.invariant_errors) > 0: paasta_print( PaastaColors.red( "Mesos task config is failing following checks: {}".format( ', '.join(str(ie) for ie in e.invariant_errors), ), ), ) else: paasta_print(PaastaColors.red(f"Mesos task config error: {e}"), ) traceback.print_exc() emit_counter_metric('paasta.remote_run.start.failed', service, instance) sys.exit(1) except PTypeError as e: paasta_print( PaastaColors.red( f"Mesos task config is failing a type check: {e}", ), ) traceback.print_exc() emit_counter_metric('paasta.remote_run.start.failed', service, instance) sys.exit(1) def handle_interrupt(_signum, _frame): paasta_print( PaastaColors.red("Signal received, shutting down scheduler."), ) if runner is not None: runner.stop() if _signum == signal.SIGTERM: sys.exit(143) else: sys.exit(1) signal.signal(signal.SIGINT, handle_interrupt) signal.signal(signal.SIGTERM, handle_interrupt) default_role = system_paasta_config.get_remote_run_config().get( 'default_role') assert default_role try: executor_stack = build_executor_stack( processor=processor, service=service, instance=instance, role=native_job_config.get_role() or default_role, pool=native_job_config.get_pool(), cluster=cluster, run_id=run_id, system_paasta_config=system_paasta_config, framework_staging_timeout=args.staging_timeout, ) runner = Sync(executor_stack) terminal_event = runner.run(task_config) runner.stop() except (Exception, ValueError) as e: paasta_print("Except while running executor stack: %s", e) traceback.print_exc() emit_counter_metric('paasta.remote_run.start.failed', service, instance) sys.exit(1) if terminal_event.success: paasta_print("Task finished successfully") sys.exit(0) else: paasta_print(PaastaColors.red(f"Task failed: {terminal_event.raw}"), ) # This is not necessarily an infrastructure failure. It may just be a # application failure. emit_counter_metric('paasta.remote_run.start.failed', service, instance) sys.exit(1)
def remote_run_start(args): system_paasta_config, service, cluster, soa_dir, instance, instance_type = extract_args(args) overrides_dict = {} constraints_json = args.constraints_json if constraints_json: try: constraints = json.loads(constraints_json) except Exception as e: paasta_print("Error while parsing constraints: %s", e) if constraints: overrides_dict['constraints'] = constraints if args.cmd: overrides_dict['cmd'] = args.cmd if args.instances: overrides_dict['instances'] = args.instances run_id = args.run_id if run_id is None: run_id = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(8) ) paasta_print("Assigned random run-id: %s" % run_id) if args.detach: paasta_print("Running in background") if os.fork() > 0: return os.setsid() if os.fork() > 0: return sys.stdout = open('/dev/null', 'w') sys.stderr = open('/dev/null', 'w') paasta_print('Scheduling a task on Mesos') processor = TaskProcessor() processor.load_plugin(provider_module='task_processing.plugins.mesos') processor.load_plugin(provider_module='task_processing.plugins.stateful') MesosExecutor = processor.executor_cls(provider='mesos') task_config = MesosExecutor.TASK_CONFIG_INTERFACE( **paasta_to_task_config_kwargs( service, instance, cluster, system_paasta_config, instance_type, soa_dir=soa_dir, config_overrides=overrides_dict, ), ) executor_stack = build_executor_stack( processor, service, instance, cluster, run_id, system_paasta_config, args.staging_timeout, ) runner = Sync(executor_stack) def handle_interrupt(_signum, _frame): paasta_print( PaastaColors.red("Signal received, shutting down scheduler."), ) runner.stop() if _signum == signal.SIGTERM: sys.exit(143) else: sys.exit(1) signal.signal(signal.SIGINT, handle_interrupt) signal.signal(signal.SIGTERM, handle_interrupt) terminal_event = runner.run(task_config) runner.stop() if terminal_event.success: paasta_print("Task finished successfully") sys.exit(0) else: paasta_print( PaastaColors.red("Task failed: {}".format(terminal_event.raw)), ) sys.exit(1)