def from_config(cls, actions_config, cleanup_action_config=None): """Create this graph from a job config.""" actions = { maybe_decode(name): action.Action.from_config(conf) for name, conf in actions_config.items() } if cleanup_action_config: cleanup_action = action.Action.from_config(cleanup_action_config) actions[maybe_decode(cleanup_action.name)] = cleanup_action return cls(cls._build_dag(actions, actions_config), actions)
def __init__( self, job_name, run_num, run_time, node, output_path=None, base_context=None, action_runs=None, action_graph=None, manual=None, ): super(JobRun, self).__init__() self.job_name = maybe_decode(job_name) self.run_num = run_num self.run_time = run_time self.node = node self.output_path = output_path or filehandler.OutputPath() self.output_path.append(self.id) self.action_runs_proxy = None self._action_runs = None self.action_graph = action_graph self.manual = manual if action_runs: self.action_runs = action_runs self.context = command_context.build_context(self, base_context)
def state_data(self): """This data is used to serialize the state of this action run.""" rendered_command = self.rendered_command if isinstance(self.action_runner, NoActionRunnerFactory): action_runner = None else: action_runner = dict( status_path=self.action_runner.status_path, exec_path=self.action_runner.exec_path, ) # Freeze command after it's run command = rendered_command if rendered_command else self.bare_command return { 'job_run_id': self.job_run_id, 'action_name': self.action_name, 'state': self.state, 'start_time': self.start_time, 'end_time': self.end_time, 'command': maybe_decode(command), 'rendered_command': maybe_decode(self.rendered_command), 'node_name': self.node.get_name() if self.node else None, 'exit_status': self.exit_status, 'retries_remaining': self.retries_remaining, 'retries_delay': self.retries_delay, 'exit_statuses': self.exit_statuses, 'action_runner': action_runner, 'executor': self.executor, 'cpus': self.cpus, 'mem': self.mem, 'disk': self.disk, 'constraints': self.constraints, 'docker_image': self.docker_image, 'docker_parameters': self.docker_parameters, 'env': self.env, 'extra_volumes': self.extra_volumes, 'mesos_task_id': self.mesos_task_id, 'trigger_downstreams': self.trigger_downstreams, 'triggered_by': self.triggered_by, 'on_upstream_rerun': self.on_upstream_rerun, 'trigger_timeout_timestamp': self.trigger_timeout_timestamp, }
def build_action_run_collection(cls, job_run, action_runner): """Create an ActionRunGraph from an ActionGraph and JobRun.""" action_run_map = { maybe_decode(name): cls.build_run_for_action( job_run, action_inst, action_runner, ) for name, action_inst in job_run.action_graph.action_map.items() } return ActionRunCollection(job_run.action_graph, action_run_map)
def getChild(self, action_name, _): if not action_name: return self action_name = maybe_decode(action_name) if action_name in self.job_run.action_runs: action_run = self.job_run.action_runs[action_name] return ActionRunResource(action_run, self.job_run) return ErrorResource( f"Cannot find action '{action_name}' for " f"'{self.job_run}'" )
def getChild(self, run_id, _): if not run_id: return self run_id = maybe_decode(run_id) run = self.get_run_from_identifier(run_id) if run: return JobRunResource(run, self.job_scheduler) job = self.job_scheduler.get_job() if run_id in job.action_graph.names(): action_runs = job.runs.get_action_runs(run_id) return ActionRunHistoryResource(action_runs) return ErrorResource(f"Cannot find job run '{run_id}' for '{job}'")
def __init__( self, name, scheduler, queueing=True, all_nodes=False, monitoring=None, node_pool=None, enabled=True, action_graph=None, run_collection=None, parent_context=None, output_path=None, allow_overlap=None, action_runner=None, max_runtime=None, time_zone=None, expected_runtime=None ): super(Job, self).__init__() self.name = maybe_decode(name) self.monitoring = monitoring self.action_graph = action_graph self.scheduler = scheduler self.runs = run_collection self.queueing = queueing self.all_nodes = all_nodes self.enabled = enabled # current enabled setting self.config_enabled = enabled # enabled attribute from file self.node_pool = node_pool self.allow_overlap = allow_overlap self.action_runner = action_runner self.max_runtime = max_runtime self.time_zone = time_zone self.expected_runtime = expected_runtime self.output_path = output_path or filehandler.OutputPath() self.output_path.append(name) self.context = command_context.build_context(self, parent_context) log.info(f'{self} created')
def action_run_collection_from_state( cls, job_run, runs_state_data, cleanup_action_state_data, ): action_runs = [ cls.action_run_from_state(job_run, state_data) for state_data in runs_state_data ] if cleanup_action_state_data: action_runs.append( cls.action_run_from_state( job_run, cleanup_action_state_data, cleanup=True, ), ) action_run_map = { maybe_decode(action_run.action_name): action_run for action_run in action_runs } return ActionRunCollection(job_run.action_graph, action_run_map)
def _get_dependencies(cls, actions_config, action_name): if action_name == action.CLEANUP_ACTION_NAME: return [] return actions_config[maybe_decode(action_name)].requires
def _get_dependencies(cls, actions_config, action_name): if action_name == action.CLEANUP_ACTION_NAME: return [] return actions_config[maybe_decode(action_name)].requires
def write_raw(path, content): with open(path, 'w') as fh: fh.write(maybe_decode(content))
def getChild(self, name, request): if not name: return self name = maybe_decode(name) return resource_from_collection(self.job_collection, name, JobResource)
def write_raw(path, content): with open(path, 'w') as fh: fh.write(maybe_decode(content))
def from_state( cls, state_data, parent_context, output_path, job_run_node, cleanup=False, ): """Restore the state of this ActionRun from a serialized state.""" pool_repo = node.NodePoolRepository.get_instance() # Support state from older version if 'id' in state_data: job_run_id, action_name = state_data['id'].rsplit('.', 1) else: job_run_id = state_data['job_run_id'] action_name = state_data['action_name'] job_run_node = pool_repo.get_node( state_data.get('node_name'), job_run_node, ) action_runner_data = state_data.get('action_runner') if action_runner_data: action_runner = SubprocessActionRunnerFactory(**action_runner_data) else: action_runner = NoActionRunnerFactory() run = cls( job_run_id=job_run_id, name=action_name, node=job_run_node, parent_context=parent_context, output_path=output_path, rendered_command=maybe_decode(state_data.get('rendered_command')), bare_command=maybe_decode(state_data['command']), cleanup=cleanup, start_time=state_data['start_time'], end_time=state_data['end_time'], run_state=state_data['state'], exit_status=state_data.get('exit_status'), retries_remaining=state_data.get('retries_remaining'), retries_delay=state_data.get('retries_delay'), exit_statuses=state_data.get('exit_statuses'), action_runner=action_runner, executor=state_data.get('executor', ExecutorTypes.ssh.value), cpus=state_data.get('cpus'), mem=state_data.get('mem'), disk=state_data.get('disk'), constraints=state_data.get('constraints'), docker_image=state_data.get('docker_image'), docker_parameters=state_data.get('docker_parameters'), env=state_data.get('env'), extra_volumes=state_data.get('extra_volumes'), mesos_task_id=state_data.get('mesos_task_id'), trigger_downstreams=state_data.get('trigger_downstreams'), triggered_by=state_data.get('triggered_by'), on_upstream_rerun=state_data.get('on_upstream_rerun'), trigger_timeout_timestamp=state_data.get('trigger_timeout_timestamp'), ) # Transition running to fail unknown because exit status was missed # Recovery will look for unknown runs if run.is_active: run.transition_and_notify('fail_unknown') return run
def __init__( self, job_run_id, name, node, bare_command=None, parent_context=None, output_path=None, cleanup=False, start_time=None, end_time=None, run_state=SCHEDULED, rendered_command=None, exit_status=None, action_runner=None, retries_remaining=None, retries_delay=None, exit_statuses=None, machine=None, executor=None, cpus=None, mem=None, disk=None, constraints=None, docker_image=None, docker_parameters=None, env=None, extra_volumes=None, mesos_task_id=None, trigger_downstreams=None, triggered_by=None, on_upstream_rerun=None, trigger_timeout_timestamp=None, ): super().__init__() self.job_run_id = maybe_decode(job_run_id) self.action_name = maybe_decode(name) self.node = node self.start_time = start_time self.end_time = end_time self.exit_status = exit_status self.bare_command = maybe_decode(bare_command) self.rendered_command = rendered_command self.action_runner = action_runner or NoActionRunnerFactory() self.machine = machine or Machine.from_machine( ActionRun.STATE_MACHINE, None, run_state ) self.is_cleanup = cleanup self.executor = executor self.cpus = cpus self.mem = mem self.disk = disk self.constraints = constraints self.docker_image = docker_image self.docker_parameters = docker_parameters self.env = env self.extra_volumes = extra_volumes self.mesos_task_id = mesos_task_id self.output_path = output_path or filehandler.OutputPath() self.output_path.append(self.id) self.context = command_context.build_context(self, parent_context) self.retries_remaining = retries_remaining self.retries_delay = retries_delay self.exit_statuses = exit_statuses self.trigger_downstreams = trigger_downstreams self.triggered_by = triggered_by self.on_upstream_rerun = on_upstream_rerun self.trigger_timeout_timestamp = trigger_timeout_timestamp self.trigger_timeout_call = None if self.exit_statuses is None: self.exit_statuses = [] self.action_command = None self.in_delay = None
def getChild(self, name, request): if not name: return self name = maybe_decode(name) return resource_from_collection(self.job_collection, name, JobResource)
def __init__(self, type, iden): self.type = maybe_decode(type) self.iden = maybe_decode(iden)
def __init__( self, job_run_id, name, node, bare_command=None, parent_context=None, output_path=None, cleanup=False, start_time=None, end_time=None, run_state=SCHEDULED, rendered_command=None, exit_status=None, action_runner=None, retries_remaining=None, retries_delay=None, exit_statuses=None, machine=None, executor=None, cpus=None, mem=None, constraints=None, docker_image=None, docker_parameters=None, env=None, extra_volumes=None, mesos_task_id=None, trigger_downstreams=None, triggered_by=None, on_upstream_rerun=None, ): super().__init__() self.job_run_id = maybe_decode(job_run_id) self.action_name = maybe_decode(name) self.node = node self.start_time = start_time self.end_time = end_time self.exit_status = exit_status self.bare_command = maybe_decode(bare_command) self.rendered_command = rendered_command self.action_runner = action_runner or NoActionRunnerFactory() self.machine = machine or Machine.from_machine( ActionRun.STATE_MACHINE, None, run_state ) self.is_cleanup = cleanup self.executor = executor self.cpus = cpus self.mem = mem self.constraints = constraints self.docker_image = docker_image self.docker_parameters = docker_parameters self.env = env self.extra_volumes = extra_volumes self.mesos_task_id = mesos_task_id self.output_path = output_path or filehandler.OutputPath() self.output_path.append(self.id) self.context = command_context.build_context(self, parent_context) self.retries_remaining = retries_remaining self.retries_delay = retries_delay self.exit_statuses = exit_statuses self.trigger_downstreams = trigger_downstreams self.triggered_by = triggered_by self.on_upstream_rerun = on_upstream_rerun if self.exit_statuses is None: self.exit_statuses = [] self.action_command = None self.in_delay = None
def _save_action(self, action_name, job_name, config): action_name = maybe_decode(action_name) full_name = f'{job_name}.{action_name}' self.action_map[full_name] = Action.from_config(config) self._actions_for_job[job_name].append(full_name) return full_name
def from_state( cls, state_data, parent_context, output_path, job_run_node, cleanup=False, ): """Restore the state of this ActionRun from a serialized state.""" pool_repo = node.NodePoolRepository.get_instance() # Support state from older version if 'id' in state_data: job_run_id, action_name = state_data['id'].rsplit('.', 1) else: job_run_id = state_data['job_run_id'] action_name = state_data['action_name'] job_run_node = pool_repo.get_node( state_data.get('node_name'), job_run_node, ) action_runner_data = state_data.get('action_runner') if action_runner_data: action_runner = SubprocessActionRunnerFactory(**action_runner_data) else: action_runner = NoActionRunnerFactory() run = cls( job_run_id=job_run_id, name=action_name, node=job_run_node, parent_context=parent_context, output_path=output_path, rendered_command=maybe_decode(state_data.get('rendered_command')), bare_command=maybe_decode(state_data['command']), cleanup=cleanup, start_time=state_data['start_time'], end_time=state_data['end_time'], run_state=state_data['state'], exit_status=state_data.get('exit_status'), retries_remaining=state_data.get('retries_remaining'), retries_delay=state_data.get('retries_delay'), exit_statuses=state_data.get('exit_statuses'), action_runner=action_runner, executor=state_data.get('executor', ExecutorTypes.ssh.value), cpus=state_data.get('cpus'), mem=state_data.get('mem'), disk=state_data.get('disk'), constraints=state_data.get('constraints'), docker_image=state_data.get('docker_image'), docker_parameters=state_data.get('docker_parameters'), env=state_data.get('env'), extra_volumes=state_data.get('extra_volumes'), mesos_task_id=state_data.get('mesos_task_id'), trigger_downstreams=state_data.get('trigger_downstreams'), triggered_by=state_data.get('triggered_by'), on_upstream_rerun=state_data.get('on_upstream_rerun'), trigger_timeout_timestamp=state_data.get( 'trigger_timeout_timestamp'), ) # Transition running to fail unknown because exit status was missed # Recovery will look for unknown runs if run.is_active: run.transition_and_notify('fail_unknown') return run