def test_workflow_done(self): wfj = self.workflow_job(states=['failed', None, None, 'successful', None]) dag = WorkflowDAG(workflow_job=wfj) is_done, has_failed = dag.is_workflow_done() self.assertTrue(is_done) self.assertFalse(has_failed) # verify that relaunched WFJ fails if a JT leaf is deleted for jt in JobTemplate.objects.all(): jt.delete() relaunched = wfj.create_relaunch_workflow_job() dag = WorkflowDAG(workflow_job=relaunched) is_done, has_failed = dag.is_workflow_done() self.assertTrue(is_done) self.assertTrue(has_failed)
def process_finished_workflow_jobs(self, workflow_jobs): result = [] for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) if workflow_job.cancel_flag: logger.debug( 'Canceling spawned jobs of %s due to cancel flag.', workflow_job.log_format) cancel_finished = dag.cancel_node_jobs() if cancel_finished: logger.info( 'Marking %s as canceled, all spawned jobs have concluded.', workflow_job.log_format) workflow_job.status = 'canceled' workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=['status', 'start_args']) workflow_job.websocket_emit_status(workflow_job.status) else: is_done, has_failed = dag.is_workflow_done() if not is_done: continue logger.info('Marking %s as %s.', workflow_job.log_format, 'failed' if has_failed else 'successful') result.append(workflow_job.id) new_status = 'failed' if has_failed else 'successful' logger.debug( six.text_type("Transitioning {} to {} status.").format( workflow_job.log_format, new_status)) workflow_job.status = new_status workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=['status', 'start_args']) workflow_job.websocket_emit_status(workflow_job.status) return result
def workflow_dag_1(wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(4)] for n in nodes: g.add_node(n) r''' 0 /\ S / \ / \ 1 | | | F | | S | | 3 | \ | F \ | \/ 2 ''' g.add_edge(nodes[0], nodes[1], "success_nodes") g.add_edge(nodes[0], nodes[2], "success_nodes") g.add_edge(nodes[1], nodes[3], "failure_nodes") g.add_edge(nodes[3], nodes[2], "failure_nodes") return (g, nodes)
def process_finished_workflow_jobs(self, workflow_jobs): result = [] for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) if workflow_job.cancel_flag: logger.debug( 'Canceling spawned jobs of %s due to cancel flag.', workflow_job.log_format) cancel_finished = dag.cancel_node_jobs() if cancel_finished: logger.info( 'Marking %s as canceled, all spawned jobs have concluded.', workflow_job.log_format) workflow_job.status = 'canceled' workflow_job.save() connection.on_commit( lambda: workflow_job.websocket_emit_status(workflow_job .status)) else: is_done, has_failed = dag.is_workflow_done() if not is_done: continue logger.info('Marking %s as %s.', workflow_job.log_format, 'failed' if has_failed else 'successful') result.append(workflow_job.id) workflow_job.status = 'failed' if has_failed else 'successful' workflow_job.save() connection.on_commit( lambda: workflow_job.websocket_emit_status(workflow_job. status)) return result
def simple_all_convergence(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(4)] for n in nodes: g.add_node(n) r''' 0 /\ S / \ S / \ 1 2 \ / F \ / S \/ 3 ''' g.add_edge(nodes[0], nodes[1], "success_nodes") g.add_edge(nodes[0], nodes[2], "success_nodes") g.add_edge(nodes[1], nodes[3], "failure_nodes") g.add_edge(nodes[2], nodes[3], "success_nodes") nodes[3].all_parents_must_converge = True nodes[0].job = Job(status='successful') nodes[1].job = Job(status='failed') nodes[2].job = Job(status='successful') return (g, nodes)
def spawn_workflow_graph_jobs(self, workflow_jobs): for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) spawn_nodes = dag.bfs_nodes_to_run() for spawn_node in spawn_nodes: if spawn_node.unified_job_template is None: continue kv = spawn_node.get_job_kwargs() job = spawn_node.unified_job_template.create_unified_job(**kv) spawn_node.job = job spawn_node.save() if job._resources_sufficient_for_launch(): can_start = job.signal_start() if not can_start: job.job_explanation = _( "Job spawned from workflow could not start because it " "was not in the right state or required manual credentials" ) else: can_start = False job.job_explanation = _( "Job spawned from workflow could not start because it " "was missing a related resource such as project or inventory" ) if not can_start: job.status = 'failed' job.save(update_fields=['status', 'job_explanation']) connection.on_commit( lambda: job.websocket_emit_status('failed'))
def test_workflow_dnr_because_parent(self, workflow_job_fn): wfj, nodes = workflow_job_fn(states=['successful', None, None, None, None, None,]) dag = WorkflowDAG(workflow_job=wfj) workflow_nodes = dag.mark_dnr_nodes() assert 2 == len(workflow_nodes) assert nodes[3] in workflow_nodes assert nodes[4] in workflow_nodes
def test_workflow_fails_leaf(self): wfj = self.workflow_job( states=['successful', 'successful', 'failed', None, None]) dag = WorkflowDAG(workflow_job=wfj) is_done, has_failed = dag.is_workflow_done() self.assertTrue(is_done) self.assertTrue(has_failed)
def test_workflow_fails_for_no_error_handler(self): wfj = self.workflow_job(states=['successful', 'failed', None, None, None]) dag = WorkflowDAG(workflow_job=wfj) dag.mark_dnr_nodes() is_done = dag.is_workflow_done() has_failed = dag.has_workflow_failed() self.assertTrue(is_done) self.assertTrue(has_failed)
def spawn_workflow_graph_jobs(self, workflow_jobs): for workflow_job in workflow_jobs: if workflow_job.cancel_flag: logger.debug('Not spawning jobs for %s because it is pending cancelation.', workflow_job.log_format) continue dag = WorkflowDAG(workflow_job) spawn_nodes = dag.bfs_nodes_to_run() if spawn_nodes: logger.debug('Spawning jobs for %s', workflow_job.log_format) else: logger.debug('No nodes to spawn for %s', workflow_job.log_format) for spawn_node in spawn_nodes: if spawn_node.unified_job_template is None: continue kv = spawn_node.get_job_kwargs() job = spawn_node.unified_job_template.create_unified_job(**kv) spawn_node.job = job spawn_node.save() logger.debug('Spawned %s in %s for node %s', job.log_format, workflow_job.log_format, spawn_node.pk) can_start = True if isinstance(spawn_node.unified_job_template, WorkflowJobTemplate): workflow_ancestors = job.get_ancestor_workflows() if spawn_node.unified_job_template in set(workflow_ancestors): can_start = False logger.info( 'Refusing to start recursive workflow-in-workflow id={}, wfjt={}, ancestors={}'.format( job.id, spawn_node.unified_job_template.pk, [wa.pk for wa in workflow_ancestors] ) ) display_list = [spawn_node.unified_job_template] + workflow_ancestors job.job_explanation = gettext_noop( "Workflow Job spawned from workflow could not start because it " "would result in recursion (spawn order, most recent first: {})" ).format(', '.join(['<{}>'.format(tmp) for tmp in display_list])) else: logger.debug( 'Starting workflow-in-workflow id={}, wfjt={}, ancestors={}'.format( job.id, spawn_node.unified_job_template.pk, [wa.pk for wa in workflow_ancestors] ) ) if not job._resources_sufficient_for_launch(): can_start = False job.job_explanation = gettext_noop( "Job spawned from workflow could not start because it " "was missing a related resource such as project or inventory" ) if can_start: if workflow_job.start_args: start_args = json.loads(decrypt_field(workflow_job, 'start_args')) else: start_args = {} can_start = job.signal_start(**start_args) if not can_start: job.job_explanation = gettext_noop( "Job spawned from workflow could not start because it " "was not in the right state or required manual credentials" ) if not can_start: job.status = 'failed' job.save(update_fields=['status', 'job_explanation']) job.websocket_emit_status('failed')
def test_workflow_not_finished(self): wfj = self.workflow_job(states=['new', None, None, None, None]) dag = WorkflowDAG(workflow_job=wfj) dag.mark_dnr_nodes() is_done = dag.is_workflow_done() has_failed, reason = dag.has_workflow_failed() self.assertFalse(is_done) self.assertFalse(has_failed) assert reason is None
def process_finished_workflow_jobs(self, workflow_jobs): result = [] for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) status_changed = False if workflow_job.cancel_flag: workflow_job.workflow_nodes.filter( do_not_run=False, job__isnull=True).update(do_not_run=True) logger.debug( 'Canceling spawned jobs of %s due to cancel flag.', workflow_job.log_format) cancel_finished = dag.cancel_node_jobs() if cancel_finished: logger.info( 'Marking %s as canceled, all spawned jobs have concluded.', workflow_job.log_format) workflow_job.status = 'canceled' workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=['status', 'start_args']) status_changed = True else: workflow_nodes = dag.mark_dnr_nodes() for n in workflow_nodes: n.save(update_fields=['do_not_run']) is_done = dag.is_workflow_done() if not is_done: continue has_failed, reason = dag.has_workflow_failed() logger.debug('Marking %s as %s.', workflow_job.log_format, 'failed' if has_failed else 'successful') result.append(workflow_job.id) new_status = 'failed' if has_failed else 'successful' logger.debug("Transitioning {} to {} status.".format( workflow_job.log_format, new_status)) update_fields = ['status', 'start_args'] workflow_job.status = new_status if reason: logger.info( f'Workflow job {workflow_job.id} failed due to reason: {reason}' ) workflow_job.job_explanation = gettext_noop( "No error handling paths found, marking workflow as failed" ) update_fields.append('job_explanation') workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=update_fields) status_changed = True if status_changed: workflow_job.websocket_emit_status(workflow_job.status) # Operations whose queries rely on modifications made during the atomic scheduling session workflow_job.send_notification_templates( 'succeeded' if workflow_job.status == 'successful' else 'failed') if workflow_job.spawned_by_workflow: schedule_task_manager() return result
def workflow_dag_canceled(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(1)] for n in nodes: g.add_node(n) r''' F0 ''' nodes[0].job = Job(status='canceled') return (g, nodes)
def test_build_WFJT_dag(self): ''' Test that building the graph uses 4 queries 1 to get the nodes 3 to get the related success, failure, and always connections ''' dag = WorkflowDAG() wfj = self.workflow_job() with self.assertNumQueries(4): dag._init_graph(wfj)
def test_workflow_done(self): wfj = self.workflow_job(states=['failed', None, None, 'successful', None]) dag = WorkflowDAG(workflow_job=wfj) assert 3 == len(dag.mark_dnr_nodes()) is_done = dag.is_workflow_done() has_failed, reason = dag.has_workflow_failed() self.assertTrue(is_done) self.assertFalse(has_failed) assert reason is None # verify that relaunched WFJ fails if a JT leaf is deleted for jt in JobTemplate.objects.all(): jt.delete() relaunched = wfj.create_relaunch_workflow_job() dag = WorkflowDAG(workflow_job=relaunched) dag.mark_dnr_nodes() is_done = dag.is_workflow_done() has_failed, reason = dag.has_workflow_failed() self.assertTrue(is_done) self.assertTrue(has_failed) assert "Workflow job node {} related unified job template missing".format(wfj.workflow_nodes.all()[0].id)
def process_finished_workflow_jobs(self, workflow_jobs): result = [] for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) status_changed = False if workflow_job.cancel_flag: workflow_job.workflow_nodes.filter( do_not_run=False, job__isnull=True).update(do_not_run=True) logger.debug( 'Canceling spawned jobs of %s due to cancel flag.', workflow_job.log_format) cancel_finished = dag.cancel_node_jobs() if cancel_finished: logger.info( 'Marking %s as canceled, all spawned jobs have concluded.', workflow_job.log_format) workflow_job.status = 'canceled' workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=['status', 'start_args']) status_changed = True else: workflow_nodes = dag.mark_dnr_nodes() map(lambda n: n.save(update_fields=['do_not_run']), workflow_nodes) is_done = dag.is_workflow_done() if not is_done: continue has_failed, reason = dag.has_workflow_failed() logger.info('Marking %s as %s.', workflow_job.log_format, 'failed' if has_failed else 'successful') result.append(workflow_job.id) new_status = 'failed' if has_failed else 'successful' logger.debug( six.text_type("Transitioning {} to {} status.").format( workflow_job.log_format, new_status)) update_fields = ['status', 'start_args'] workflow_job.status = new_status if reason: logger.info(reason) workflow_job.job_explanation = "No error handling paths found, marking workflow as failed" update_fields.append('job_explanation') workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=update_fields) status_changed = True if status_changed: workflow_job.websocket_emit_status(workflow_job.status) if workflow_job.spawned_by_workflow: schedule_task_manager() return result
def workflow_dag_canceled(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(4)] for n in nodes: g.add_node(n) r''' C0 / | \ F / A| \ S / | \ 1 2 3 ''' g.add_edge(nodes[0], nodes[1], "failure_nodes") g.add_edge(nodes[0], nodes[2], "always_nodes") g.add_edge(nodes[0], nodes[3], "success_nodes") nodes[0].job = Job(status='canceled') return (g, nodes)
def spawn_workflow_graph_jobs(self, workflow_jobs): for workflow_job in workflow_jobs: if workflow_job.cancel_flag: logger.debug( 'Not spawning jobs for %s because it is pending cancelation.', workflow_job.log_format) continue dag = WorkflowDAG(workflow_job) spawn_nodes = dag.bfs_nodes_to_run() if spawn_nodes: logger.info('Spawning jobs for %s', workflow_job.log_format) else: logger.debug('No nodes to spawn for %s', workflow_job.log_format) for spawn_node in spawn_nodes: if spawn_node.unified_job_template is None: continue kv = spawn_node.get_job_kwargs() job = spawn_node.unified_job_template.create_unified_job(**kv) spawn_node.job = job spawn_node.save() logger.info('Spawned %s in %s for node %s', job.log_format, workflow_job.log_format, spawn_node.pk) if job._resources_sufficient_for_launch(): if workflow_job.start_args: start_args = json.loads( decrypt_field(workflow_job, 'start_args')) else: start_args = {} can_start = job.signal_start(**start_args) if not can_start: job.job_explanation = _( "Job spawned from workflow could not start because it " "was not in the right state or required manual credentials" ) else: can_start = False job.job_explanation = _( "Job spawned from workflow could not start because it " "was missing a related resource such as project or inventory" ) if not can_start: job.status = 'failed' job.save(update_fields=['status', 'job_explanation']) connection.on_commit( lambda: job.websocket_emit_status('failed'))
def process_finished_workflow_jobs(self, workflow_jobs): result = [] for workflow_job in workflow_jobs: dag = WorkflowDAG(workflow_job) if workflow_job.cancel_flag: workflow_job.status = 'canceled' workflow_job.save() dag.cancel_node_jobs() connection.on_commit(lambda: workflow_job.websocket_emit_status(workflow_job.status)) else: is_done, has_failed = dag.is_workflow_done() if not is_done: continue result.append(workflow_job.id) workflow_job.status = 'failed' if has_failed else 'successful' workflow_job.save() connection.on_commit(lambda: workflow_job.websocket_emit_status(workflow_job.status)) return result
def workflow_dag_root_children(self, wf_node_generator): g = WorkflowDAG() wf_root_nodes = [wf_node_generator() for i in range(0, 10)] wf_leaf_nodes = [wf_node_generator() for i in range(0, 10)] for n in wf_root_nodes + wf_leaf_nodes: g.add_node(n) ''' Pair up a root node with a single child via an edge R1 R2 ... Rx | | | | | | C1 C2 Cx ''' for i, n in enumerate(wf_leaf_nodes): g.add_edge(wf_root_nodes[i], n, 'label') return (g, wf_root_nodes, wf_leaf_nodes)
def workflow_all_converge_1(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(3)] for n in nodes: g.add_node(n) r''' 0 |\ F | \ S| 1 | / |/ A 2 ''' g.add_edge(nodes[0], nodes[1], "failure_nodes") g.add_edge(nodes[0], nodes[2], "success_nodes") g.add_edge(nodes[1], nodes[2], "always_nodes") nodes[2].all_parents_must_converge = True nodes[0].job = Job(status='successful') return (g, nodes)
def complex_dag(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(10)] for n in nodes: g.add_node(n) g.add_edge(nodes[0], nodes[1], "failure_nodes") g.add_edge(nodes[0], nodes[2], "success_nodes") g.add_edge(nodes[0], nodes[3], "always_nodes") g.add_edge(nodes[1], nodes[4], "success_nodes") g.add_edge(nodes[1], nodes[5], "failure_nodes") g.add_edge(nodes[2], nodes[6], "failure_nodes") g.add_edge(nodes[3], nodes[6], "success_nodes") g.add_edge(nodes[4], nodes[6], "always_nodes") g.add_edge(nodes[6], nodes[7], "always_nodes") g.add_edge(nodes[6], nodes[8], "success_nodes") g.add_edge(nodes[6], nodes[9], "failure_nodes") return (g, nodes)
def workflow_all_converge_deep_dnr_tree(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(7)] for n in nodes: g.add_node(n) r''' 0 1 2 \ | / S \ S| / F \ | / \|/ | 3 /\ S / \ S / \ 4| | 5 \ / S \ / S \/ 6 ''' g.add_edge(nodes[0], nodes[3], "success_nodes") g.add_edge(nodes[1], nodes[3], "success_nodes") g.add_edge(nodes[2], nodes[3], "failure_nodes") g.add_edge(nodes[3], nodes[4], "success_nodes") g.add_edge(nodes[3], nodes[5], "success_nodes") g.add_edge(nodes[4], nodes[6], "success_nodes") g.add_edge(nodes[5], nodes[6], "success_nodes") nodes[3].all_parents_must_converge = True nodes[4].all_parents_must_converge = True nodes[5].all_parents_must_converge = True nodes[6].all_parents_must_converge = True nodes[0].job = Job(status='successful') nodes[1].job = Job(status='successful') nodes[2].job = Job(status='successful') return (g, nodes)
def workflow_all_converge_dnr(self, wf_node_generator): g = WorkflowDAG() nodes = [wf_node_generator() for i in range(4)] for n in nodes: g.add_node(n) r''' 0 1 2 S \ F | / F \ | / \ | / \|/ | 3 ''' g.add_edge(nodes[0], nodes[3], "success_nodes") g.add_edge(nodes[1], nodes[3], "failure_nodes") g.add_edge(nodes[2], nodes[3], "failure_nodes") nodes[3].all_parents_must_converge = True nodes[0].job = Job(status='successful') nodes[1].job = Job(status='running') nodes[2].job = Job(status='failed') return (g, nodes)
def workflow_all_converge_2(self, wf_node_generator): """The ordering of _1 and this test, _2, is _slightly_ different. The hope is that topological sorting results in 2 being processed before 3 and/or 3 before 2. """ g = WorkflowDAG() nodes = [wf_node_generator() for i in range(3)] for n in nodes: g.add_node(n) r''' 0 |\ S | \ F| 1 | / |/ A 2 ''' g.add_edge(nodes[0], nodes[1], "success_nodes") g.add_edge(nodes[0], nodes[2], "failure_nodes") g.add_edge(nodes[1], nodes[2], "always_nodes") nodes[2].all_parents_must_converge = True nodes[0].job = Job(status='successful') return (g, nodes)
def test_workflow_fails_for_unfinished_node(self): wfj = self.workflow_job(states=['error', None, None, None, None]) dag = WorkflowDAG(workflow_job=wfj) is_done, has_failed = dag.is_workflow_done() self.assertTrue(is_done) self.assertTrue(has_failed)
def test_workflow_not_finished(self): wfj = self.workflow_job(states=['new', None, None, None, None]) dag = WorkflowDAG(workflow_job=wfj) is_done, has_failed = dag.is_workflow_done() self.assertFalse(is_done) self.assertFalse(has_failed)
def spawn_workflow_graph_jobs(self): result = [] for workflow_job in self.all_tasks: if self.timed_out(): logger.warning( "Workflow manager has reached time out while processing running workflows, exiting loop early" ) ScheduleWorkflowManager().schedule() # Do not process any more workflow jobs. Stop here. # Maybe we should schedule another WorkflowManager run break dag = WorkflowDAG(workflow_job) status_changed = False if workflow_job.cancel_flag: workflow_job.workflow_nodes.filter( do_not_run=False, job__isnull=True).update(do_not_run=True) logger.debug( 'Canceling spawned jobs of %s due to cancel flag.', workflow_job.log_format) cancel_finished = dag.cancel_node_jobs() if cancel_finished: logger.info( 'Marking %s as canceled, all spawned jobs have concluded.', workflow_job.log_format) workflow_job.status = 'canceled' workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=['status', 'start_args']) status_changed = True else: workflow_nodes = dag.mark_dnr_nodes() WorkflowJobNode.objects.bulk_update(workflow_nodes, ['do_not_run']) # If workflow is now done, we do special things to mark it as done. is_done = dag.is_workflow_done() if is_done: has_failed, reason = dag.has_workflow_failed() logger.debug('Marking %s as %s.', workflow_job.log_format, 'failed' if has_failed else 'successful') result.append(workflow_job.id) new_status = 'failed' if has_failed else 'successful' logger.debug("Transitioning {} to {} status.".format( workflow_job.log_format, new_status)) update_fields = ['status', 'start_args'] workflow_job.status = new_status if reason: logger.info( f'Workflow job {workflow_job.id} failed due to reason: {reason}' ) workflow_job.job_explanation = gettext_noop( "No error handling paths found, marking workflow as failed" ) update_fields.append('job_explanation') workflow_job.start_args = '' # blank field to remove encrypted passwords workflow_job.save(update_fields=update_fields) status_changed = True if status_changed: if workflow_job.spawned_by_workflow: ScheduleWorkflowManager().schedule() workflow_job.websocket_emit_status(workflow_job.status) # Operations whose queries rely on modifications made during the atomic scheduling session workflow_job.send_notification_templates( 'succeeded' if workflow_job.status == 'successful' else 'failed') if workflow_job.status == 'running': spawn_nodes = dag.bfs_nodes_to_run() if spawn_nodes: logger.debug('Spawning jobs for %s', workflow_job.log_format) else: logger.debug('No nodes to spawn for %s', workflow_job.log_format) for spawn_node in spawn_nodes: if spawn_node.unified_job_template is None: continue kv = spawn_node.get_job_kwargs() job = spawn_node.unified_job_template.create_unified_job( **kv) spawn_node.job = job spawn_node.save() logger.debug('Spawned %s in %s for node %s', job.log_format, workflow_job.log_format, spawn_node.pk) can_start = True if isinstance(spawn_node.unified_job_template, WorkflowJobTemplate): workflow_ancestors = job.get_ancestor_workflows() if spawn_node.unified_job_template in set( workflow_ancestors): can_start = False logger.info( 'Refusing to start recursive workflow-in-workflow id={}, wfjt={}, ancestors={}' .format(job.id, spawn_node.unified_job_template.pk, [wa.pk for wa in workflow_ancestors])) display_list = [spawn_node.unified_job_template ] + workflow_ancestors job.job_explanation = gettext_noop( "Workflow Job spawned from workflow could not start because it " "would result in recursion (spawn order, most recent first: {})" ).format(', '.join('<{}>'.format(tmp) for tmp in display_list)) else: logger.debug( 'Starting workflow-in-workflow id={}, wfjt={}, ancestors={}' .format(job.id, spawn_node.unified_job_template.pk, [wa.pk for wa in workflow_ancestors])) if not job._resources_sufficient_for_launch(): can_start = False job.job_explanation = gettext_noop( "Job spawned from workflow could not start because it was missing a related resource such as project or inventory" ) if can_start: if workflow_job.start_args: start_args = json.loads( decrypt_field(workflow_job, 'start_args')) else: start_args = {} can_start = job.signal_start(**start_args) if not can_start: job.job_explanation = gettext_noop( "Job spawned from workflow could not start because it was not in the right state or required manual credentials" ) if not can_start: job.status = 'failed' job.save(update_fields=['status', 'job_explanation']) job.websocket_emit_status('failed') # TODO: should we emit a status on the socket here similar to tasks.py awx_periodic_scheduler() ? # emit_websocket_notification('/socket.io/jobs', '', dict(id=)) return result