def test_create_tasks_fails_if_create_fails(self, create_task): "creat_tasks fails if a single create_task call fails" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) def fail(*args): print("UHOH") raise RuntimeError('oh noes!') create_task.side_effect = fail with self.assertRaises(RuntimeError): create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", )
def test_create_tasks(self): tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), 'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task['payload'], 'hello world') self.assertEqual(task['schedulerId'], 'domain-level-4') # make sure the dependencies exist, at least for depid in task.get('dependencies', []): if depid == 'decisiontask': # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. If you wish to create the tasks in a new group, leave out decision_task_id.""" to_run = set(to_run) target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: full_task_graph[l] for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) optimized_task_graph, label_to_taskid = optimize_task_graph( target_task_graph, params, to_run, label_to_taskid) write_artifact('task-graph.json', optimized_task_graph.to_json()) write_artifact('label-to-taskid.json', label_to_taskid) write_artifact('to-run.json', list(to_run)) create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def create_tasks(graph_config, to_run, full_task_graph, label_to_taskid, params, decision_task_id, suffix='', modifier=lambda t: t): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. Optionally this function takes a `modifier` function that is passed in each task before it is put into a new graph. It should return a valid task. Note that this is passed _all_ tasks in the graph, not just the set in to_run. You may want to skip modifying tasks not in your to_run list. If `suffix` is given, then it is used to give unique names to the resulting artifacts. If you call this function multiple times in the same action, pass a different suffix each time to avoid overwriting artifacts. If you wish to create the tasks in a new group, leave out decision_task_id. Returns an updated label_to_taskid containing the new tasks""" if suffix != '': suffix = '-{}'.format(suffix) to_run = set(to_run) # Copy to avoid side-effects later full_task_graph = copy.deepcopy(full_task_graph) label_to_taskid = label_to_taskid.copy() target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) if decision_task_id and decision_task_id != os.environ.get('TASK_ID'): target_task_graph.for_each_task(update_dependencies) optimized_task_graph, label_to_taskid = optimize_task_graph( target_task_graph, to_run, params, to_run, decision_task_id, existing_tasks=label_to_taskid) write_artifact('task-graph{}.json'.format(suffix), optimized_task_graph.to_json()) write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid) write_artifact('to-run{}.json'.format(suffix), list(to_run)) create.create_tasks( graph_config, optimized_task_graph, label_to_taskid, params, decision_task_id, ) return label_to_taskid
def test_create_tasks(self): tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), "tid-b": Task(kind="test", label="b", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a", "b": "tid-b"} graph = Graph(nodes={"tid-a", "tid-b"}, edges={("tid-a", "tid-b", "edge")}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task["payload"], "hello world") self.assertEqual(task["schedulerId"], "domain-level-4") # make sure the dependencies exist, at least for depid in task.get("dependencies", []): if depid == "decisiontask": # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" os.environ['TASK_ID'] = 'decisiontask' tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" os.environ['TASK_ID'] = 'decisiontask' tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'}) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested.""" to_run = set(to_run) target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: full_task_graph[l] for l in target_graph.nodes}, target_graph) optimized_task_graph, label_to_taskid = optimize_task_graph( target_task_graph, params, to_run, label_to_taskid) create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def test_create_tasks_fails_if_create_fails(self, create_task): "creat_tasks fails if a single create_task call fails" os.environ['TASK_ID'] = 'decisiontask' tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) def fail(*args): print("UHOH") raise RuntimeError('oh noes!') create_task.side_effect = fail with self.assertRaises(RuntimeError): create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
def test_create_tasks(self): tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), 'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) taskgraph = TaskGraph(tasks, graph) create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task['payload'], 'hello world') self.assertEqual(task['schedulerId'], 'gecko-level-4') # make sure the dependencies exist, at least for depid in task.get('dependencies', []): if depid is 'decisiontask': # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. If you wish to create the tasks in a new group, leave out decision_task_id.""" to_run = set(to_run) target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: full_task_graph[l] for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, params, to_run, label_to_taskid) write_artifact('task-graph.json', optimized_task_graph.to_json()) write_artifact('label-to-taskid.json', label_to_taskid) write_artifact('to-run.json', list(to_run)) create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task.get('dependencies'), ["decisiontask"])
def test_create_task_without_dependencies(self): "a task with no dependencies depends on the decision task" tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a"} graph = Graph(nodes={"tid-a"}, edges=set()) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task.get("dependencies"), ["decisiontask"])