def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError('oh noes!')
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def test_create_tasks(self):
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
'tid-b':
Task(kind='test',
label='b',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'},
edges={('tid-a', 'tid-b', 'edge')})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['payload'], 'hello world')
self.assertEqual(task['schedulerId'], 'domain-level-4')
# make sure the dependencies exist, at least
for depid in task.get('dependencies', []):
if depid == 'decisiontask':
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def create_tasks(to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l]
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph, params, to_run, label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params,
decision_task_id)
def create_tasks(graph_config,
to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id,
suffix='',
modifier=lambda t: t):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
Optionally this function takes a `modifier` function that is passed in each
task before it is put into a new graph. It should return a valid task. Note
that this is passed _all_ tasks in the graph, not just the set in to_run. You
may want to skip modifying tasks not in your to_run list.
If `suffix` is given, then it is used to give unique names to the resulting
artifacts. If you call this function multiple times in the same action,
pass a different suffix each time to avoid overwriting artifacts.
If you wish to create the tasks in a new group, leave out decision_task_id.
Returns an updated label_to_taskid containing the new tasks"""
if suffix != '':
suffix = '-{}'.format(suffix)
to_run = set(to_run)
# Copy to avoid side-effects later
full_task_graph = copy.deepcopy(full_task_graph)
label_to_taskid = label_to_taskid.copy()
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: modifier(full_task_graph[l])
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
if decision_task_id and decision_task_id != os.environ.get('TASK_ID'):
target_task_graph.for_each_task(update_dependencies)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph,
to_run,
params,
to_run,
decision_task_id,
existing_tasks=label_to_taskid)
write_artifact('task-graph{}.json'.format(suffix),
optimized_task_graph.to_json())
write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
write_artifact('to-run{}.json'.format(suffix), list(to_run))
create.create_tasks(
graph_config,
optimized_task_graph,
label_to_taskid,
params,
decision_task_id,
)
return label_to_taskid
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
os.environ['TASK_ID'] = 'decisiontask'
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
os.environ['TASK_ID'] = 'decisiontask'
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(GRAPH_CONFIG, taskgraph, label_to_taskid, {'level': '4'})
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
def create_tasks(to_run, full_task_graph, label_to_taskid, params,
decision_task_id):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l]
for l in target_graph.nodes}, target_graph)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph, params, to_run, label_to_taskid)
create.create_tasks(optimized_task_graph, label_to_taskid, params,
decision_task_id)
def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
os.environ['TASK_ID'] = 'decisiontask'
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError('oh noes!')
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
def test_create_tasks(self):
tasks = {
'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}),
'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['payload'], 'hello world')
self.assertEqual(task['schedulerId'], 'gecko-level-4')
# make sure the dependencies exist, at least
for depid in task.get('dependencies', []):
if depid is 'decisiontask':
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l] for l in target_graph.nodes},
target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
params,
to_run,
label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get('dependencies'), ["decisiontask"])
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a"}
graph = Graph(nodes={"tid-a"}, edges=set())
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task.get("dependencies"), ["decisiontask"])
