def test_multiple_subs(self):
input = [{'task-reference': 'toolchain=<toolchain>, build=<build>'}]
self.assertEqual(
resolve_task_references('lable', input, {
'toolchain': 'abcd',
'build': 'def'
}), ['toolchain=abcd, build=def'])
def test_invalid(self):
"resolve_task_references raises a KeyError on reference to an invalid task"
self.assertRaisesRegexp(
KeyError,
"task 'subject' has no dependency named 'no-such'",
lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {})
)
def test_buried_replacement(self):
input = {"key": [{"key2": [{'task-reference': 'taskid=<toolchain>'}]}]}
self.assertEqual(
resolve_task_references('lable', input, {'toolchain': 'abcd'}),
{u'key': [{
u'key2': [u'taskid=abcd']
}]})
def test_badly_formed(self):
"resolve_task_references ignores badly-formatted artifact references"
for inv in ['<edge1>', 'edge1/foo>', '<edge1>/foo', '<edge1>foo']:
resolved = resolve_task_references("subject",
{"artifact-reference": inv},
"tid-self", "tid-decision", {})
self.assertEqual(resolved, inv)
def test_no_change(self):
input = {
"key": "value",
"numeric": 10,
"list": ["a", True, False, None]
}
self.assertEqual(resolve_task_references('lable', input, {}), input)
def add_new_jobs_action(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
for elem in input['tasks']:
if elem in full_task_graph.tasks:
task = full_task_graph.tasks[elem]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task,
dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
# actually create the new task
create_task(slugid(), task_def, parameters['level'])
else:
raise Exception('{} was not found in the task-graph'.format(elem))
def test_appears_with_other_keys(self):
input = [{'task-reference': '<toolchain>', 'another-key': True}]
self.assertEqual(
resolve_task_references('lable', input, {'toolchain': 'abcd'}),
[{
'task-reference': '<toolchain>',
'another-key': True
}])
def do(self, input, output):
taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
with mock.patch.dict(
os.environ,
{'TASKCLUSTER_ROOT_URL': 'https://tc-tests.localhost'}):
self.assertEqual(
resolve_task_references('subject', input,
taskid_for_edge_name), output)
def test_invalid(self):
"resolve_task_references raises a KeyError on reference to an invalid task"
self.assertRaisesRegexp(
KeyError,
"task 'subject' has no dependency named 'no-such'",
lambda: resolve_task_references(
"subject", {"artifact-reference": "<no-such/public/artifact>"
}, "tid-self", "tid-decision", {}),
)
def test_self(self):
"resolve_task_references raises KeyError on artifact references to `self`"
self.assertRaisesRegexp(
KeyError,
"task 'subject' can't reference artifacts of self",
lambda: resolve_task_references(
"subject", {"artifact-reference": "<self/public/artifact>"
}, "tid-self", "tid-decision", {}),
)
def mochitest_retrigger_action(parameters, graph_config, input, task_group_id, task_id, task):
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
parameters, graph_config)
pre_task = full_task_graph.tasks[task['metadata']['name']]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {name: label_to_taskid[label]
for name, label in pre_task.dependencies.iteritems()}
new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false',
'-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += ['--log-tbpl=-',
'--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input.get('repeat', 30))]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def mochitest_retrigger_action(parameters, input, task_group_id, task_id, task):
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(parameters)
pre_task = full_task_graph.tasks[task['metadata']['name']]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {name: label_to_taskid[label]
for name, label in pre_task.dependencies.iteritems()}
new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false',
'-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += ['--log-tbpl=-',
'--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input.get('repeat', 30))]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def do(self, input, output):
taskid_for_edge_name = {"edge%d" % n: "tid%d" % n for n in range(1, 4)}
with mock.patch.dict(
os.environ,
{"TASKCLUSTER_ROOT_URL": "https://tc-tests.localhost"}):
self.assertEqual(
resolve_task_references("subject", input, "tid-self",
"tid-decision", taskid_for_edge_name),
output,
)
def do(self, input, output):
taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
self.assertEqual(
resolve_task_references(
"subject",
input,
"tid-self",
"tid-decision",
taskid_for_edge_name,
),
output,
)
def backfill_action(parameters, input, task_group_id, task_id, task):
label = task['metadata']['name']
pushes = []
depth = input.get('depth', 5)
end_id = int(parameters['pushlog_id']) - 1
while True:
start_id = max(end_id - depth, 0)
pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'],
start_id, end_id)
r = requests.get(pushlog_url)
r.raise_for_status()
pushes = pushes + r.json()['pushes'].keys()
if len(pushes) >= depth:
break
end_id = start_id - 1
start_id -= depth
if start_id < 0:
break
pushes = sorted(pushes)[-depth:]
for push in pushes:
full_task_graph = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/full-task-graph.json')
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/label-to-taskid.json')
if label in full_task_graph.tasks.keys():
task = full_task_graph.tasks[label]
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task,
dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
create_task(slugid(), task_def, parameters['level'])
else:
logging.info('Could not find {} on {}. Skipping.'.format(
label, push))
def run_missing_tests(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
target_tasks = get_artifact(decision_task_id, "public/target-tasks.json")
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
# The idea here is to schedule all tasks of the `test` kind that were
# targetted but did not appear in the final task-graph -- those were the
# optimized tasks.
to_run = []
already_run = 0
for label in target_tasks:
task = full_task_graph.tasks[label]
if task.kind != 'test':
continue # not a test
if label in label_to_taskid:
already_run += 1
continue
to_run.append(task)
for task in to_run:
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task, dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
create_task(slugid(), task_def, parameters['level'])
logger.info(
'Out of {} test tasks, {} already existed and the action created {}'.
format(already_run + len(to_run), already_run, len(to_run)))
def test_escaped(self):
input = [{'task-reference': '<<><toolchain>>'}]
self.assertEqual(
resolve_task_references('lable', input, {'toolchain': 'abcd'}),
['<abcd>'])
def do(self, input, output):
taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), output)
def get_subgraph(
target_task_graph,
removed_tasks,
replaced_tasks,
label_to_taskid,
decision_task_id,
):
"""
Return the subgraph of target_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# check for any dependency edges from included to removed tasks
bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges
if l not in removed_tasks and r in removed_tasks]
if bad_edges:
probs = ', '.join(
'{} depends on {} as {} but it has been removed'.format(l, r, n)
for l, r, n in bad_edges)
raise Exception("Optimization error: " + probs)
# fill in label_to_taskid for anything not removed or replaced
assert replaced_tasks <= set(label_to_taskid)
for label in sorted(target_task_graph.graph.nodes - removed_tasks -
set(label_to_taskid)):
label_to_taskid[label] = slugid().decode('ascii')
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
named_links_dict = target_task_graph.graph.named_links_dict()
omit = removed_tasks | replaced_tasks
for label, task in six.iteritems(target_task_graph.tasks):
if label in omit:
continue
task.task_id = label_to_taskid[label]
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).items()
}
# Add remaining soft dependencies
if task.soft_dependencies:
named_task_dependencies.update({
label: label_to_taskid[label]
for label in task.soft_dependencies
if label in label_to_taskid and label not in omit
})
task.task = resolve_task_references(
task.label,
task.task,
task_id=task.task_id,
decision_task_id=decision_task_id,
dependencies=named_task_dependencies,
)
deps = task.task.setdefault('dependencies', [])
deps.extend(sorted(named_task_dependencies.values()))
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right),
name) for (left, right,
name) in target_task_graph.graph.edges)
# ..and drop edges that are no longer entirely in the task graph
# (note that this omits edges to replaced tasks, but they are still in task.dependnecies)
edges_by_taskid = set(
(left, right, name) for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid)
return TaskGraph(tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def do(self, input, output):
taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
self.assertEqual(
resolve_task_references('subject', input, taskid_for_edge_name),
output)
def test_invalid(self):
"resolve_task_references ignores badly-formatted artifact references"
for inv in ['<edge1>', 'edge1/foo>', '<edge1>/foo', '<edge1>foo']:
resolved = resolve_task_references('subject',
{'artifact-reference': inv}, {})
self.assertEqual(resolved, inv)
def test_invalid(self):
"resolve_task_references raises a KeyError on reference to an invalid task"
self.assertRaisesRegexp(
KeyError, "task 'subject' has no dependency named 'no-such'",
lambda: resolve_task_references(
'subject', {'task-reference': '<no-such>'}, {}))