def combine_task_graph_files(suffixes):
"""Combine task-graph-{suffix}.json files into a single task-graph.json file.
Since Chain of Trust verification requires a task-graph.json file that
contains all children tasks, we can combine the various task-graph-0.json
type files into a master task-graph.json file at the end.
Actions also look for various artifacts, so we combine those in a similar
fashion.
In the case where there is only one suffix, we simply rename it to avoid the
additional cost of uploading two copies of the same data.
"""
if len(suffixes) == 1:
for filename in ["task-graph", "label-to-taskid", "to-run"]:
rename_artifact(
"{}-{}.json".format(filename, suffixes[0]), "{}.json".format(filename)
)
return
def combine(file_contents, base):
return reduce(_update_reducer, file_contents, base)
files = [read_artifact("task-graph-{}.json".format(suffix)) for suffix in suffixes]
write_artifact("task-graph.json", combine(files, dict()))
files = [
read_artifact("label-to-taskid-{}.json".format(suffix)) for suffix in suffixes
]
write_artifact("label-to-taskid.json", combine(files, dict()))
files = [read_artifact("to-run-{}.json".format(suffix)) for suffix in suffixes]
write_artifact("to-run.json", list(combine(files, set())))
def test_several_suffixes(self):
files = {
"artifacts/task-graph-0.json": json.dumps({"taska": {}}),
"artifacts/label-to-taskid-0.json": json.dumps({"taska": "TASKA"}),
"artifacts/to-run-0.json": json.dumps(["taska"]),
"artifacts/task-graph-1.json": json.dumps({"taskb": {}}),
"artifacts/label-to-taskid-1.json": json.dumps({"taskb": "TASKB"}),
"artifacts/to-run-1.json": json.dumps(["taskb"]),
}
with MockedOpen(files):
combine_task_graph_files(["0", "1"])
self.assertEqual(
read_artifact("task-graph.json"),
{
"taska": {},
"taskb": {},
},
)
self.assertEqual(
read_artifact("label-to-taskid.json"),
{
"taska": "TASKA",
"taskb": "TASKB",
},
)
self.assertEqual(
sorted(read_artifact("to-run.json")),
[
"taska",
"taskb",
],
)
def combine_task_graph_files(suffixes):
"""Combine task-graph-{suffix}.json files into a single task-graph.json file.
Since Chain of Trust verification requires a task-graph.json file that
contains all children tasks, we can combine the various task-graph-0.json
type files into a master task-graph.json file at the end."""
all = {}
for suffix in suffixes:
all.update(read_artifact('task-graph-{}.json'.format(suffix)))
write_artifact('task-graph.json', all)
def test_several_suffixes(self):
files = {
'artifacts/task-graph-0.json': json.dumps({'taska': {}}),
'artifacts/label-to-taskid-0.json': json.dumps({'taska': 'TASKA'}),
'artifacts/to-run-0.json': json.dumps(['taska']),
'artifacts/task-graph-1.json': json.dumps({'taskb': {}}),
'artifacts/label-to-taskid-1.json': json.dumps({'taskb': 'TASKB'}),
'artifacts/to-run-1.json': json.dumps(['taskb']),
}
with MockedOpen(files):
combine_task_graph_files(['0', '1'])
self.assertEqual(read_artifact('task-graph.json'), {
'taska': {},
'taskb': {},
})
self.assertEqual(read_artifact('label-to-taskid.json'), {
'taska': 'TASKA',
'taskb': 'TASKB',
})
self.assertEqual(sorted(read_artifact('to-run.json')), [
'taska',
'taskb',
])
