def combine_task_graph_files(suffixes): """Combine task-graph-{suffix}.json files into a single task-graph.json file. Since Chain of Trust verification requires a task-graph.json file that contains all children tasks, we can combine the various task-graph-0.json type files into a master task-graph.json file at the end. Actions also look for various artifacts, so we combine those in a similar fashion. In the case where there is only one suffix, we simply rename it to avoid the additional cost of uploading two copies of the same data. """ if len(suffixes) == 1: for filename in ["task-graph", "label-to-taskid", "to-run"]: rename_artifact( "{}-{}.json".format(filename, suffixes[0]), "{}.json".format(filename) ) return def combine(file_contents, base): return reduce(_update_reducer, file_contents, base) files = [read_artifact("task-graph-{}.json".format(suffix)) for suffix in suffixes] write_artifact("task-graph.json", combine(files, dict())) files = [ read_artifact("label-to-taskid-{}.json".format(suffix)) for suffix in suffixes ] write_artifact("label-to-taskid.json", combine(files, dict())) files = [read_artifact("to-run-{}.json".format(suffix)) for suffix in suffixes] write_artifact("to-run.json", list(combine(files, set())))
def test_several_suffixes(self): files = { "artifacts/task-graph-0.json": json.dumps({"taska": {}}), "artifacts/label-to-taskid-0.json": json.dumps({"taska": "TASKA"}), "artifacts/to-run-0.json": json.dumps(["taska"]), "artifacts/task-graph-1.json": json.dumps({"taskb": {}}), "artifacts/label-to-taskid-1.json": json.dumps({"taskb": "TASKB"}), "artifacts/to-run-1.json": json.dumps(["taskb"]), } with MockedOpen(files): combine_task_graph_files(["0", "1"]) self.assertEqual( read_artifact("task-graph.json"), { "taska": {}, "taskb": {}, }, ) self.assertEqual( read_artifact("label-to-taskid.json"), { "taska": "TASKA", "taskb": "TASKB", }, ) self.assertEqual( sorted(read_artifact("to-run.json")), [ "taska", "taskb", ], )
def combine_task_graph_files(suffixes): """Combine task-graph-{suffix}.json files into a single task-graph.json file. Since Chain of Trust verification requires a task-graph.json file that contains all children tasks, we can combine the various task-graph-0.json type files into a master task-graph.json file at the end.""" all = {} for suffix in suffixes: all.update(read_artifact('task-graph-{}.json'.format(suffix))) write_artifact('task-graph.json', all)
def test_several_suffixes(self): files = { 'artifacts/task-graph-0.json': json.dumps({'taska': {}}), 'artifacts/label-to-taskid-0.json': json.dumps({'taska': 'TASKA'}), 'artifacts/to-run-0.json': json.dumps(['taska']), 'artifacts/task-graph-1.json': json.dumps({'taskb': {}}), 'artifacts/label-to-taskid-1.json': json.dumps({'taskb': 'TASKB'}), 'artifacts/to-run-1.json': json.dumps(['taskb']), } with MockedOpen(files): combine_task_graph_files(['0', '1']) self.assertEqual(read_artifact('task-graph.json'), { 'taska': {}, 'taskb': {}, }) self.assertEqual(read_artifact('label-to-taskid.json'), { 'taska': 'TASKA', 'taskb': 'TASKB', }) self.assertEqual(sorted(read_artifact('to-run.json')), [ 'taska', 'taskb', ])