def combine_task_graph_files(suffixes):
"""Combine task-graph-{suffix}.json files into a single task-graph.json file.
Since Chain of Trust verification requires a task-graph.json file that
contains all children tasks, we can combine the various task-graph-0.json
type files into a master task-graph.json file at the end."""
all = {}
for suffix in suffixes:
all.update(read_artifact('task-graph-{}.json'.format(suffix)))
write_artifact('task-graph.json', all)
def test_write_artifact_yml(self):
data = [{"some": "data"}]
tmpdir = tempfile.mkdtemp()
try:
decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
decision.write_artifact("artifact.yml", data)
self.assertEqual(load_yaml(decision.ARTIFACTS_DIR, "artifact.yml"), data)
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
decision.ARTIFACTS_DIR = "artifacts"
def test_write_artifact_yml(self):
data = [{'some': 'data'}]
tmpdir = tempfile.mkdtemp()
try:
decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
decision.write_artifact("artifact.yml", data)
with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f:
self.assertEqual(yaml.safe_load(f), data)
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
decision.ARTIFACTS_DIR = 'artifacts'
def test_write_artifact_json(self):
data = [{"some": "data"}]
tmpdir = tempfile.mkdtemp()
try:
decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
decision.write_artifact("artifact.json", data)
with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
self.assertEqual(json.load(f), data)
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
decision.ARTIFACTS_DIR = "artifacts"
def test_write_artifact_yml(self):
data = [{'some': 'data'}]
tmpdir = tempfile.mkdtemp()
try:
decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
decision.write_artifact("artifact.yml", data)
with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f:
self.assertEqual(yaml.safe_load(f), data)
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
decision.ARTIFACTS_DIR = 'artifacts'
def combine_task_graph_files(suffixes):
"""Combine task-graph-{suffix}.json files into a single task-graph.json file.
Since Chain of Trust verification requires a task-graph.json file that
contains all children tasks, we can combine the various task-graph-0.json
type files into a master task-graph.json file at the end.
Actions also look for various artifacts, so we combine those in a similar
fashion.
In the case where there is only one suffix, we simply rename it to avoid the
additional cost of uploading two copies of the same data.
"""
if len(suffixes) == 1:
for filename in ["task-graph", "label-to-taskid", "to-run"]:
rename_artifact(
"{}-{}.json".format(filename, suffixes[0]), "{}.json".format(filename)
)
return
def combine(file_contents, base):
return reduce(_update_reducer, file_contents, base)
files = [read_artifact("task-graph-{}.json".format(suffix)) for suffix in suffixes]
write_artifact("task-graph.json", combine(files, dict()))
files = [
read_artifact("label-to-taskid-{}.json".format(suffix)) for suffix in suffixes
]
write_artifact("label-to-taskid.json", combine(files, dict()))
files = [read_artifact("to-run-{}.json".format(suffix)) for suffix in suffixes]
write_artifact("to-run.json", list(combine(files, set())))
def create_tasks(to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l]
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph, params, to_run, label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params,
decision_task_id)
def create_tasks(graph_config,
to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id,
suffix='',
modifier=lambda t: t):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
Optionally this function takes a `modifier` function that is passed in each
task before it is put into a new graph. It should return a valid task. Note
that this is passed _all_ tasks in the graph, not just the set in to_run. You
may want to skip modifying tasks not in your to_run list.
If `suffix` is given, then it is used to give unique names to the resulting
artifacts. If you call this function multiple times in the same action,
pass a different suffix each time to avoid overwriting artifacts.
If you wish to create the tasks in a new group, leave out decision_task_id.
Returns an updated label_to_taskid containing the new tasks"""
if suffix != '':
suffix = '-{}'.format(suffix)
to_run = set(to_run)
# Copy to avoid side-effects later
full_task_graph = copy.deepcopy(full_task_graph)
label_to_taskid = label_to_taskid.copy()
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: modifier(full_task_graph[l])
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
if decision_task_id and decision_task_id != os.environ.get('TASK_ID'):
target_task_graph.for_each_task(update_dependencies)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph,
to_run,
params,
to_run,
decision_task_id,
existing_tasks=label_to_taskid)
write_artifact('task-graph{}.json'.format(suffix),
optimized_task_graph.to_json())
write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
write_artifact('to-run{}.json'.format(suffix), list(to_run))
create.create_tasks(
graph_config,
optimized_task_graph,
label_to_taskid,
params,
decision_task_id,
)
return label_to_taskid
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l] for l in target_graph.nodes},
target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
params,
to_run,
label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
