def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
description='Task A',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimization={'skip-unless-has-relevant-tests': None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'description': 'Task A',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'soft_dependencies': [],
'optimization': None,
},
'b': {
'kind': 'test',
'label': 'b',
'description': '',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'soft_dependencies': [],
'optimization': {
'skip-unless-has-relevant-tests': None
},
}
})
def test_round_trip(self):
graph = TaskGraph(
tasks={
"a": Task(
kind="fancy",
label="a",
description="Task A",
attributes={},
dependencies={"prereq": "b"}, # must match edges, below
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def"},
),
"b": Task(
kind="pre",
label="b",
attributes={},
dependencies={},
optimization={"skip-unless-has-relevant-tests": None},
task={"task": "def2"},
),
},
graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}),
)
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def create_tasks(to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l]
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph, params, to_run, label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params,
decision_task_id)
def test_taskgraph_to_json(self):
tasks = {
'a':
Task(kind='test',
label='a',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b':
Task(
kind='test',
label='b',
attributes={},
task={'task': 'def'},
optimizations=[['seta']],
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res, {
'a': {
'kind': 'test',
'label': 'a',
'attributes': {
'attr': 'a-task',
'kind': 'test'
},
'task': {
'taskdef': True
},
'dependencies': {
'edgelabel': 'b'
},
'optimizations': [],
},
'b': {
'kind': 'test',
'label': 'b',
'attributes': {
'kind': 'test'
},
'task': {
'task': 'def'
},
'dependencies': {},
'optimizations': [['seta']],
}
})
def create_tasks(graph_config,
to_run,
full_task_graph,
label_to_taskid,
params,
decision_task_id=None,
suffix='',
modifier=lambda t: t):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
Optionally this function takes a `modifier` function that is passed in each
task before it is put into a new graph. It should return a valid task. Note
that this is passed _all_ tasks in the graph, not just the set in to_run. You
may want to skip modifying tasks not in your to_run list.
If `suffix` is given, then it is used to give unique names to the resulting
artifacts. If you call this function multiple times in the same action,
pass a different suffix each time to avoid overwriting artifacts.
If you wish to create the tasks in a new group, leave out decision_task_id.
Returns an updated label_to_taskid containing the new tasks"""
if suffix != '':
suffix = '-{}'.format(suffix)
to_run = set(to_run)
# Copy to avoid side-effects later
full_task_graph = copy.deepcopy(full_task_graph)
label_to_taskid = label_to_taskid.copy()
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: modifier(full_task_graph[l])
for l in target_graph.nodes}, target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(
target_task_graph, params, to_run, label_to_taskid)
write_artifact('task-graph{}.json'.format(suffix),
optimized_task_graph.to_json())
write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid)
write_artifact('to-run{}.json'.format(suffix), list(to_run))
create.create_tasks(
graph_config,
optimized_task_graph,
label_to_taskid,
params,
decision_task_id,
)
return label_to_taskid
def test_taskgraph_to_json(self):
tasks = {
"a": Task(
kind="test",
label="a",
description="Task A",
attributes={"attr": "a-task"},
task={"taskdef": True},
),
"b": Task(
kind="test",
label="b",
attributes={},
task={"task": "def"},
optimization={"skip-unless-has-relevant-tests": None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={"first": "a"},
),
}
graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(
res,
{
"a": {
"kind": "test",
"label": "a",
"description": "Task A",
"attributes": {"attr": "a-task", "kind": "test"},
"task": {"taskdef": True},
"dependencies": {"edgelabel": "b"},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": None,
},
"b": {
"kind": "test",
"label": "b",
"description": "",
"attributes": {"kind": "test"},
"task": {"task": "def"},
"dependencies": {},
"soft_dependencies": [],
"if_dependencies": [],
"optimization": {"skip-unless-has-relevant-tests": None},
},
},
)
def test_create_tasks(self):
tasks = {
"tid-a":
Task(kind="test",
label="a",
attributes={},
task={"payload": "hello world"}),
"tid-b":
Task(kind="test",
label="b",
attributes={},
task={"payload": "hello world"}),
}
label_to_taskid = {"a": "tid-a", "b": "tid-b"}
graph = Graph(nodes={"tid-a", "tid-b"},
edges={("tid-a", "tid-b", "edge")})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.items():
self.assertEqual(task["payload"], "hello world")
self.assertEqual(task["schedulerId"], "domain-level-4")
# make sure the dependencies exist, at least
for depid in task.get("dependencies", []):
if depid == "decisiontask":
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def generate_tasks(params=None, full=False, disable_target_task_filter=False):
cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph")
attr = "full_task_set" if full else "target_task_set"
cache = os.path.join(cache_dir, attr)
invalidate(cache)
if os.path.isfile(cache):
with open(cache, "r") as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(
attr.replace("_", " ")))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(build.topsrcdir)
root = os.path.join(build.topsrcdir, "taskcluster", "ci")
target_tasks_method = ("try_select_tasks" if not disable_target_task_filter
else "try_select_tasks_uncommon")
params = parameters_loader(
params,
strict=False,
overrides={
"try_mode": "try_select",
"target_tasks_method": target_tasks_method,
},
)
# Cache both full_task_set and target_task_set regardless of whether or not
# --full was requested. Caching is cheap and can potentially save a lot of
# time.
generator = TaskGraphGenerator(root_dir=root, parameters=params)
def generate(attr):
try:
tg = getattr(generator, attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
# write cache
with open(os.path.join(cache_dir, attr), "w") as fh:
json.dump(tg.to_json(), fh)
return tg
tg_full = generate("full_task_set")
tg_target = generate("target_task_set")
# discard results from these, we only need cache.
if full:
generate("full_task_graph")
generate("target_task_graph")
os.chdir(cwd)
if full:
return tg_full
return tg_target
def run_missing_tests(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
target_tasks = get_artifact(decision_task_id, "public/target-tasks.json")
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
# The idea here is to schedule all tasks of the `test` kind that were
# targetted but did not appear in the final task-graph -- those were the
# optimized tasks.
to_run = []
already_run = 0
for label in target_tasks:
task = full_task_graph.tasks[label]
if task.kind != 'test':
continue # not a test
if label in label_to_taskid:
already_run += 1
continue
to_run.append(label)
create_tasks(to_run, full_task_graph, label_to_taskid, parameters,
decision_task_id)
logger.info(
'Out of {} test tasks, {} already existed and the action created {}'.
format(already_run + len(to_run), already_run, len(to_run)))
def retrigger_action(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
label = task['metadata']['name']
with_downstream = ' '
to_run = [label]
if input.get('downstream'):
to_run = full_task_graph.graph.transitive_closure(set(to_run),
reverse=True).nodes
to_run = to_run & set(label_to_taskid.keys())
with_downstream = ' (with downstream) '
times = input.get('times', 1)
for i in xrange(times):
create_tasks(to_run, full_task_graph, label_to_taskid, parameters,
decision_task_id)
logger.info('Scheduled {}{}(time {}/{})'.format(
label, with_downstream, i + 1, times))
def test_create_tasks(self):
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
'tid-b':
Task(kind='test',
label='b',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'},
edges={('tid-a', 'tid-b', 'edge')})
taskgraph = TaskGraph(tasks, graph)
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['payload'], 'hello world')
self.assertEqual(task['schedulerId'], 'domain-level-4')
# make sure the dependencies exist, at least
for depid in task.get('dependencies', []):
if depid == 'decisiontask':
# Don't look for decisiontask here
continue
self.assertIn(depid, self.created_tasks)
def add_new_jobs_action(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
for elem in input['tasks']:
if elem in full_task_graph.tasks:
task = full_task_graph.tasks[elem]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task,
dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
# actually create the new task
create_task(slugid(), task_def, parameters['level'])
else:
raise Exception('{} was not found in the task-graph'.format(elem))
def inner(tasks):
label_to_taskid = {k: k + '-tid' for k in tasks}
for label, task_id in label_to_taskid.iteritems():
tasks[label].task_id = task_id
graph = Graph(nodes=set(tasks), edges=set())
taskgraph = TaskGraph(tasks, graph)
return taskgraph, label_to_taskid
def make_taskgraph(self, tasks):
label_to_taskid = {k: k + "-tid" for k in tasks}
for label, task_id in label_to_taskid.items():
tasks[label].task_id = task_id
graph = Graph(nodes=set(tasks), edges=set())
taskgraph = TaskGraph(tasks, graph)
return taskgraph, label_to_taskid
def fetch_graph_and_labels(parameters):
decision_task_id = find_decision_task(parameters)
# First grab the graph and labels generated during the initial decision task
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
# Now fetch any modifications made by action tasks and swap out new tasks
# for old ones
namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format(
parameters['project'], parameters['pushlog_id'])
for action in list_tasks(namespace):
try:
run_label_to_id = get_artifact(action,
"public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
logger.info(
'Skipping {} due to missing artifact! Error: {}'.format(
action, e))
continue
return (decision_task_id, full_task_graph, label_to_taskid)
def get_filtered_taskgraph(self, taskgraph, tasksregex):
from taskgraph.graph import Graph
from taskgraph.taskgraph import TaskGraph
"""
This class method filters all the tasks on basis of a regular expression
and returns a new TaskGraph object
"""
# return original taskgraph if no regular expression is passed
if not tasksregex:
return taskgraph
named_links_dict = taskgraph.graph.named_links_dict()
filteredtasks = {}
filterededges = set()
regexprogram = re.compile(tasksregex)
for key in taskgraph.graph.visit_postorder():
task = taskgraph.tasks[key]
if regexprogram.match(task.label):
filteredtasks[key] = task
for depname, dep in named_links_dict[key].iteritems():
if regexprogram.match(dep):
filterededges.add((key, dep, depname))
filtered_taskgraph = TaskGraph(
filteredtasks, Graph(set(filteredtasks), filterededges))
return filtered_taskgraph
def test_create_tasks_fails_if_create_fails(self, create_task):
"creat_tasks fails if a single create_task call fails"
tasks = {
'tid-a':
Task(kind='test',
label='a',
attributes={},
task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
taskgraph = TaskGraph(tasks, graph)
def fail(*args):
print("UHOH")
raise RuntimeError('oh noes!')
create_task.side_effect = fail
with self.assertRaises(RuntimeError):
create.create_tasks(
GRAPH_CONFIG,
taskgraph,
label_to_taskid,
{"level": "4"},
decision_task_id="decisiontask",
)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][
release_promotion_flavor]
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds') or promotion_config.get(
'rebuild-kinds', [])
do_not_optimize = input.get('do_not_optimize') or promotion_config.get(
'do-not-optimize', [])
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids`` or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
parameters['xpi_name'] = input['xpi_name']
# TODO
# - require this is a specific revision
# - possibly also check that this is on a reviewed PR or merged into
# a trusted branch. this will require an oauth token
parameters['xpi_revision'] = input.get('revision', 'master')
parameters['shipping_phase'] = input['release_promotion_flavor']
# We blow away `tasks_for` when we load the on-push decision task's
# parameters.yml. Let's set this back to `action`.
parameters['tasks_for'] = "action"
if input.get('version'):
parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def default_matches(self, attributes, parameters):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build', label='a', attributes=attributes, task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
return 'a' in method(graph, parameters, {})
def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id):
task = taskcluster.get_task_definition(task_id)
label = task['metadata']['name']
pushes = []
depth = 2
end_id = int(parameters['pushlog_id'])
while True:
start_id = max(end_id - depth, 0)
pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id)
r = requests.get(pushlog_url)
r.raise_for_status()
pushes = pushes + list(r.json()['pushes'].keys())
if len(pushes) >= depth:
break
end_id = start_id - 1
start_id -= depth
if start_id < 0:
break
pushes = sorted(pushes)[-depth:]
backfill_pushes = []
for push in pushes:
try:
full_task_graph = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/full-task-graph.json')
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/label-to-taskid.json')
push_params = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/parameters.yml')
push_decision_task_id = find_decision_task(push_params, graph_config)
except HTTPError as e:
logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e))
continue
if label in full_task_graph.tasks.keys():
def modifier(task):
if task.label != label:
return task
cmd = task.task['payload']['command']
task.task['payload']['command'] = add_args_to_perf_command(
cmd, ['--gecko-profile'])
task.task['extra']['treeherder']['symbol'] += '-p'
return task
create_tasks(graph_config, [label], full_task_graph, label_to_taskid,
push_params, push_decision_task_id, push, modifier=modifier)
backfill_pushes.append(push)
else:
logging.info('Could not find {} on {}. Skipping.'.format(label, push))
combine_task_graph_files(backfill_pushes)
def generate_tasks(params=None, full=False):
# TODO: Remove after January 1st, 2020.
# Try to delete the old taskgraph cache directories.
root = build.topsrcdir
root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
old_cache_dirs = [
os.path.join(get_state_dir(), 'cache', 'taskgraph'),
os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph'),
]
for cache_dir in old_cache_dirs:
if os.path.isdir(cache_dir):
shutil.rmtree(cache_dir)
cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph')
attr = 'full_task_set' if full else 'target_task_set'
cache = os.path.join(cache_dir, attr)
invalidate(cache, root)
if os.path.isfile(cache):
with open(cache, 'r') as fh:
return TaskGraph.from_json(json.load(fh))[1]
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
taskgraph.fast = True
cwd = os.getcwd()
os.chdir(root)
root = os.path.join(root, 'taskcluster', 'ci')
params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'})
# Cache both full_task_set and target_task_set regardless of whether or not
# --full was requested. Caching is cheap and can potentially save a lot of
# time.
generator = TaskGraphGenerator(root_dir=root, parameters=params)
def generate(attr):
try:
tg = getattr(generator, attr)
except ParameterMismatch as e:
print(PARAMETER_MISMATCH.format(e.args[0]))
sys.exit(1)
# write cache
with open(os.path.join(cache_dir, attr), 'w') as fh:
json.dump(tg.to_json(), fh)
return tg
tg_full = generate('full_task_set')
tg_target = generate('target_task_set')
os.chdir(cwd)
if full:
return tg_full
return tg_target
def fetch_graph_and_labels(parameters, graph_config):
decision_task_id = find_decision_task(parameters, graph_config)
# First grab the graph and labels generated during the initial decision task
full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json")
# fetch everything in parallel; this avoids serializing any delay in downloading
# each artifact (such as waiting for the artifact to be mirrored locally)
with futures.ThreadPoolExecutor(CONCURRENCY) as e:
fetches = []
# fetch any modifications made by action tasks and swap out new tasks
# for old ones
def fetch_action(task_id):
logger.info(f"fetching label-to-taskid.json for action task {task_id}")
try:
run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
if e.response.status_code != 404:
raise
logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
namespace = "{}.v2.{}.pushlog-id.{}.actions".format(
graph_config["trust-domain"],
parameters["project"],
parameters["pushlog_id"],
)
for task_id in list_tasks(namespace):
fetches.append(e.submit(fetch_action, task_id))
# Similarly for cron tasks..
def fetch_cron(task_id):
logger.info(f"fetching label-to-taskid.json for cron task {task_id}")
try:
run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
if e.response.status_code != 404:
raise
logger.debug(f"No label-to-taskid.json found for {task_id}: {e}")
namespace = "{}.v2.{}.revision.{}.cron".format(
graph_config["trust-domain"], parameters["project"], parameters["head_rev"]
)
for task_id in list_tasks(namespace):
fetches.append(e.submit(fetch_cron, task_id))
# now wait for each fetch to complete, raising an exception if there
# were any issues
for f in futures.as_completed(fetches):
f.result()
return (decision_task_id, full_task_graph, label_to_taskid)
def make_graph(*tasks_and_edges, **kwargs):
tasks = {t.label: t for t in tasks_and_edges if isinstance(t, Task)}
edges = {e for e in tasks_and_edges if not isinstance(e, Task)}
tg = TaskGraph(tasks, graph.Graph(set(tasks), edges))
if kwargs.get("deps", True):
# set dependencies based on edges
for l, r, name in tg.graph.edges:
tg.tasks[l].dependencies[name] = r
return tg
def tg(request):
if not hasattr(request.module, 'TASKS'):
pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable")
tasks = request.module.TASKS
for task in tasks:
task.setdefault('task', {})
task['task'].setdefault('tags', {})
tasks = {t['label']: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_round_trip(self):
graph = TaskGraph(tasks={
'a': Task(
kind='fancy',
label='a',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimization={'seta': None},
task={'task': 'def'}),
'b': Task(
kind='pre',
label='b',
attributes={},
dependencies={},
optimization={'seta': None},
task={'task': 'def2'}),
}, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def test_round_trip(self):
graph = TaskGraph(tasks={
'a': Task(
kind='fancy',
label='a',
description='Task A',
attributes={},
dependencies={'prereq': 'b'}, # must match edges, below
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def'}),
'b': Task(
kind='pre',
label='b',
attributes={},
dependencies={},
optimization={'skip-unless-has-relevant-tests': None},
task={'task': 'def2'}),
}, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')}))
tasks, new_graph = TaskGraph.from_json(graph.to_json())
self.assertEqual(graph, new_graph)
def default_matches(self, run_on_projects, project):
method = target_tasks.get_method('default')
graph = TaskGraph(tasks={
'a':
Task(kind='build',
label='a',
attributes={'run_on_projects': run_on_projects},
task={}),
},
graph=Graph(nodes={'a'}, edges=set()))
parameters = {'project': project}
return 'a' in method(graph, parameters)
def default_matches(self, attributes, parameters):
method = target_tasks.get_method("default")
graph = TaskGraph(
tasks={
"a": Task(kind="build",
label="a",
attributes=attributes,
task={}),
},
graph=Graph(nodes={"a"}, edges=set()),
)
return "a" in method(graph, parameters, {})
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None):
"""Create new tasks. The task definition will have {relative-datestamp':
'..'} rendered just like in a decision task. Action callbacks should use
this function to create new tasks,
allowing easy debugging with `mach taskgraph action-callback --test`.
This builds up all required tasks to run in order to run the tasks requested.
If you wish to create the tasks in a new group, leave out decision_task_id."""
to_run = set(to_run)
target_graph = full_task_graph.graph.transitive_closure(to_run)
target_task_graph = TaskGraph(
{l: full_task_graph[l] for l in target_graph.nodes},
target_graph)
target_task_graph.for_each_task(update_parent)
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
params,
to_run,
label_to_taskid)
write_artifact('task-graph.json', optimized_task_graph.to_json())
write_artifact('label-to-taskid.json', label_to_taskid)
write_artifact('to-run.json', list(to_run))
create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def tg(request):
if not hasattr(request.module, "TASKS"):
pytest.fail(
"'tg' fixture used from a module that didn't define the TASKS variable"
)
tasks = request.module.TASKS
for task in tasks:
task.setdefault("task", {})
task["task"].setdefault("tags", {})
tasks = {t["label"]: Task(**t) for t in tasks}
return TaskGraph(tasks, Graph(tasks.keys(), set()))
def make_task_graph(self):
tasks = {
'a':
Task(kind=None, label='a', attributes={}, task={}),
'b':
Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}),
'c':
Task(kind=None,
label='c',
attributes={'run_on_projects': ['try']},
task={}),
}
graph = Graph(nodes=set('abc'), edges=set())
return TaskGraph(tasks, graph)
def make_task_graph(self):
tasks = {
"a":
Task(kind=None, label="a", attributes={}, task={}),
"b":
Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}),
"c":
Task(kind=None,
label="c",
attributes={"run_on_projects": ["try"]},
task={}),
}
graph = Graph(nodes=set("abc"), edges=set())
return TaskGraph(tasks, graph)
def test_taskgraph_to_json(self):
tasks = {
'a': Task(kind='test', label='a',
attributes={'attr': 'a-task'},
task={'taskdef': True}),
'b': Task(kind='test', label='b',
attributes={},
task={'task': 'def'},
optimization={'seta': None},
# note that this dep is ignored, superseded by that
# from the taskgraph's edges
dependencies={'first': 'a'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
res = taskgraph.to_json()
self.assertEqual(res, {
'a': {
'kind': 'test',
'label': 'a',
'attributes': {'attr': 'a-task', 'kind': 'test'},
'task': {'taskdef': True},
'dependencies': {'edgelabel': 'b'},
'optimization': None,
},
'b': {
'kind': 'test',
'label': 'b',
'attributes': {'kind': 'test'},
'task': {'task': 'def'},
'dependencies': {},
'optimization': {'seta': None},
}
})
def fetch_graph_and_labels(parameters):
decision_task_id = find_decision_task(parameters)
# First grab the graph and labels generated during the initial decision task
full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json")
# Now fetch any modifications made by action tasks and swap out new tasks
# for old ones
namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format(
parameters['project'],
parameters['pushlog_id'])
for action in list_tasks(namespace):
try:
run_label_to_id = get_artifact(action, "public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
logger.info('Skipping {} due to missing artifact! Error: {}'.format(action, e))
continue
return (decision_task_id, full_task_graph, label_to_taskid)
def release_promotion_action(parameters, input, task_group_id, task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
release_history = {}
desktop_release_type = None
next_version = str(input.get('next_version') or '')
if release_promotion_flavor in VERSION_BUMP_FLAVORS:
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for %s "
"targets." % ', '.join(VERSION_BUMP_FLAVORS)
)
if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS:
desktop_release_type = input.get('desktop_release_type', None)
if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES:
raise Exception("`desktop_release_type` must be one of: %s" %
", ".join(VALID_DESKTOP_RELEASE_TYPES))
if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS:
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for %s "
"targets." % ', '.join(PARTIAL_UPDATES_FLAVORS)
)
balrog_prefix = 'Firefox'
if desktop_release_type == 'devedition':
balrog_prefix = 'Devedition'
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix, parameters['project'],
partial_updates=input['partial_updates']
)
if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS:
uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', []))
if partial_updates == "[]":
raise Exception(
"`uptake_monitoring_platforms` property needs to be provided for %s "
"targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS)
)
os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
target_tasks_method = input.get(
'target_tasks_method',
promotion_config['target_tasks_method'].format(project=parameters['project'])
)
rebuild_kinds = input.get(
'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
)
do_not_optimize = input.get(
'do_not_optimize', promotion_config.get('do_not_optimize', [])
)
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
parameters['pushlog_id'] = parameters['pushlog_id'] or \
find_hg_revision_pushlog_id(parameters, revision)
previous_graph_ids = [find_decision_task(parameters)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['desktop_release_type'] = desktop_release_type
parameters['release_eta'] = input.get('release_eta', '')
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({}, parameters=parameters)
