def create_tasks(taskgraph, label_to_taskid):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
session = requests.Session()
# Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
# that limit. Connections are established as needed, so using a large value
# should not negatively impact performance.
http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY,
pool_maxsize=CONCURRENCY)
session.mount('https://', http_adapter)
session.mount('http://', http_adapter)
decision_task_id = os.environ.get('TASK_ID')
with futures.ThreadPoolExecutor(CONCURRENCY) as e:
fs = {}
# We can't submit a task until its dependencies have been submitted.
# So our strategy is to walk the graph and submit tasks once all
# their dependencies have been submitted.
#
# Using visit_postorder() here isn't the most efficient: we'll
# block waiting for dependencies of task N to submit even though
# dependencies for task N+1 may be finished. If we need to optimize
# this further, we can build a graph of task dependencies and walk
# that.
for task_id in taskgraph.graph.visit_postorder():
task_def = taskgraph.tasks[task_id].task
attributes = taskgraph.tasks[task_id].attributes
# if this task has no dependencies, make it depend on this decision
# task so that it does not start immediately; and so that if this loop
# fails halfway through, none of the already-created tasks run.
if decision_task_id and not task_def.get('dependencies'):
task_def['dependencies'] = [decision_task_id]
task_def['taskGroupId'] = task_group_id
task_def['schedulerId'] = '-'
# Wait for dependencies before submitting this.
deps_fs = [fs[dep] for dep in task_def.get('dependencies', [])
if dep in fs]
for f in futures.as_completed(deps_fs):
f.result()
fs[task_id] = e.submit(_create_task, session, task_id,
taskid_to_label[task_id], task_def)
# Schedule tasks as many times as task_duplicates indicates
for i in range(1, attributes.get('task_duplicates', 1)):
# We use slugid() since we want a distinct task id
fs[task_id] = e.submit(_create_task, session, slugid(),
taskid_to_label[task_id], task_def)
# Wait for all futures to complete.
for f in futures.as_completed(fs.values()):
f.result()
def add_new_jobs_action(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
for elem in input['tasks']:
if elem in full_task_graph.tasks:
task = full_task_graph.tasks[elem]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task,
dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
# actually create the new task
create_task(slugid(), task_def, parameters['level'])
else:
raise Exception('{} was not found in the task-graph'.format(elem))
def test_retrigger_action(parameters, input, task_group_id, task_id, task):
new_task_definition = copy.copy(task)
# set new created, deadline, and expiry fields
new_task_definition['created'] = current_json_time()
new_task_definition['deadline'] = json_time_from_now('1d')
new_task_definition['expires'] = json_time_from_now('30d')
# reset artifact expiry
for artifact in new_task_definition['payload'].get('artifacts',
{}).values():
artifact['expires'] = new_task_definition['expires']
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false', '-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += [
'--log-tbpl=-', '--log-tbpl-level={}'.format(input['logLevel'])
]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input['repeat'])]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
logger.info("Creating new mochitest task with id %s", new_task_id)
session = requests.Session()
create_task(session, new_task_id, 'test-retrigger', new_task_definition)
def test_retrigger_action(parameters, input, task_group_id, task_id, task):
new_task_definition = copy.copy(task)
# set new created, deadline, and expiry fields
new_task_definition['created'] = current_json_time()
new_task_definition['deadline'] = json_time_from_now('1d')
new_task_definition['expires'] = json_time_from_now('30d')
# reset artifact expiry
for artifact in new_task_definition['payload'].get('artifacts', {}).values():
artifact['expires'] = new_task_definition['expires']
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false',
'-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += ['--log-tbpl=-',
'--log-tbpl-level={}'.format(input['logLevel'])]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input['repeat'])]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
logger.info("Creating new mochitest task with id %s", new_task_id)
session = requests.Session()
create_task(session, new_task_id, 'test-retrigger', new_task_definition)
def derive_index_task(task, taskgraph, label_to_taskid, parameters,
graph_config):
"""Create the shell of a task that depends on `task` and on the given docker
image."""
purpose = "index-task"
label = f"{purpose}-{task.label}"
provisioner_id, worker_type = get_worker_type(graph_config, "misc",
parameters["level"])
task_def = {
"provisionerId": provisioner_id,
"workerType": worker_type,
"dependencies": [task.task_id],
"created": {
"relative-datestamp": "0 seconds"
},
"deadline": task.task["deadline"],
# no point existing past the parent task's deadline
"expires": task.task["deadline"],
"metadata": {
"name":
label,
"description":
"{} for {}".format(purpose, task.task["metadata"]["description"]),
"owner":
task.task["metadata"]["owner"],
"source":
task.task["metadata"]["source"],
},
"scopes": [],
"payload": {
"image": {
"path": "public/image.tar.zst",
"namespace":
"taskgraph.cache.level-3.docker-images.v2.index-task.latest",
"type": "indexed-image",
},
"features": {
"taskclusterProxy": True,
},
"maxRunTime": 600,
},
}
# only include the docker-image dependency here if it is actually in the
# taskgraph (has not been optimized). It is included in
# task_def['dependencies'] unconditionally.
dependencies = {"parent": task.task_id}
task = Task(
kind="misc",
label=label,
attributes={},
task=task_def,
dependencies=dependencies,
)
task.task_id = slugid()
return task, taskgraph, label_to_taskid
def handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id):
task = taskcluster.get_task_definition(task_id)
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
parameters, graph_config)
pre_task = full_task_graph.tasks[task['metadata']['name']]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {name: label_to_taskid[label]
for name, label in pre_task.dependencies.iteritems()}
new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false',
'-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += ['--log-tbpl=-',
'--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input.get('repeat', 30))]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def derive_misc_task(task, purpose, image, taskgraph, label_to_taskid):
"""Create the shell of a task that depends on `task` and on the given docker
image."""
label = '{}-{}'.format(purpose, task.label)
# this is why all docker image tasks are included in the target task graph: we
# need to find them in label_to_taskid, if if nothing else required them
image_taskid = label_to_taskid['build-docker-image-' + image]
task_def = {
'provisionerId': 'gecko-t',
'workerType': 'misc',
'dependencies': [task.task_id, image_taskid],
'created': {
'relative-datestamp': '0 seconds'
},
'deadline': task.task['deadline'],
# no point existing past the parent task's deadline
'expires': task.task['deadline'],
'metadata': {
'name':
label,
'description':
'{} for {}'.format(purpose, task.task['metadata']['description']),
'owner':
task.task['metadata']['owner'],
'source':
task.task['metadata']['source'],
},
'scopes': [],
'payload': {
'image': {
'path': 'public/image.tar.zst',
'taskId': image_taskid,
'type': 'task-image',
},
'features': {
'taskclusterProxy': True,
},
'maxRunTime': 600,
},
}
# only include the docker-image dependency here if it is actually in the
# taskgraph (has not been optimized). It is included in
# task_def['dependencies'] unconditionally.
dependencies = {'parent': task.task_id}
if image_taskid in taskgraph.tasks:
dependencies['docker-image'] = image_taskid
task = Task(kind='misc',
label=label,
attributes={},
task=task_def,
dependencies=dependencies)
task.task_id = slugid()
return task
def get_subgraph(target_task_graph, removed_tasks, replaced_tasks,
label_to_taskid):
"""
Return the subgraph of target_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# check for any dependency edges from included to removed tasks
bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges
if l not in removed_tasks and r in removed_tasks]
if bad_edges:
probs = ', '.join(
'{} depends on {} as {} but it has been removed'.format(l, r, n)
for l, r, n in bad_edges)
raise Exception("Optimization error: " + probs)
# fill in label_to_taskid for anything not removed or replaced
assert replaced_tasks <= set(label_to_taskid)
for label in sorted(target_task_graph.graph.nodes - removed_tasks -
set(label_to_taskid)):
label_to_taskid[label] = slugid()
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
named_links_dict = target_task_graph.graph.named_links_dict()
omit = removed_tasks | replaced_tasks
for label, task in target_task_graph.tasks.iteritems():
if label in omit:
continue
task.task_id = label_to_taskid[label]
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).iteritems()
}
task.task = resolve_task_references(task.label, task.task,
named_task_dependencies)
deps = task.task.setdefault('dependencies', [])
deps.extend(sorted(named_task_dependencies.itervalues()))
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right),
name) for (left, right,
name) in target_task_graph.graph.edges)
# ..and drop edges that are no longer entirely in the task graph
# (note that this omits edges to replaced tasks, but they are still in task.dependnecies)
edges_by_taskid = set(
(left, right, name) for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid)
return TaskGraph(tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def mochitest_retrigger_action(parameters, input, task_group_id, task_id, task):
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(parameters)
pre_task = full_task_graph.tasks[task['metadata']['name']]
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {name: label_to_taskid[label]
for name, label in pre_task.dependencies.iteritems()}
new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies)
new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues())
# don't want to run mozharness tests, want a custom mach command instead
new_task_definition['payload']['command'] += ['--no-run-tests']
custom_mach_command = [task['tags']['test-type']]
# mochitests may specify a flavor
if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'):
custom_mach_command += [
'--keep-open=false',
'-f',
new_task_definition['payload']['env']['MOCHITEST_FLAVOR']
]
enable_e10s = json.loads(new_task_definition['payload']['env'].get(
'ENABLE_E10S', 'true'))
if not enable_e10s:
custom_mach_command += ['--disable-e10s']
custom_mach_command += ['--log-tbpl=-',
'--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))]
if input.get('runUntilFail'):
custom_mach_command += ['--run-until-failure']
if input.get('repeat'):
custom_mach_command += ['--repeat', str(input.get('repeat', 30))]
# add any custom gecko preferences
for (key, val) in input.get('preferences', {}).iteritems():
custom_mach_command += ['--setpref', '{}={}'.format(key, val)]
custom_mach_command += [input['path']]
new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join(
custom_mach_command)
# update environment
new_task_definition['payload']['env'].update(input.get('environment', {}))
# tweak the treeherder symbol
new_task_definition['extra']['treeherder']['symbol'] += '-custom'
logging.info("New task definition: %s", new_task_definition)
# actually create the new task
new_task_id = slugid()
create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def get_subgraph(target_task_graph, removed_tasks, replaced_tasks, label_to_taskid):
"""
Return the subgraph of target_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# check for any dependency edges from included to removed tasks
bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges
if l not in removed_tasks and r in removed_tasks]
if bad_edges:
probs = ', '.join('{} depends on {} as {} but it has been removed'.format(l, r, n)
for l, r, n in bad_edges)
raise Exception("Optimization error: " + probs)
# fill in label_to_taskid for anything not removed or replaced
assert replaced_tasks <= set(label_to_taskid)
for label in sorted(target_task_graph.graph.nodes - removed_tasks - set(label_to_taskid)):
label_to_taskid[label] = slugid()
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
named_links_dict = target_task_graph.graph.named_links_dict()
omit = removed_tasks | replaced_tasks
for label, task in target_task_graph.tasks.iteritems():
if label in omit:
continue
task.task_id = label_to_taskid[label]
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).iteritems()}
task.task = resolve_task_references(task.label, task.task, named_task_dependencies)
deps = task.task.setdefault('dependencies', [])
deps.extend(sorted(named_task_dependencies.itervalues()))
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = (
(label_to_taskid.get(left), label_to_taskid.get(right), name)
for (left, right, name) in target_task_graph.graph.edges
)
# ..and drop edges that are no longer entirely in the task graph
# (note that this omits edges to replaced tasks, but they are still in task.dependnecies)
edges_by_taskid = set(
(left, right, name)
for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid
)
return TaskGraph(
tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id):
"""For a single task, we try to just run exactly the same task once more.
It's quite possible that we don't have the scopes to do so (especially for
an action), but this is best-effort."""
# make all of the timestamps relative; they will then be turned back into
# absolute timestamps relative to the current time.
task = taskcluster.get_task_definition(task_id)
task = relativize_datestamps(task)
create_task_from_def(slugid(), task, parameters['level'])
def __call__(self, name):
'''
So this object can easily be passed to mustache we allow it to be called
directly...
'''
if name in self._names:
return self._names[name]
self._names[name] = slugid()
return self._names[name]
def __call__(self, name):
'''
So this object can easily be passed to mustache we allow it to be called
directly...
'''
if name in self._names:
return self._names[name];
self._names[name] = slugid()
return self._names[name]
def create_isolate_failure_tasks(task_definition, failures, level):
"""
Create tasks to re-run the original tasks plus tasks to test
each failing test directory and individual path.
"""
# redo the original task...
logger.info("create_isolate_failure_tasks: task_definition: {},"
"failures: {}".format(task_definition, failures))
new_task_id = slugid()
new_task_definition = copy.deepcopy(task_definition)
th_dict = new_task_definition['extra']['treeherder']
th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I'
th_dict['tier'] = 3
logger.info('Cloning original task')
create_task_from_def(new_task_id, new_task_definition, level)
for failure_group in failures:
failure_group_suffix = '-id' if failure_group == 'dirs' else '-it'
for failure_path in failures[failure_group]:
new_task_id = slugid()
new_task_definition = copy.deepcopy(task_definition)
th_dict = new_task_definition['extra']['treeherder']
th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I'
th_dict['symbol'] = th_dict['symbol'] + failure_group_suffix
th_dict['tier'] = 3
suite = new_task_definition['extra']['suite']
if '-chunked' in suite:
suite = suite[:suite.index('-chunked')]
if '-coverage' in suite:
suite = suite[:suite.index('-coverage')]
env_dict = new_task_definition['payload']['env']
if 'MOZHARNESS_TEST_PATHS' not in env_dict:
env_dict['MOZHARNESS_TEST_PATHS'] = {}
if 'windows' in th_dict['machine']['platform']:
failure_path = '\\'.join(failure_path.split('/'))
env_dict['MOZHARNESS_TEST_PATHS'] = json.dumps({suite: [failure_path]})
logger.info('Creating task for {}'.format(failure_path))
create_task_from_def(new_task_id, new_task_definition, level)
def build_callback_action_task(parameters):
if not available(parameters):
return None
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$',
parameters['head_repository'])
if not match:
raise Exception('Unrecognized head_repository')
repo_scope = 'assume:repo:{}/{}:*'.format(match.group(1),
match.group(2))
task_group_id = os.environ.get('TASK_ID', slugid())
template = os.path.join(GECKO, '.taskcluster.yml')
with open(template, 'r') as f:
taskcluster_yml = yaml.safe_load(f)
if taskcluster_yml['version'] != 1:
raise Exception(
'actions.json must be updated to work with .taskcluster.yml'
)
if not isinstance(taskcluster_yml['tasks'], list):
raise Exception(
'.taskcluster.yml "tasks" must be a list for action tasks'
)
return {
'$let': {
'tasks_for': 'action',
'repository': {
'url': parameters['head_repository'],
'project': parameters['project'],
'level': parameters['level'],
},
'push': {
'owner':
'*****@*****.**',
'pushlog_id': parameters['pushlog_id'],
'revision': parameters['head_rev'],
},
'action': {
'name': name,
'title': title,
'description': description,
'taskGroupId': task_group_id,
'repo_scope': repo_scope,
'cb_name': cb.__name__,
'symbol': symbol,
},
},
'in': taskcluster_yml['tasks'][0]
}
def create_task_from_def(task_def, level):
"""Create a new task from a definition rather than from a label
that is already in the full-task-graph. The task definition will
have {relative-datestamp': '..'} rendered just like in a decision task.
Use this for entirely new tasks or ones that change internals of the task.
It is useful if you want to "edit" the full_task_graph and then hand
it to this function. No dependencies will be scheduled. You must handle
this yourself. Seeing how create_tasks handles it might prove helpful."""
task_def["schedulerId"] = "gecko-level-{}".format(level)
label = task_def["metadata"]["name"]
task_id = slugid().decode("ascii")
session = get_session()
create.create_task(session, task_id, label, task_def)
def derive_misc_task(task, purpose, image, taskgraph, label_to_taskid):
"""Create the shell of a task that depends on `task` and on the given docker
image."""
label = '{}-{}'.format(purpose, task.label)
# this is why all docker image tasks are included in the target task graph: we
# need to find them in label_to_taskid, if if nothing else required them
image_taskid = label_to_taskid['build-docker-image-' + image]
task_def = {
'provisionerId': 'aws-provisioner-v1',
'workerType': 'gecko-misc',
'dependencies': [task.task_id, image_taskid],
'created': {'relative-datestamp': '0 seconds'},
'deadline': task.task['deadline'],
# no point existing past the parent task's deadline
'expires': task.task['deadline'],
'metadata': {
'name': label,
'description': '{} for {}'.format(purpose, task.task['metadata']['description']),
'owner': task.task['metadata']['owner'],
'source': task.task['metadata']['source'],
},
'scopes': [],
'payload': {
'image': {
'path': 'public/image.tar.zst',
'taskId': image_taskid,
'type': 'task-image',
},
'features': {
'taskclusterProxy': True,
},
'maxRunTime': 600,
}
}
# only include the docker-image dependency here if it is actually in the
# taskgraph (has not been optimized). It is included in
# task_def['dependencies'] unconditionally.
dependencies = {'parent': task.task_id}
if image_taskid in taskgraph.tasks:
dependencies['docker-image'] = image_taskid
task = Task(kind='misc', label=label, attributes={}, task=task_def,
dependencies=dependencies)
task.task_id = slugid()
return task
def backfill_action(parameters, input, task_group_id, task_id, task):
label = task['metadata']['name']
pushes = []
depth = input.get('depth', 5)
end_id = int(parameters['pushlog_id']) - 1
while True:
start_id = max(end_id - depth, 0)
pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'],
start_id, end_id)
r = requests.get(pushlog_url)
r.raise_for_status()
pushes = pushes + r.json()['pushes'].keys()
if len(pushes) >= depth:
break
end_id = start_id - 1
start_id -= depth
if start_id < 0:
break
pushes = sorted(pushes)[-depth:]
for push in pushes:
full_task_graph = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/full-task-graph.json')
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact_from_index(
INDEX_TMPL.format(parameters['project'], push),
'public/label-to-taskid.json')
if label in full_task_graph.tasks.keys():
task = full_task_graph.tasks[label]
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task,
dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
create_task(slugid(), task_def, parameters['level'])
else:
logging.info('Could not find {} on {}. Skipping.'.format(
label, push))
def build_callback_action_task(parameters):
if not available(parameters):
return None
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository'])
if not match:
raise Exception('Unrecognized head_repository')
repo_scope = 'assume:repo:{}/{}:*'.format(
match.group(1), match.group(2))
task_group_id = os.environ.get('TASK_ID', slugid())
template = os.path.join(GECKO, '.taskcluster.yml')
with open(template, 'r') as f:
taskcluster_yml = yaml.safe_load(f)
if taskcluster_yml['version'] != 1:
raise Exception('actions.json must be updated to work with .taskcluster.yml')
if not isinstance(taskcluster_yml['tasks'], list):
raise Exception('.taskcluster.yml "tasks" must be a list for action tasks')
return {
'$let': {
'tasks_for': 'action',
'repository': {
'url': parameters['head_repository'],
'project': parameters['project'],
'level': parameters['level'],
},
'push': {
'owner': '*****@*****.**',
'pushlog_id': parameters['pushlog_id'],
'revision': parameters['head_rev'],
},
'action': {
'name': name,
'title': title,
'description': description,
'taskGroupId': task_group_id,
'repo_scope': repo_scope,
'cb_name': cb.__name__,
'symbol': symbol,
},
},
'in': taskcluster_yml['tasks'][0]
}
def create_tasks(taskgraph, label_to_taskid):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
session = requests.Session()
decision_task_id = os.environ.get('TASK_ID')
with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e:
fs = {}
# We can't submit a task until its dependencies have been submitted.
# So our strategy is to walk the graph and submit tasks once all
# their dependencies have been submitted.
#
# Using visit_postorder() here isn't the most efficient: we'll
# block waiting for dependencies of task N to submit even though
# dependencies for task N+1 may be finished. If we need to optimize
# this further, we can build a graph of task dependencies and walk
# that.
for task_id in taskgraph.graph.visit_postorder():
task_def = taskgraph.tasks[task_id].task
# if this task has no dependencies, make it depend on this decision
# task so that it does not start immediately; and so that if this loop
# fails halfway through, none of the already-created tasks run.
if decision_task_id and not task_def.get('dependencies'):
task_def['dependencies'] = [decision_task_id]
task_def['taskGroupId'] = task_group_id
# Wait for dependencies before submitting this.
deps_fs = [
fs[dep] for dep in task_def['dependencies'] if dep in fs
]
for f in futures.as_completed(deps_fs):
f.result()
fs[task_id] = e.submit(_create_task, session, task_id,
taskid_to_label[task_id], task_def)
# Wait for all futures to complete.
for f in futures.as_completed(fs.values()):
f.result()
def create_tasks(taskgraph, label_to_taskid):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
session = requests.Session()
decision_task_id = os.environ.get('TASK_ID')
with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e:
fs = {}
# We can't submit a task until its dependencies have been submitted.
# So our strategy is to walk the graph and submit tasks once all
# their dependencies have been submitted.
#
# Using visit_postorder() here isn't the most efficient: we'll
# block waiting for dependencies of task N to submit even though
# dependencies for task N+1 may be finished. If we need to optimize
# this further, we can build a graph of task dependencies and walk
# that.
for task_id in taskgraph.graph.visit_postorder():
task_def = taskgraph.tasks[task_id].task
# if this task has no dependencies, make it depend on this decision
# task so that it does not start immediately; and so that if this loop
# fails halfway through, none of the already-created tasks run.
if decision_task_id and not task_def.get('dependencies'):
task_def['dependencies'] = [decision_task_id]
task_def['taskGroupId'] = task_group_id
# Wait for dependencies before submitting this.
deps_fs = [fs[dep] for dep in task_def.get('dependencies', [])
if dep in fs]
for f in futures.as_completed(deps_fs):
f.result()
fs[task_id] = e.submit(_create_task, session, task_id,
taskid_to_label[task_id], task_def)
# Wait for all futures to complete.
for f in futures.as_completed(fs.values()):
f.result()
def create_tasks(taskgraph):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
label_to_taskid = collections.defaultdict(slugid)
session = requests.Session()
for label in taskgraph.graph.visit_postorder():
task = taskgraph.tasks[label]
deps_by_name = {
n: label_to_taskid[r]
for (l, r, n) in taskgraph.graph.edges
if l == label}
task_def = task.kind.get_task_definition(task, deps_by_name)
task_def['taskGroupId'] = task_group_id
task_def['dependencies'] = deps_by_name.values()
task_def['requires'] = 'all-completed'
_create_task(session, label_to_taskid[label], label, task_def)
def create_tasks(taskgraph):
# TODO: use the taskGroupId of the decision task
task_group_id = slugid()
label_to_taskid = collections.defaultdict(slugid)
session = requests.Session()
for label in taskgraph.graph.visit_postorder():
task = taskgraph.tasks[label]
deps_by_name = {
n: label_to_taskid[r]
for (l, r, n) in taskgraph.graph.edges if l == label
}
task_def = task.kind.get_task_definition(task, deps_by_name)
task_def['taskGroupId'] = task_group_id
task_def['dependencies'] = deps_by_name.values()
task_def['requires'] = 'all-completed'
_create_task(session, label_to_taskid[label], label, task_def)
def task_id_for_image(seen_images, project, name):
if name in seen_images:
return seen_images[name]['taskId']
context_path = os.path.join('testing', 'docker', name)
context_hash = generate_context_hash(context_path)
task_id = get_task_id_for_namespace(project, name, context_hash)
if task_id:
seen_images[name] = {'taskId': task_id}
return task_id
task_id = slugid()
seen_images[name] = {
'taskId': task_id,
'path': context_path,
'hash': context_hash
}
return task_id
def task_id_for_image(seen_images, project, name):
if name in seen_images:
return seen_images[name]['taskId']
context_path = os.path.join('testing', 'docker', name)
context_hash = generate_context_hash(context_path)
task_id = get_task_id_for_namespace(project, name, context_hash)
if task_id:
seen_images[name] = {'taskId': task_id}
return task_id
task_id = slugid()
seen_images[name] = {
'taskId': task_id,
'path': context_path,
'hash': context_hash
}
return task_id
def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid):
"""
Return the subgraph of annotated_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
for label in annotated_task_graph.graph.visit_postorder():
task = annotated_task_graph.tasks[label]
if task.optimized:
continue
task.task_id = label_to_taskid[label] = slugid()
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).iteritems()}
task.task = resolve_task_references(task.label, task.task, named_task_dependencies)
task.task.setdefault('dependencies', []).extend(named_task_dependencies.itervalues())
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = (
(label_to_taskid.get(left), label_to_taskid.get(right), name)
for (left, right, name) in annotated_task_graph.graph.edges
)
# ..and drop edges that are no longer in the task graph
edges_by_taskid = set(
(left, right, name)
for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid
)
return TaskGraph(
tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def run_missing_tests(parameters, input, task_group_id, task_id, task):
decision_task_id = find_decision_task(parameters)
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
target_tasks = get_artifact(decision_task_id, "public/target-tasks.json")
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
# The idea here is to schedule all tasks of the `test` kind that were
# targetted but did not appear in the final task-graph -- those were the
# optimized tasks.
to_run = []
already_run = 0
for label in target_tasks:
task = full_task_graph.tasks[label]
if task.kind != 'test':
continue # not a test
if label in label_to_taskid:
already_run += 1
continue
to_run.append(task)
for task in to_run:
# fix up the task's dependencies, similar to how optimization would
# have done in the decision
dependencies = {
name: label_to_taskid[label]
for name, label in task.dependencies.iteritems()
}
task_def = resolve_task_references(task.label, task.task, dependencies)
task_def.setdefault('dependencies',
[]).extend(dependencies.itervalues())
create_task(slugid(), task_def, parameters['level'])
logger.info(
'Out of {} test tasks, {} already existed and the action created {}'.
format(already_run + len(to_run), already_run, len(to_run)))
def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid):
"""
Return the subgraph of annotated_task_graph consisting only of
non-optimized tasks and edges between them.
To avoid losing track of taskIds for tasks optimized away, this method
simultaneously substitutes real taskIds for task labels in the graph, and
populates each task definition's `dependencies` key with the appropriate
taskIds. Task references are resolved in the process.
"""
# resolve labels to taskIds and populate task['dependencies']
tasks_by_taskid = {}
for label in annotated_task_graph.graph.visit_postorder():
task = annotated_task_graph.tasks[label]
if task.optimized:
continue
task.task_id = label_to_taskid[label] = slugid()
named_task_dependencies = {
name: label_to_taskid[label]
for name, label in named_links_dict.get(label, {}).iteritems()
}
task.task = resolve_task_references(task.label, task.task,
named_task_dependencies)
task.task.setdefault('dependencies',
[]).extend(named_task_dependencies.itervalues())
tasks_by_taskid[task.task_id] = task
# resolve edges to taskIds
edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right),
name) for (left, right,
name) in annotated_task_graph.graph.edges)
# ..and drop edges that are no longer in the task graph
edges_by_taskid = set(
(left, right, name) for (left, right, name) in edges_by_taskid
if left in tasks_by_taskid and right in tasks_by_taskid)
return TaskGraph(tasks_by_taskid,
Graph(set(tasks_by_taskid), edges_by_taskid))
def schedule_tasks(f=None):
to_remove = set()
for task_id in tasklist:
task_def = taskgraph.tasks[task_id].task
# If we haven't finished submitting all our dependencies yet,
# come back to this later.
# Some dependencies aren't in our graph, so make sure to filter
# those out
deps = set(task_def.get('dependencies', [])) & alltasks
if any((d not in fs or not fs[d].done()) for d in deps):
continue
fs[task_id] = e.submit(create_task, session, task_id,
taskid_to_label[task_id], task_def)
to_remove.add(task_id)
# Schedule tasks as many times as task_duplicates indicates
attributes = taskgraph.tasks[task_id].attributes
for i in range(1, attributes.get('task_duplicates', 1)):
# We use slugid() since we want a distinct task id
fs[task_id] = e.submit(create_task, session, slugid(),
taskid_to_label[task_id], task_def)
tasklist.difference_update(to_remove)
def schedule_tasks():
# bail out early if any futures have failed
if any(f.done() and f.exception() for f in fs.values()):
return
to_remove = set()
new = set()
def submit(task_id, label, task_def):
fut = e.submit(create_task, session, task_id, label, task_def)
new.add(fut)
fs[task_id] = fut
for task_id in tasklist:
task_def = taskgraph.tasks[task_id].task
# If we haven't finished submitting all our dependencies yet,
# come back to this later.
# Some dependencies aren't in our graph, so make sure to filter
# those out
deps = set(task_def.get('dependencies', [])) & alltasks
if any((d not in fs or not fs[d].done()) for d in deps):
continue
submit(task_id, taskid_to_label[task_id], task_def)
to_remove.add(task_id)
# Schedule tasks as many times as task_duplicates indicates
attributes = taskgraph.tasks[task_id].attributes
for i in range(1, attributes.get('task_duplicates', 1)):
# We use slugid() since we want a distinct task id
submit(slugid().decode("ascii"), taskid_to_label[task_id],
task_def)
tasklist.difference_update(to_remove)
# as each of those futures complete, try to schedule more tasks
for f in futures.as_completed(new):
schedule_tasks()
def build_callback_action_task(parameters):
if not available(parameters):
return None
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$',
parameters['head_repository'])
if not match:
raise Exception('Unrecognized head_repository')
repo_scope = 'assume:repo:{}/{}:*'.format(match.group(1),
match.group(2))
task_group_id = os.environ.get('TASK_ID', slugid())
return {
'created': {
'$fromNow': ''
},
'deadline': {
'$fromNow': '12 hours'
},
'expires': {
'$fromNow': '14 days'
},
'metadata': {
'owner':
'*****@*****.**',
'source':
'{}raw-file/{}/{}'.format(
parameters['head_repository'],
parameters['head_rev'],
source_path,
),
'name':
'Action: {}'.format(title),
'description':
'Task executing callback for action.\n\n---\n' +
description,
},
'workerType':
'gecko-{}-decision'.format(parameters['level']),
'provisionerId':
'aws-provisioner-v1',
'taskGroupId':
task_group_id,
'schedulerId':
'gecko-level-{}'.format(parameters['level']),
'scopes': [
repo_scope,
],
'tags': {
'createdForUser': parameters['owner'],
'kind': 'action-callback',
},
'routes': [
'tc-treeherder.v2.{}.{}.{}'.format(
parameters['project'], parameters['head_rev'],
parameters['pushlog_id']),
'tc-treeherder-stage.v2.{}.{}.{}'.format(
parameters['project'], parameters['head_rev'],
parameters['pushlog_id']),
],
'payload': {
'env': {
'GECKO_BASE_REPOSITORY':
'https://hg.mozilla.org/mozilla-unified',
'GECKO_HEAD_REPOSITORY': parameters['head_repository'],
'GECKO_HEAD_REF': parameters['head_ref'],
'GECKO_HEAD_REV': parameters['head_rev'],
'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
'ACTION_TASK_GROUP_ID': task_group_id,
'ACTION_TASK_ID': {
'$json': {
'$eval': 'taskId'
}
},
'ACTION_TASK': {
'$json': {
'$eval': 'task'
}
},
'ACTION_INPUT': {
'$json': {
'$eval': 'input'
}
},
'ACTION_CALLBACK': cb.__name__,
'ACTION_PARAMETERS': {
'$json': {
'$eval': 'parameters'
}
},
},
'cache': {
'level-{}-checkouts'.format(parameters['level']):
'/home/worker/checkouts',
},
'features': {
'taskclusterProxy': True,
'chainOfTrust': True,
},
'image':
docker_image('decision'),
'maxRunTime':
1800,
'command': [
'/home/worker/bin/run-task',
'--vcs-checkout=/home/worker/checkouts/gecko',
'--',
'bash',
'-cx',
"""\
cd /home/worker/checkouts/gecko &&
ln -s /home/worker/artifacts artifacts &&
./mach --log-no-times taskgraph action-callback""",
],
},
'extra': {
'treeherder': {
'groupName': 'action-callback',
'groupSymbol': 'AC',
'symbol': symbol,
},
},
}
def action_builder(parameters, graph_config):
if not available(parameters):
return None
actionPerm = 'generic' if generic else cb_name
# gather up the common decision-task-supplied data for this action
repo_param = '{}head_repository'.format(
graph_config['project-repo-param-prefix'])
repository = {
'url': parameters[repo_param],
'project': parameters['project'],
'level': parameters['level'],
}
revision = parameters['{}head_rev'.format(
graph_config['project-repo-param-prefix'])]
push = {
'owner': '*****@*****.**',
'pushlog_id': parameters['pushlog_id'],
'revision': revision,
}
task_group_id = os.environ.get('TASK_ID', slugid())
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$',
parameters[repo_param])
if not match:
raise Exception('Unrecognized {}'.format(repo_param))
action = {
'name': name,
'title': title,
'description': description,
# target taskGroupId (the task group this decision task is creating)
'taskGroupId': task_group_id,
'cb_name': cb_name,
'symbol': symbol,
}
rv = {
'name': name,
'title': title,
'description': description,
'context': context(parameters),
}
if schema:
rv['schema'] = schema(
graph_config=graph_config) if callable(schema) else schema
# for kind=task, we embed the task from .taskcluster.yml in the action, with
# suitable context
if kind == 'task':
# tasks get all of the scopes the original push did, yuck; this is not
# done with kind = hook.
repo_scope = 'assume:repo:{}/{}:branch:default'.format(
match.group(1), match.group(2))
action['repo_scope'] = repo_scope
taskcluster_yml = read_taskcluster_yml(
graph_config.taskcluster_yml)
if taskcluster_yml['version'] != 1:
raise Exception(
'actions.json must be updated to work with .taskcluster.yml'
)
tasks = taskcluster_yml['tasks']
if not isinstance(tasks, list):
raise Exception(
'.taskcluster.yml "tasks" must be a list for action tasks'
)
rv.update({
'kind': 'task',
'task': {
'$let': {
'tasks_for': 'action',
'repository': repository,
'push': push,
'action': action,
},
'in': tasks[0],
},
})
# for kind=hook
elif kind == 'hook':
trustDomain = graph_config['trust-domain']
level = parameters['level']
tcyml_hash = hash_taskcluster_yml(graph_config.taskcluster_yml)
# the tcyml_hash is prefixed with `/` in the hookId, so users will be granted
# hooks:trigger-hook:project-gecko/in-tree-action-3-myaction/*; if another
# action was named `myaction/release`, then the `*` in the scope would also
# match that action. To prevent such an accident, we prohibit `/` in hook
# names.
if '/' in actionPerm:
raise Exception(
'`/` is not allowed in action names; use `-`')
rv.update({
'kind':
'hook',
'hookGroupId':
'project-{}'.format(trustDomain),
'hookId':
'in-tree-action-{}-{}/{}'.format(level, actionPerm,
tcyml_hash),
'hookPayload': {
# provide the decision-task parameters as context for triggerHook
"decision": {
'action': action,
'repository': repository,
'push': push,
# parameters is long, so fetch it from the actions.json variables
'parameters': {
'$eval': 'parameters'
},
},
# and pass everything else through from our own context
"user": {
'input': {
'$eval': 'input'
},
'taskId': {
'$eval': 'taskId'
}, # target taskId (or null)
'taskGroupId': {
'$eval': 'taskGroupId'
}, # target task group
}
},
'extra': {
'actionPerm': actionPerm,
},
})
return rv
def build_callback_action_task(parameters, graph_config):
if not available(parameters):
return None
repo_param = '{}head_repository'.format(
graph_config['project-repo-param-prefix'])
revision = parameters['{}head_rev'.format(
graph_config['project-repo-param-prefix'])]
match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$',
parameters[repo_param])
if not match:
raise Exception('Unrecognized {}'.format(repo_param))
repo_scope = 'assume:repo:{}/{}:branch:default'.format(
match.group(1), match.group(2))
task_group_id = os.environ.get('TASK_ID', slugid())
# FIXME: https://bugzilla.mozilla.org/show_bug.cgi?id=1454034
# trust-domain works, but isn't semantically correct here.
if graph_config['trust-domain'] == 'comm':
template = os.path.join(GECKO, 'comm', '.taskcluster.yml')
else:
template = os.path.join(GECKO, '.taskcluster.yml')
with open(template, 'r') as f:
taskcluster_yml = yaml.safe_load(f)
if taskcluster_yml['version'] != 1:
raise Exception(
'actions.json must be updated to work with .taskcluster.yml'
)
if not isinstance(taskcluster_yml['tasks'], list):
raise Exception(
'.taskcluster.yml "tasks" must be a list for action tasks'
)
return {
'$let': {
'tasks_for': 'action',
'repository': {
'url': parameters[repo_param],
'project': parameters['project'],
'level': parameters['level'],
},
'push': {
'owner':
'*****@*****.**',
'pushlog_id': parameters['pushlog_id'],
'revision': revision,
},
'action': {
'name': name,
'title': title,
'description': description,
'taskGroupId': task_group_id,
'repo_scope': repo_scope,
'cb_name': cb.__name__,
'symbol': symbol,
},
},
'in': taskcluster_yml['tasks'][0]
}
def create_tasks(taskgraph, label_to_taskid, params, decision_task_id=None):
taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
session = requests.Session()
# Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
# that limit. Connections are established as needed, so using a large value
# should not negatively impact performance.
http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY,
pool_maxsize=CONCURRENCY)
session.mount('https://', http_adapter)
session.mount('http://', http_adapter)
decision_task_id = decision_task_id or os.environ.get('TASK_ID')
# when running as an actual decision task, we use the decision task's
# taskId as the taskGroupId. The process that created the decision task
# helpfully placed it in this same taskGroup. If there is no $TASK_ID,
# fall back to a slugid
task_group_id = decision_task_id or slugid()
scheduler_id = 'gecko-level-{}'.format(params['level'])
# Add the taskGroupId, schedulerId and optionally the decision task
# dependency
for task_id in taskgraph.graph.nodes:
task_def = taskgraph.tasks[task_id].task
# if this task has no dependencies *within* this taskgraph, make it
# depend on this decision task. If it has another dependency within
# the taskgraph, then it already implicitly depends on the decision
# task. The result is that tasks do not start immediately. if this
# loop fails halfway through, none of the already-created tasks run.
if decision_task_id:
if not any(t in taskgraph.tasks
for t in task_def.get('dependencies', [])):
task_def.setdefault('dependencies',
[]).append(decision_task_id)
task_def['taskGroupId'] = task_group_id
task_def['schedulerId'] = scheduler_id
with futures.ThreadPoolExecutor(CONCURRENCY) as e:
fs = {}
# We can't submit a task until its dependencies have been submitted.
# So our strategy is to walk the graph and submit tasks once all
# their dependencies have been submitted.
tasklist = set(taskgraph.graph.visit_postorder())
alltasks = tasklist.copy()
def schedule_tasks(f=None):
to_remove = set()
for task_id in tasklist:
task_def = taskgraph.tasks[task_id].task
# If we haven't finished submitting all our dependencies yet,
# come back to this later.
# Some dependencies aren't in our graph, so make sure to filter
# those out
deps = set(task_def.get('dependencies', [])) & alltasks
if any((d not in fs or not fs[d].done()) for d in deps):
continue
fs[task_id] = e.submit(create_task, session, task_id,
taskid_to_label[task_id], task_def)
to_remove.add(task_id)
# Schedule tasks as many times as task_duplicates indicates
attributes = taskgraph.tasks[task_id].attributes
for i in range(1, attributes.get('task_duplicates', 1)):
# We use slugid() since we want a distinct task id
fs[task_id] = e.submit(create_task, session, slugid(),
taskid_to_label[task_id], task_def)
tasklist.difference_update(to_remove)
schedule_tasks()
while tasklist:
for f in futures.as_completed(fs.values()):
f.result()
schedule_tasks()
def mklabel():
return TASKID_PLACEHOLDER.format(slugid())
def mklabel():
return slugid()
def add_code_review_task(taskgraph, label_to_taskid, parameters, graph_config):
logger.debug("Morphing: adding index tasks")
review_config = parameters.get("code-review")
if not review_config:
return taskgraph, label_to_taskid
code_review_tasks = {}
for label, task in taskgraph.tasks.items():
if task.attributes.get("code-review"):
code_review_tasks[task.label] = task.task_id
if code_review_tasks:
code_review_task_def = {
"provisionerId": "built-in",
"workerType": "succeed",
"dependencies": sorted(code_review_tasks.values()),
# This option permits to run the task
# regardless of the dependencies tasks exit status
# as we are interested in the task failures
"requires": "all-resolved",
"created": {
"relative-datestamp": "0 seconds"
},
"deadline": {
"relative-datestamp": "1 day"
},
# no point existing past the parent task's deadline
"expires": {
"relative-datestamp": "1 day"
},
"metadata": {
"name": "code-review",
"description":
"List all issues found in static analysis and linting tasks",
"owner": parameters["owner"],
"source": _get_morph_url(),
},
"scopes": [],
"payload": {},
"routes": ["project.relman.codereview.v1.try_ending"],
"extra": {
"code-review": {
"phabricator-build-target":
review_config["phabricator-build-target"],
"repository":
parameters["head_repository"],
"revision":
parameters["head_rev"],
}
},
}
task = Task(
kind="misc",
label="code-review",
attributes={},
task=code_review_task_def,
dependencies=code_review_tasks,
)
task.task_id = slugid()
taskgraph, label_to_taskid = amend_taskgraph(taskgraph,
label_to_taskid, [task])
logger.info("Added code review task.")
return taskgraph, label_to_taskid
def create_graph(self, **params):
from taskcluster_graph.commit_parser import parse_commit
from slugid import nice as slugid
from taskcluster_graph.from_now import json_time_from_now, current_json_time
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
project = params["project"]
message = params.get("message", "") if project == "try" else DEFAULT_TRY
# Message would only be blank when not created from decision task
if project == "try" and not message:
sys.stderr.write(
"Must supply commit message when creating try graph. " "Example: --message='try: -b do -p all -u all'"
)
sys.exit(1)
templates = Templates(ROOT)
job_path = os.path.join(ROOT, "tasks", "branches", project, "job_flags.yml")
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph = parse_commit(message, jobs)
cmdline_interactive = params.get("interactive", False)
# Template parameters used when expanding the graph
parameters = dict(
gaia_info().items()
+ {
"index": "index",
"project": project,
"pushlog_id": params.get("pushlog_id", 0),
"docker_image": docker_image,
"base_repository": params["base_repository"] or params["head_repository"],
"head_repository": params["head_repository"],
"head_ref": params["head_ref"] or params["head_rev"],
"head_rev": params["head_rev"],
"owner": params["owner"],
"from_now": json_time_from_now,
"now": current_json_time(),
"revision_hash": params["revision_hash"],
}.items()
)
treeherder_route = "{}.{}".format(params["project"], params.get("revision_hash", ""))
routes_file = os.path.join(ROOT, "routes.json")
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents["routes"]
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {"tasks": [], "scopes": []}
if params["revision_hash"]:
for env in TREEHERDER_ROUTES:
graph["scopes"].append("queue:route:{}.{}".format(TREEHERDER_ROUTES[env], treeherder_route))
graph["metadata"] = {
"source": "http://todo.com/what/goes/here",
"owner": params["owner"],
# TODO: Add full mach commands to this example?
"description": "Task graph generated via ./mach taskcluster-graph",
"name": "task graph local",
}
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = dict(parameters)
build_parameters["build_slugid"] = slugid()
build_task = templates.load(build["task"], build_parameters)
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
if params["revision_hash"]:
decorate_task_treeherder_routes(build_task["task"], treeherder_route)
decorate_task_json_routes(build, build_task["task"], json_routes, build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph["tasks"].append(build_task)
test_packages_url, tests_url, mozharness_url = None, None, None
if "test_packages" in build_task["task"]["extra"]["locations"]:
test_packages_url = ARTIFACT_URL.format(
build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["test_packages"]
)
if "tests" in build_task["task"]["extra"]["locations"]:
tests_url = ARTIFACT_URL.format(
build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["tests"]
)
if "mozharness" in build_task["task"]["extra"]["locations"]:
mozharness_url = ARTIFACT_URL.format(
build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["mozharness"]
)
build_url = ARTIFACT_URL.format(
build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["build"]
)
build_parameters["build_url"] = build_url
# img_url is only necessary for device builds
img_url = ARTIFACT_URL.format(
build_parameters["build_slugid"], build_task["task"]["extra"]["locations"].get("img", "")
)
build_parameters["img_url"] = img_url
define_task = DEFINE_TASK.format(build_task["task"]["workerType"])
for route in build_task["task"].get("routes", []):
if route.startswith("index.gecko.v2") and route in all_routes:
raise Exception(
"Error: route '%s' is in use by multiple tasks: '%s' and '%s'"
% (route, build_task["task"]["metadata"]["name"], all_routes[route])
)
all_routes[route] = build_task["task"]["metadata"]["name"]
graph["scopes"].append(define_task)
graph["scopes"].extend(build_task["task"].get("scopes", []))
route_scopes = map(lambda route: "queue:route:" + route, build_task["task"].get("routes", []))
graph["scopes"].extend(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task["task"]["extra"]["treeherder"]
if "machine" not in build_treeherder_config:
message = "({}), extra.treeherder.machine required for all builds"
raise ValueError(message.format(build["task"]))
if "build" not in build_treeherder_config:
build_treeherder_config["build"] = build_treeherder_config["machine"]
if "collection" not in build_treeherder_config:
build_treeherder_config["collection"] = {"opt": True}
if len(build_treeherder_config["collection"].keys()) != 1:
message = "({}), extra.treeherder.collection must contain one type"
raise ValueError(message.fomrat(build["task"]))
for post_build in build["post-build"]:
# copy over the old parameters to update the template
post_parameters = copy.copy(build_parameters)
post_task = configure_dependent_task(
post_build["task"], post_parameters, slugid(), templates, build_treeherder_config
)
set_interactive_task(post_task, interactive)
graph["tasks"].append(post_task)
for test in build["dependents"]:
test = test["allowed_build_tasks"][build["task"]]
test_parameters = copy.copy(build_parameters)
if tests_url:
test_parameters["tests_url"] = tests_url
if test_packages_url:
test_parameters["test_packages_url"] = test_packages_url
if mozharness_url:
test_parameters["mozharness_url"] = mozharness_url
test_definition = templates.load(test["task"], {})["task"]
chunk_config = test_definition["extra"]["chunks"]
# Allow branch configs to override task level chunking...
if "chunks" in test:
chunk_config["total"] = test["chunks"]
test_parameters["total_chunks"] = chunk_config["total"]
for chunk in range(1, chunk_config["total"] + 1):
if "only_chunks" in test and chunk not in test["only_chunks"]:
continue
test_parameters["chunk"] = chunk
test_task = configure_dependent_task(
test["task"], test_parameters, slugid(), templates, build_treeherder_config
)
set_interactive_task(test_task, interactive)
if params["revision_hash"]:
decorate_task_treeherder_routes(test_task["task"], treeherder_route)
graph["tasks"].append(test_task)
define_task = DEFINE_TASK.format(test_task["task"]["workerType"])
graph["scopes"].append(define_task)
graph["scopes"].extend(test_task["task"].get("scopes", []))
graph["scopes"] = list(set(graph["scopes"]))
if params["print_names_only"]:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print("{}- {}".format(" " * indent, task["task"]["metadata"]["name"]))
for child in tIDs[task["taskId"]]:
print_task(child, indent=indent + 2)
# build a dependency map
for task in graph["tasks"]:
if "requires" in task:
for tID in task["requires"]:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph["tasks"]:
if "requires" not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params["ci"] is True:
graph.pop("scopes", None)
graph.pop("metadata", None)
print(json.dumps(graph, indent=4))
def mklabel():
return TASKID_PLACEHOLDER.format(slugid())
def create_graph(self, **params):
from functools import partial
from slugid import nice as slugid
import taskcluster_graph.transform.routes as routes_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (docker_image,
normalize_image_details,
task_id_for_image)
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
if params['dry_run']:
from taskcluster_graph.dry_run import (
json_time_from_now,
current_json_time,
slugid,
)
project = params['project']
message = params.get('message',
'') if project == 'try' else DEFAULT_TRY
# Message would only be blank when not created from decision task
if project == 'try' and not message:
sys.stderr.write(
"Must supply commit message when creating try graph. " \
"Example: --message='try: -b do -p all -u all'"
)
sys.exit(1)
templates = Templates(ROOT)
job_path = os.path.join(ROOT, 'tasks', 'branches', project,
'job_flags.yml')
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph = parse_commit(message, jobs)
cmdline_interactive = params.get('interactive', False)
# Default to current time if querying the head rev fails
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
pushinfo = query_pushinfo(params['head_repository'],
params['head_rev'])
if pushinfo:
pushdate = time.strftime('%Y%m%d%H%M%S',
time.gmtime(pushinfo.pushdate))
# Template parameters used when expanding the graph
seen_images = {}
parameters = dict(gaia_info().items() + {
'index': 'index',
'project': project,
'pushlog_id': params.get('pushlog_id', 0),
'docker_image': docker_image,
'task_id_for_image': partial(task_id_for_image, seen_images, project),
'base_repository': params['base_repository'] or \
params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'from_now': json_time_from_now,
'now': current_json_time(),
'revision_hash': params['revision_hash']
}.items())
treeherder_route = '{}.{}'.format(params['project'],
params.get('revision_hash', ''))
routes_file = os.path.join(ROOT, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {'tasks': [], 'scopes': []}
if params['revision_hash']:
for env in routes_transform.TREEHERDER_ROUTES:
route = 'queue:route:{}.{}'.format(
routes_transform.TREEHERDER_ROUTES[env], treeherder_route)
graph['scopes'].append(route)
graph['metadata'] = {
'source': 'http://todo.com/what/goes/here',
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = merge_dicts(parameters,
build['additional-parameters'])
build_parameters['build_slugid'] = slugid()
build_task = templates.load(build['task'], build_parameters)
# Copy build_* attributes to expose them to post-build tasks
# as well as json routes and tests
task_extra = build_task['task']['extra']
build_parameters['build_name'] = task_extra['build_name']
build_parameters['build_type'] = task_extra['build_type']
build_parameters['build_product'] = task_extra['build_product']
normalize_image_details(graph, build_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
if params['revision_hash']:
routes_transform.decorate_task_treeherder_routes(
build_task['task'], treeherder_route)
routes_transform.decorate_task_json_routes(
build_task['task'], json_routes, build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph['tasks'].append(build_task)
test_packages_url, tests_url, mozharness_url = None, None, None
if 'test_packages' in build_task['task']['extra']['locations']:
test_packages_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['test_packages'])
if 'tests' in build_task['task']['extra']['locations']:
tests_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['tests'])
if 'mozharness' in build_task['task']['extra']['locations']:
mozharness_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['mozharness'])
build_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['build'])
build_parameters['build_url'] = build_url
# img_url is only necessary for device builds
img_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations'].get('img', ''))
build_parameters['img_url'] = img_url
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
for route in build_task['task'].get('routes', []):
if route.startswith('index.gecko.v2') and route in all_routes:
raise Exception(
"Error: route '%s' is in use by multiple tasks: '%s' and '%s'"
% (
route,
build_task['task']['metadata']['name'],
all_routes[route],
))
all_routes[route] = build_task['task']['metadata']['name']
graph['scopes'].append(define_task)
graph['scopes'].extend(build_task['task'].get('scopes', []))
route_scopes = map(lambda route: 'queue:route:' + route,
build_task['task'].get('routes', []))
graph['scopes'].extend(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task['task']['extra']['treeherder']
if 'machine' not in build_treeherder_config:
message = '({}), extra.treeherder.machine required for all builds'
raise ValueError(message.format(build['task']))
if 'build' not in build_treeherder_config:
build_treeherder_config['build'] = \
build_treeherder_config['machine']
if 'collection' not in build_treeherder_config:
build_treeherder_config['collection'] = {'opt': True}
if len(build_treeherder_config['collection'].keys()) != 1:
message = '({}), extra.treeherder.collection must contain one type'
raise ValueError(message.fomrat(build['task']))
for post_build in build['post-build']:
# copy over the old parameters to update the template
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
post_parameters = merge_dicts(
build_parameters,
post_build.get('additional-parameters', {}))
post_task = configure_dependent_task(post_build['task'],
post_parameters, slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph, post_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(post_task, interactive)
graph['tasks'].append(post_task)
for test in build['dependents']:
test = test['allowed_build_tasks'][build['task']]
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
test_parameters = merge_dicts(
build_parameters, test.get('additional-parameters', {}))
test_parameters = copy.copy(build_parameters)
if tests_url:
test_parameters['tests_url'] = tests_url
if test_packages_url:
test_parameters['test_packages_url'] = test_packages_url
if mozharness_url:
test_parameters['mozharness_url'] = mozharness_url
test_definition = templates.load(test['task'], {})['task']
chunk_config = test_definition['extra'].get('chunks', {})
# Allow branch configs to override task level chunking...
if 'chunks' in test:
chunk_config['total'] = test['chunks']
chunked = 'total' in chunk_config
if chunked:
test_parameters['total_chunks'] = chunk_config['total']
if 'suite' in test_definition['extra']:
suite_config = test_definition['extra']['suite']
test_parameters['suite'] = suite_config['name']
test_parameters['flavor'] = suite_config.get('flavor', '')
for chunk in range(1, chunk_config.get('total', 1) + 1):
if 'only_chunks' in test and chunked and \
chunk not in test['only_chunks']:
continue
if chunked:
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(
test['task'], test_parameters, slugid(), templates,
build_treeherder_config)
normalize_image_details(graph, test_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(test_task, interactive)
if params['revision_hash']:
routes_transform.decorate_task_treeherder_routes(
test_task['task'], treeherder_route)
graph['tasks'].append(test_task)
define_task = DEFINE_TASK.format(
test_task['task']['workerType'])
graph['scopes'].append(define_task)
graph['scopes'].extend(test_task['task'].get('scopes', []))
graph['scopes'] = list(set(graph['scopes']))
if params['print_names_only']:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print('{}- {}'.format(' ' * indent,
task['task']['metadata']['name']))
for child in tIDs[task['taskId']]:
print_task(child, indent=indent + 2)
# build a dependency map
for task in graph['tasks']:
if 'requires' in task:
for tID in task['requires']:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph['tasks']:
if 'requires' not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params['ci'] is True:
graph.pop('scopes', None)
graph.pop('metadata', None)
print(json.dumps(graph, indent=4))
def create_graph(self, **params):
from taskcluster_graph.commit_parser import parse_commit
from slugid import nice as slugid
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
if params['dry_run']:
from taskcluster_graph.dry_run import (
json_time_from_now,
current_json_time,
slugid,
)
project = params['project']
message = params.get('message', '') if project == 'try' else DEFAULT_TRY
# Message would only be blank when not created from decision task
if project == 'try' and not message:
sys.stderr.write(
"Must supply commit message when creating try graph. " \
"Example: --message='try: -b do -p all -u all'"
)
sys.exit(1)
templates = Templates(ROOT)
job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph = parse_commit(message, jobs)
cmdline_interactive = params.get('interactive', False)
# Default to current time if querying the head rev fails
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
pushinfo = query_pushinfo(params['head_repository'], params['head_rev'])
if pushinfo:
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
# Template parameters used when expanding the graph
parameters = dict(gaia_info().items() + {
'index': 'index',
'project': project,
'pushlog_id': params.get('pushlog_id', 0),
'docker_image': docker_image,
'base_repository': params['base_repository'] or \
params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'pushdate': pushdate,
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'from_now': json_time_from_now,
'now': current_json_time(),
'revision_hash': params['revision_hash']
}.items())
treeherder_route = '{}.{}'.format(
params['project'],
params.get('revision_hash', '')
)
routes_file = os.path.join(ROOT, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {
'tasks': [],
'scopes': []
}
if params['revision_hash']:
for env in TREEHERDER_ROUTES:
graph['scopes'].append('queue:route:{}.{}'.format(TREEHERDER_ROUTES[env], treeherder_route))
graph['metadata'] = {
'source': 'http://todo.com/what/goes/here',
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = dict(parameters)
build_parameters['build_slugid'] = slugid()
build_task = templates.load(build['task'], build_parameters)
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
if params['revision_hash']:
decorate_task_treeherder_routes(build_task['task'],
treeherder_route)
decorate_task_json_routes(build,
build_task['task'],
json_routes,
build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph['tasks'].append(build_task)
test_packages_url, tests_url, mozharness_url = None, None, None
if 'test_packages' in build_task['task']['extra']['locations']:
test_packages_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['test_packages']
)
if 'tests' in build_task['task']['extra']['locations']:
tests_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['tests']
)
if 'mozharness' in build_task['task']['extra']['locations']:
mozharness_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['mozharness']
)
build_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations']['build']
)
build_parameters['build_url'] = build_url
# img_url is only necessary for device builds
img_url = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations'].get('img', '')
)
build_parameters['img_url'] = img_url
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
for route in build_task['task'].get('routes', []):
if route.startswith('index.gecko.v2') and route in all_routes:
raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
route,
build_task['task']['metadata']['name'],
all_routes[route],
))
all_routes[route] = build_task['task']['metadata']['name']
graph['scopes'].append(define_task)
graph['scopes'].extend(build_task['task'].get('scopes', []))
route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
graph['scopes'].extend(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task['task']['extra']['treeherder']
if 'machine' not in build_treeherder_config:
message = '({}), extra.treeherder.machine required for all builds'
raise ValueError(message.format(build['task']))
if 'build' not in build_treeherder_config:
build_treeherder_config['build'] = \
build_treeherder_config['machine']
if 'collection' not in build_treeherder_config:
build_treeherder_config['collection'] = { 'opt': True }
if len(build_treeherder_config['collection'].keys()) != 1:
message = '({}), extra.treeherder.collection must contain one type'
raise ValueError(message.fomrat(build['task']))
for post_build in build['post-build']:
# copy over the old parameters to update the template
post_parameters = copy.copy(build_parameters)
post_task = configure_dependent_task(post_build['task'],
post_parameters,
slugid(),
templates,
build_treeherder_config)
set_interactive_task(post_task, interactive)
graph['tasks'].append(post_task)
for test in build['dependents']:
test = test['allowed_build_tasks'][build['task']]
test_parameters = copy.copy(build_parameters)
if tests_url:
test_parameters['tests_url'] = tests_url
if test_packages_url:
test_parameters['test_packages_url'] = test_packages_url
if mozharness_url:
test_parameters['mozharness_url'] = mozharness_url
test_definition = templates.load(test['task'], {})['task']
chunk_config = test_definition['extra']['chunks']
# Allow branch configs to override task level chunking...
if 'chunks' in test:
chunk_config['total'] = test['chunks']
test_parameters['total_chunks'] = chunk_config['total']
for chunk in range(1, chunk_config['total'] + 1):
if 'only_chunks' in test and \
chunk not in test['only_chunks']:
continue
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(test['task'],
test_parameters,
slugid(),
templates,
build_treeherder_config)
set_interactive_task(test_task, interactive)
if params['revision_hash']:
decorate_task_treeherder_routes(
test_task['task'], treeherder_route)
graph['tasks'].append(test_task)
define_task = DEFINE_TASK.format(
test_task['task']['workerType']
)
graph['scopes'].append(define_task)
graph['scopes'].extend(test_task['task'].get('scopes', []))
graph['scopes'] = list(set(graph['scopes']))
if params['print_names_only']:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print('{}- {}'.format(' ' * indent, task['task']['metadata']['name']))
for child in tIDs[task['taskId']]:
print_task(child, indent=indent+2)
# build a dependency map
for task in graph['tasks']:
if 'requires' in task:
for tID in task['requires']:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph['tasks']:
if 'requires' not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params['ci'] is True:
graph.pop('scopes', None)
graph.pop('metadata', None)
print(json.dumps(graph, indent=4))
def create_graph(self, **params):
from functools import partial
from slugid import nice as slugid
import taskcluster_graph.transform.routes as routes_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (
docker_image,
normalize_image_details,
task_id_for_image
)
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
if params['dry_run']:
from taskcluster_graph.dry_run import (
json_time_from_now,
current_json_time,
slugid,
)
project = params['project']
message = params.get('message', '') if project == 'try' else DEFAULT_TRY
# Message would only be blank when not created from decision task
if project == 'try' and not message:
sys.stderr.write(
"Must supply commit message when creating try graph. " \
"Example: --message='try: -b do -p all -u all'"
)
sys.exit(1)
templates = Templates(ROOT)
job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph = parse_commit(message, jobs)
cmdline_interactive = params.get('interactive', False)
# Default to current time if querying the head rev fails
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
pushinfo = query_pushinfo(params['head_repository'], params['head_rev'])
if pushinfo:
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
# Template parameters used when expanding the graph
seen_images = {}
parameters = dict(gaia_info().items() + {
'index': 'index',
'project': project,
'pushlog_id': params.get('pushlog_id', 0),
'docker_image': docker_image,
'task_id_for_image': partial(task_id_for_image, seen_images, project),
'base_repository': params['base_repository'] or \
params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
'revision_hash': params['revision_hash']
}.items())
treeherder_route = '{}.{}'.format(
params['project'],
params.get('revision_hash', '')
)
routes_file = os.path.join(ROOT, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {
'tasks': [],
'scopes': set(),
}
if params['revision_hash']:
for env in routes_transform.TREEHERDER_ROUTES:
route = 'queue:route:{}.{}'.format(
routes_transform.TREEHERDER_ROUTES[env],
treeherder_route)
graph['scopes'].add(route)
graph['metadata'] = {
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = merge_dicts(parameters, build['additional-parameters']);
build_parameters['build_slugid'] = slugid()
build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
build_task = templates.load(build['task'], build_parameters)
# Copy build_* attributes to expose them to post-build tasks
# as well as json routes and tests
task_extra = build_task['task']['extra']
build_parameters['build_name'] = task_extra['build_name']
build_parameters['build_type'] = task_extra['build_type']
build_parameters['build_product'] = task_extra['build_product']
normalize_image_details(graph,
build_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
if params['revision_hash']:
routes_transform.decorate_task_treeherder_routes(build_task['task'],
treeherder_route)
routes_transform.decorate_task_json_routes(build_task['task'],
json_routes,
build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph['tasks'].append(build_task)
for location in build_task['task']['extra'].get('locations', {}):
build_parameters['{}_url'.format(location)] = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations'][location]
)
for url in build_task['task']['extra'].get('url', {}):
build_parameters['{}_url'.format(url)] = \
build_task['task']['extra']['url'][url]
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
for route in build_task['task'].get('routes', []):
if route.startswith('index.gecko.v2') and route in all_routes:
raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
route,
build_task['task']['metadata']['name'],
all_routes[route],
))
all_routes[route] = build_task['task']['metadata']['name']
graph['scopes'].add(define_task)
graph['scopes'] |= set(build_task['task'].get('scopes', []))
route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
graph['scopes'] |= set(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task['task']['extra']['treeherder']
if 'machine' not in build_treeherder_config:
message = '({}), extra.treeherder.machine required for all builds'
raise ValueError(message.format(build['task']))
if 'build' not in build_treeherder_config:
build_treeherder_config['build'] = \
build_treeherder_config['machine']
if 'collection' not in build_treeherder_config:
build_treeherder_config['collection'] = { 'opt': True }
if len(build_treeherder_config['collection'].keys()) != 1:
message = '({}), extra.treeherder.collection must contain one type'
raise ValueError(message.fomrat(build['task']))
for post_build in build['post-build']:
# copy over the old parameters to update the template
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
post_parameters = merge_dicts(build_parameters,
post_build.get('additional-parameters', {}))
post_task = configure_dependent_task(post_build['task'],
post_parameters,
slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph,
post_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(post_task, interactive)
graph['tasks'].append(post_task)
for test in build['dependents']:
test = test['allowed_build_tasks'][build['task']]
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
test_parameters = merge_dicts(build_parameters,
test.get('additional-parameters', {}))
test_parameters = copy.copy(build_parameters)
test_definition = templates.load(test['task'], {})['task']
chunk_config = test_definition['extra'].get('chunks', {})
# Allow branch configs to override task level chunking...
if 'chunks' in test:
chunk_config['total'] = test['chunks']
chunked = 'total' in chunk_config
if chunked:
test_parameters['total_chunks'] = chunk_config['total']
if 'suite' in test_definition['extra']:
suite_config = test_definition['extra']['suite']
test_parameters['suite'] = suite_config['name']
test_parameters['flavor'] = suite_config.get('flavor', '')
for chunk in range(1, chunk_config.get('total', 1) + 1):
if 'only_chunks' in test and chunked and \
chunk not in test['only_chunks']:
continue
if chunked:
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(test['task'],
test_parameters,
slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph,
test_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(test_task, interactive)
if params['revision_hash']:
routes_transform.decorate_task_treeherder_routes(
test_task['task'],
treeherder_route
)
graph['tasks'].append(test_task)
define_task = DEFINE_TASK.format(
test_task['task']['workerType']
)
graph['scopes'].add(define_task)
graph['scopes'] |= set(test_task['task'].get('scopes', []))
graph['scopes'] = sorted(graph['scopes'])
if params['print_names_only']:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print('{}- {}'.format(' ' * indent, task['task']['metadata']['name']))
for child in tIDs[task['taskId']]:
print_task(child, indent=indent+2)
# build a dependency map
for task in graph['tasks']:
if 'requires' in task:
for tID in task['requires']:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph['tasks']:
if 'requires' not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params['ci'] is True:
graph.pop('scopes', None)
graph.pop('metadata', None)
print(json.dumps(graph, indent=4, sort_keys=True))
def create_isolate_failure_tasks(task_definition, failures, level, times):
"""
Create tasks to re-run the original task plus tasks to test
each failing test directory and individual path.
"""
logger.info("Isolate task:\n{}".format(
json.dumps(task_definition, indent=2)))
task_name = task_definition['metadata']['name']
repeatable_task = False
if ('crashtest' in task_name or 'mochitest' in task_name
or 'reftest' in task_name and 'jsreftest' not in task_name):
repeatable_task = True
th_dict = task_definition['extra']['treeherder']
symbol = th_dict['symbol']
is_windows = 'windows' in th_dict['machine']['platform']
suite = task_definition['extra']['suite']
if '-coverage' in suite:
suite = suite[:suite.index('-coverage')]
is_wpt = 'web-platform-tests' in suite
# command is a copy of task_definition['payload']['command'] from the original task.
# It is used to create the new version containing the
# task_definition['payload']['command'] with repeat_args which is updated every time
# through the failure_group loop.
command = copy.deepcopy(task_definition['payload']['command'])
th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I'
th_dict['tier'] = 3
for i in range(times):
create_task_from_def(slugid(), task_definition, level)
if repeatable_task:
task_definition['payload']['maxRunTime'] = 3600 * 3
for failure_group in failures:
if failure_group == 'dirs':
failure_group_suffix = '-id'
# execute 5 total loops
repeat_args = ['--repeat=4'] if repeatable_task else []
else:
failure_group_suffix = '-it'
# execute 20 total loops
repeat_args = ['--repeat=19'] if repeatable_task else []
if repeat_args:
task_definition['payload']['command'] = add_args_to_command(
command, extra_args=repeat_args)
else:
task_definition['payload']['command'] = command
# saved_command is a saved version of the
# task_definition['payload']['command'] with the repeat
# arguments. We need to save it since
# task_definition['payload']['command'] will be modified in the failure_path loop
# when we are isolating web-platform-tests.
saved_command = copy.deepcopy(task_definition['payload']['command'])
for failure_path in failures[failure_group]:
th_dict['symbol'] = symbol + failure_group_suffix
if is_windows and not is_wpt:
failure_path = '\\'.join(failure_path.split('/'))
if is_wpt:
include_args = ['--include={}'.format(failure_path)]
task_definition['payload']['command'] = add_args_to_command(
saved_command, extra_args=include_args)
else:
task_definition['payload']['env'][
'MOZHARNESS_TEST_PATHS'] = six.ensure_text(
json.dumps({suite: [failure_path]}))
logger.info("Creating task for path {} with command {}".format(
failure_path, task_definition['payload']['command']))
for i in range(times):
create_task_from_def(slugid(), task_definition, level)
def mklabel():
return slugid()
def create_graph(self, **params):
from functools import partial
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (docker_image,
normalize_image_details,
task_id_for_image)
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
if params['dry_run']:
from taskcluster_graph.dry_run import (
json_time_from_now,
current_json_time,
slugid,
)
project = params['project']
message = params.get('message',
'') if project == 'try' else DEFAULT_TRY
templates = Templates(ROOT)
job_path = os.path.join(ROOT, 'tasks', 'branches', project,
'job_flags.yml')
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph, trigger_tests = parse_commit(message, jobs)
cmdline_interactive = params.get('interactive', False)
# Default to current time if querying the head rev fails
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
vcs_info = query_vcs_info(params['head_repository'],
params['head_rev'])
changed_files = set()
if vcs_info:
pushdate = time.strftime('%Y%m%d%H%M%S',
time.gmtime(vcs_info.pushdate))
sys.stderr.write('%d commits influencing task scheduling:\n' %
len(vcs_info.changesets))
for c in vcs_info.changesets:
sys.stderr.write(
'%s %s\n' %
(c['node'][0:12], c['desc'].splitlines()[0].encode(
'ascii', 'ignore')))
changed_files |= set(c['files'])
# Template parameters used when expanding the graph
seen_images = {}
parameters = dict(gaia_info().items() + {
'index': 'index',
'project': project,
'pushlog_id': params.get('pushlog_id', 0),
'docker_image': docker_image,
'task_id_for_image': partial(task_id_for_image, seen_images, project),
'base_repository': params['base_repository'] or \
params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
'revision_hash': params['revision_hash']
}.items())
treeherder_route = '{}.{}'.format(params['project'],
params.get('revision_hash', ''))
routes_file = os.path.join(ROOT, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {
'tasks': [],
'scopes': set(),
}
if params['revision_hash']:
for env in routes_transform.TREEHERDER_ROUTES:
route = 'queue:route:{}.{}'.format(
routes_transform.TREEHERDER_ROUTES[env], treeherder_route)
graph['scopes'].add(route)
graph['metadata'] = {
'source':
'{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
repo=params['head_repository'], rev=params['head_rev']),
'owner':
params['owner'],
# TODO: Add full mach commands to this example?
'description':
'Task graph generated via ./mach taskcluster-graph',
'name':
'task graph local'
}
# Filter the job graph according to conditions met by this invocation run.
def should_run(task):
# Old style build or test task that doesn't define conditions. Always runs.
if 'when' not in task:
return True
# Command line override to not filter.
if params['ignore_conditions']:
return True
when = task['when']
# If the task defines file patterns and we have a set of changed
# files to compare against, only run if a file pattern matches one
# of the changed files.
file_patterns = when.get('file_patterns', None)
if file_patterns and changed_files:
for pattern in file_patterns:
for path in changed_files:
if mozpackmatch(path, pattern):
sys.stderr.write(
'scheduling %s because pattern %s '
'matches %s\n' % (task['task'], pattern, path))
return True
# No file patterns matched. Discard task.
sys.stderr.write(
'discarding %s because no relevant files changed\n' %
task['task'])
return False
return True
job_graph = filter(should_run, job_graph)
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = merge_dicts(parameters,
build['additional-parameters'])
build_parameters['build_slugid'] = slugid()
build_parameters[
'source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(
repo=params['head_repository'],
rev=params['head_rev'],
file=build['task'])
build_task = templates.load(build['task'], build_parameters)
# Copy build_* attributes to expose them to post-build tasks
# as well as json routes and tests
task_extra = build_task['task']['extra']
build_parameters['build_name'] = task_extra['build_name']
build_parameters['build_type'] = task_extra['build_type']
build_parameters['build_product'] = task_extra['build_product']
normalize_image_details(graph, build_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
if params['revision_hash']:
treeherder_transform.add_treeherder_revision_info(
build_task['task'], params['head_rev'],
params['revision_hash'])
routes_transform.decorate_task_treeherder_routes(
build_task['task'], treeherder_route)
routes_transform.decorate_task_json_routes(
build_task['task'], json_routes, build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph['tasks'].append(build_task)
for location in build_task['task']['extra'].get('locations', {}):
build_parameters['{}_url'.format(
location)] = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations'][location])
for url in build_task['task']['extra'].get('url', {}):
build_parameters['{}_url'.format(url)] = \
build_task['task']['extra']['url'][url]
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
for route in build_task['task'].get('routes', []):
if route.startswith('index.gecko.v2') and route in all_routes:
raise Exception(
"Error: route '%s' is in use by multiple tasks: '%s' and '%s'"
% (
route,
build_task['task']['metadata']['name'],
all_routes[route],
))
all_routes[route] = build_task['task']['metadata']['name']
graph['scopes'].add(define_task)
graph['scopes'] |= set(build_task['task'].get('scopes', []))
route_scopes = map(lambda route: 'queue:route:' + route,
build_task['task'].get('routes', []))
graph['scopes'] |= set(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task['task']['extra']['treeherder']
if 'machine' not in build_treeherder_config:
message = '({}), extra.treeherder.machine required for all builds'
raise ValueError(message.format(build['task']))
if 'build' not in build_treeherder_config:
build_treeherder_config['build'] = \
build_treeherder_config['machine']
if 'collection' not in build_treeherder_config:
build_treeherder_config['collection'] = {'opt': True}
if len(build_treeherder_config['collection'].keys()) != 1:
message = '({}), extra.treeherder.collection must contain one type'
raise ValueError(message.fomrat(build['task']))
for post_build in build['post-build']:
# copy over the old parameters to update the template
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
post_parameters = merge_dicts(
build_parameters,
post_build.get('additional-parameters', {}))
post_task = configure_dependent_task(post_build['task'],
post_parameters, slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph, post_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(post_task, interactive)
treeherder_transform.add_treeherder_revision_info(
post_task['task'], params['head_rev'],
params['revision_hash'])
graph['tasks'].append(post_task)
for test in build['dependents']:
test = test['allowed_build_tasks'][build['task']]
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
test_parameters = merge_dicts(
build_parameters, test.get('additional-parameters', {}))
test_parameters = copy.copy(build_parameters)
test_definition = templates.load(test['task'], {})['task']
chunk_config = test_definition['extra'].get('chunks', {})
# Allow branch configs to override task level chunking...
if 'chunks' in test:
chunk_config['total'] = test['chunks']
chunked = 'total' in chunk_config
if chunked:
test_parameters['total_chunks'] = chunk_config['total']
if 'suite' in test_definition['extra']:
suite_config = test_definition['extra']['suite']
test_parameters['suite'] = suite_config['name']
test_parameters['flavor'] = suite_config.get('flavor', '')
for chunk in range(1, chunk_config.get('total', 1) + 1):
if 'only_chunks' in test and chunked and \
chunk not in test['only_chunks']:
continue
if chunked:
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(
test['task'], test_parameters, slugid(), templates,
build_treeherder_config)
normalize_image_details(graph, test_task, seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(test_task, interactive)
if params['revision_hash']:
treeherder_transform.add_treeherder_revision_info(
test_task['task'], params['head_rev'],
params['revision_hash'])
routes_transform.decorate_task_treeherder_routes(
test_task['task'], treeherder_route)
# This will schedule test jobs N times
for i in range(0, trigger_tests):
graph['tasks'].append(test_task)
# If we're scheduling more tasks each have to be unique
test_task = copy.deepcopy(test_task)
test_task['taskId'] = slugid()
define_task = DEFINE_TASK.format(
test_task['task']['workerType'])
graph['scopes'].add(define_task)
graph['scopes'] |= set(test_task['task'].get('scopes', []))
graph['scopes'] = sorted(graph['scopes'])
if params['print_names_only']:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print('{}- {}'.format(' ' * indent,
task['task']['metadata']['name']))
for child in tIDs[task['taskId']]:
print_task(child, indent=indent + 2)
# build a dependency map
for task in graph['tasks']:
if 'requires' in task:
for tID in task['requires']:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph['tasks']:
if 'requires' not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params['ci'] is True:
graph.pop('scopes', None)
graph.pop('metadata', None)
print(json.dumps(graph, indent=4, sort_keys=True))
def create_graph(self, **params):
from functools import partial
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
from taskcluster_graph.mach_util import (
merge_dicts,
gaia_info,
configure_dependent_task,
set_interactive_task,
remove_caches_from_task,
query_vcs_info
)
import taskcluster_graph.transform.routes as routes_transform
import taskcluster_graph.transform.treeherder as treeherder_transform
from taskcluster_graph.commit_parser import parse_commit
from taskcluster_graph.image_builder import (
docker_image,
normalize_image_details,
task_id_for_image
)
from taskcluster_graph.from_now import (
json_time_from_now,
current_json_time,
)
from taskcluster_graph.templates import Templates
import taskcluster_graph.build_task
if params['dry_run']:
from taskcluster_graph.dry_run import (
json_time_from_now,
current_json_time,
slugid,
)
project = params['project']
message = params.get('message', '') if project == 'try' else DEFAULT_TRY
templates = Templates(ROOT)
job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
jobs = templates.load(job_path, {})
job_graph, trigger_tests = parse_commit(message, jobs)
cmdline_interactive = params.get('interactive', False)
# Default to current time if querying the head rev fails
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
changed_files = set()
if vcs_info:
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate))
sys.stderr.write('%d commits influencing task scheduling:\n' %
len(vcs_info.changesets))
for c in vcs_info.changesets:
sys.stderr.write('%s %s\n' % (
c['node'][0:12], c['desc'].splitlines()[0].encode('ascii', 'ignore')))
changed_files |= set(c['files'])
# Template parameters used when expanding the graph
seen_images = {}
parameters = dict(gaia_info().items() + {
'index': 'index',
'project': project,
'pushlog_id': params.get('pushlog_id', 0),
'docker_image': docker_image,
'task_id_for_image': partial(task_id_for_image, seen_images, project),
'base_repository': params['base_repository'] or \
params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'pushdate': pushdate,
'pushtime': pushdate[8:],
'year': pushdate[0:4],
'month': pushdate[4:6],
'day': pushdate[6:8],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
'revision_hash': params['revision_hash']
}.items())
treeherder_route = '{}.{}'.format(
params['project'],
params.get('revision_hash', '')
)
routes_file = os.path.join(ROOT, 'routes.json')
with open(routes_file) as f:
contents = json.load(f)
json_routes = contents['routes']
# TODO: Nightly and/or l10n routes
# Task graph we are generating for taskcluster...
graph = {
'tasks': [],
'scopes': set(),
}
if params['revision_hash']:
for env in routes_transform.TREEHERDER_ROUTES:
route = 'queue:route:{}.{}'.format(
routes_transform.TREEHERDER_ROUTES[env],
treeherder_route)
graph['scopes'].add(route)
graph['metadata'] = {
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']),
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
'name': 'task graph local'
}
# Filter the job graph according to conditions met by this invocation run.
def should_run(task):
# Old style build or test task that doesn't define conditions. Always runs.
if 'when' not in task:
return True
# Command line override to not filter.
if params['ignore_conditions']:
return True
when = task['when']
# If the task defines file patterns and we have a set of changed
# files to compare against, only run if a file pattern matches one
# of the changed files.
file_patterns = when.get('file_patterns', None)
if file_patterns and changed_files:
# Always consider changes to the task definition itself
file_patterns.append('testing/taskcluster/{task}'.format(task=task['task']))
for pattern in file_patterns:
for path in changed_files:
if mozpackmatch(path, pattern):
sys.stderr.write('scheduling %s because pattern %s '
'matches %s\n' % (task['task'],
pattern,
path))
return True
# No file patterns matched. Discard task.
sys.stderr.write('discarding %s because no relevant files changed\n' %
task['task'])
return False
return True
job_graph = filter(should_run, job_graph)
all_routes = {}
for build in job_graph:
interactive = cmdline_interactive or build["interactive"]
build_parameters = merge_dicts(parameters, build['additional-parameters']);
build_parameters['build_slugid'] = slugid()
build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task'])
build_task = templates.load(build['task'], build_parameters)
# Copy build_* attributes to expose them to post-build tasks
# as well as json routes and tests
task_extra = build_task['task']['extra']
build_parameters['build_name'] = task_extra['build_name']
build_parameters['build_type'] = task_extra['build_type']
build_parameters['build_product'] = task_extra['build_product']
normalize_image_details(graph,
build_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(build_task, interactive)
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
if params['revision_hash']:
treeherder_transform.add_treeherder_revision_info(build_task['task'],
params['head_rev'],
params['revision_hash'])
routes_transform.decorate_task_treeherder_routes(build_task['task'],
treeherder_route)
routes_transform.decorate_task_json_routes(build_task['task'],
json_routes,
build_parameters)
# Ensure each build graph is valid after construction.
taskcluster_graph.build_task.validate(build_task)
graph['tasks'].append(build_task)
for location in build_task['task']['extra'].get('locations', {}):
build_parameters['{}_url'.format(location)] = ARTIFACT_URL.format(
build_parameters['build_slugid'],
build_task['task']['extra']['locations'][location]
)
for url in build_task['task']['extra'].get('url', {}):
build_parameters['{}_url'.format(url)] = \
build_task['task']['extra']['url'][url]
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
for route in build_task['task'].get('routes', []):
if route.startswith('index.gecko.v2') and route in all_routes:
raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (
route,
build_task['task']['metadata']['name'],
all_routes[route],
))
all_routes[route] = build_task['task']['metadata']['name']
graph['scopes'].add(define_task)
graph['scopes'] |= set(build_task['task'].get('scopes', []))
route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
graph['scopes'] |= set(route_scopes)
# Treeherder symbol configuration for the graph required for each
# build so tests know which platform they belong to.
build_treeherder_config = build_task['task']['extra']['treeherder']
if 'machine' not in build_treeherder_config:
message = '({}), extra.treeherder.machine required for all builds'
raise ValueError(message.format(build['task']))
if 'build' not in build_treeherder_config:
build_treeherder_config['build'] = \
build_treeherder_config['machine']
if 'collection' not in build_treeherder_config:
build_treeherder_config['collection'] = { 'opt': True }
if len(build_treeherder_config['collection'].keys()) != 1:
message = '({}), extra.treeherder.collection must contain one type'
raise ValueError(message.fomrat(build['task']))
for post_build in build['post-build']:
# copy over the old parameters to update the template
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
post_parameters = merge_dicts(build_parameters,
post_build.get('additional-parameters', {}))
post_task = configure_dependent_task(post_build['task'],
post_parameters,
slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph,
post_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(post_task, interactive)
treeherder_transform.add_treeherder_revision_info(post_task['task'],
params['head_rev'],
params['revision_hash'])
if project == "try":
set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
graph['tasks'].append(post_task)
for test in build['dependents']:
test = test['allowed_build_tasks'][build['task']]
# TODO additional-parameters is currently not an option, only
# enabled for build tasks
test_parameters = merge_dicts(build_parameters,
test.get('additional-parameters', {}))
test_parameters = copy.copy(build_parameters)
test_definition = templates.load(test['task'], {})['task']
chunk_config = test_definition['extra'].get('chunks', {})
# Allow branch configs to override task level chunking...
if 'chunks' in test:
chunk_config['total'] = test['chunks']
chunked = 'total' in chunk_config
if chunked:
test_parameters['total_chunks'] = chunk_config['total']
if 'suite' in test_definition['extra']:
suite_config = test_definition['extra']['suite']
test_parameters['suite'] = suite_config['name']
test_parameters['flavor'] = suite_config.get('flavor', '')
for chunk in range(1, chunk_config.get('total', 1) + 1):
if 'only_chunks' in test and chunked and \
chunk not in test['only_chunks']:
continue
if chunked:
test_parameters['chunk'] = chunk
test_task = configure_dependent_task(test['task'],
test_parameters,
slugid(),
templates,
build_treeherder_config)
normalize_image_details(graph,
test_task,
seen_images,
build_parameters,
os.environ.get('TASK_ID', None))
set_interactive_task(test_task, interactive)
if params['revision_hash']:
treeherder_transform.add_treeherder_revision_info(test_task['task'],
params['head_rev'],
params['revision_hash'])
routes_transform.decorate_task_treeherder_routes(
test_task['task'],
treeherder_route
)
if project == "try":
set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
# This will schedule test jobs N times
for i in range(0, trigger_tests):
graph['tasks'].append(test_task)
# If we're scheduling more tasks each have to be unique
test_task = copy.deepcopy(test_task)
test_task['taskId'] = slugid()
define_task = DEFINE_TASK.format(
test_task['task']['workerType']
)
graph['scopes'].add(define_task)
graph['scopes'] |= set(test_task['task'].get('scopes', []))
graph['scopes'] = sorted(graph['scopes'])
if params['print_names_only']:
tIDs = defaultdict(list)
def print_task(task, indent=0):
print('{}- {}'.format(' ' * indent, task['task']['metadata']['name']))
for child in tIDs[task['taskId']]:
print_task(child, indent=indent+2)
# build a dependency map
for task in graph['tasks']:
if 'requires' in task:
for tID in task['requires']:
tIDs[tID].append(task)
# recursively print root tasks
for task in graph['tasks']:
if 'requires' not in task:
print_task(task)
return
# When we are extending the graph remove extra fields...
if params['ci'] is True:
graph.pop('scopes', None)
graph.pop('metadata', None)
print(json.dumps(graph, indent=4, sort_keys=True))
