def create_tasks(taskgraph, label_to_taskid): # TODO: use the taskGroupId of the decision task task_group_id = slugid() taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()} session = requests.Session() # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase # that limit. Connections are established as needed, so using a large value # should not negatively impact performance. http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY, pool_maxsize=CONCURRENCY) session.mount('https://', http_adapter) session.mount('http://', http_adapter) decision_task_id = os.environ.get('TASK_ID') with futures.ThreadPoolExecutor(CONCURRENCY) as e: fs = {} # We can't submit a task until its dependencies have been submitted. # So our strategy is to walk the graph and submit tasks once all # their dependencies have been submitted. # # Using visit_postorder() here isn't the most efficient: we'll # block waiting for dependencies of task N to submit even though # dependencies for task N+1 may be finished. If we need to optimize # this further, we can build a graph of task dependencies and walk # that. for task_id in taskgraph.graph.visit_postorder(): task_def = taskgraph.tasks[task_id].task attributes = taskgraph.tasks[task_id].attributes # if this task has no dependencies, make it depend on this decision # task so that it does not start immediately; and so that if this loop # fails halfway through, none of the already-created tasks run. if decision_task_id and not task_def.get('dependencies'): task_def['dependencies'] = [decision_task_id] task_def['taskGroupId'] = task_group_id task_def['schedulerId'] = '-' # Wait for dependencies before submitting this. deps_fs = [fs[dep] for dep in task_def.get('dependencies', []) if dep in fs] for f in futures.as_completed(deps_fs): f.result() fs[task_id] = e.submit(_create_task, session, task_id, taskid_to_label[task_id], task_def) # Schedule tasks as many times as task_duplicates indicates for i in range(1, attributes.get('task_duplicates', 1)): # We use slugid() since we want a distinct task id fs[task_id] = e.submit(_create_task, session, slugid(), taskid_to_label[task_id], task_def) # Wait for all futures to complete. for f in futures.as_completed(fs.values()): f.result()
def add_new_jobs_action(parameters, input, task_group_id, task_id, task): decision_task_id = find_decision_task(parameters) full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") for elem in input['tasks']: if elem in full_task_graph.tasks: task = full_task_graph.tasks[elem] # fix up the task's dependencies, similar to how optimization would # have done in the decision dependencies = { name: label_to_taskid[label] for name, label in task.dependencies.iteritems() } task_def = resolve_task_references(task.label, task.task, dependencies) task_def.setdefault('dependencies', []).extend(dependencies.itervalues()) # actually create the new task create_task(slugid(), task_def, parameters['level']) else: raise Exception('{} was not found in the task-graph'.format(elem))
def test_retrigger_action(parameters, input, task_group_id, task_id, task): new_task_definition = copy.copy(task) # set new created, deadline, and expiry fields new_task_definition['created'] = current_json_time() new_task_definition['deadline'] = json_time_from_now('1d') new_task_definition['expires'] = json_time_from_now('30d') # reset artifact expiry for artifact in new_task_definition['payload'].get('artifacts', {}).values(): artifact['expires'] = new_task_definition['expires'] # don't want to run mozharness tests, want a custom mach command instead new_task_definition['payload']['command'] += ['--no-run-tests'] custom_mach_command = [task['tags']['test-type']] # mochitests may specify a flavor if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'): custom_mach_command += [ '--keep-open=false', '-f', new_task_definition['payload']['env']['MOCHITEST_FLAVOR'] ] enable_e10s = json.loads(new_task_definition['payload']['env'].get( 'ENABLE_E10S', 'true')) if not enable_e10s: custom_mach_command += ['--disable-e10s'] custom_mach_command += [ '--log-tbpl=-', '--log-tbpl-level={}'.format(input['logLevel']) ] if input.get('runUntilFail'): custom_mach_command += ['--run-until-failure'] if input.get('repeat'): custom_mach_command += ['--repeat', str(input['repeat'])] # add any custom gecko preferences for (key, val) in input.get('preferences', {}).iteritems(): custom_mach_command += ['--setpref', '{}={}'.format(key, val)] custom_mach_command += [input['path']] new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join( custom_mach_command) # update environment new_task_definition['payload']['env'].update(input.get('environment', {})) # tweak the treeherder symbol new_task_definition['extra']['treeherder']['symbol'] += '-custom' logging.info("New task definition: %s", new_task_definition) # actually create the new task new_task_id = slugid() logger.info("Creating new mochitest task with id %s", new_task_id) session = requests.Session() create_task(session, new_task_id, 'test-retrigger', new_task_definition)
def test_retrigger_action(parameters, input, task_group_id, task_id, task): new_task_definition = copy.copy(task) # set new created, deadline, and expiry fields new_task_definition['created'] = current_json_time() new_task_definition['deadline'] = json_time_from_now('1d') new_task_definition['expires'] = json_time_from_now('30d') # reset artifact expiry for artifact in new_task_definition['payload'].get('artifacts', {}).values(): artifact['expires'] = new_task_definition['expires'] # don't want to run mozharness tests, want a custom mach command instead new_task_definition['payload']['command'] += ['--no-run-tests'] custom_mach_command = [task['tags']['test-type']] # mochitests may specify a flavor if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'): custom_mach_command += [ '--keep-open=false', '-f', new_task_definition['payload']['env']['MOCHITEST_FLAVOR'] ] enable_e10s = json.loads(new_task_definition['payload']['env'].get( 'ENABLE_E10S', 'true')) if not enable_e10s: custom_mach_command += ['--disable-e10s'] custom_mach_command += ['--log-tbpl=-', '--log-tbpl-level={}'.format(input['logLevel'])] if input.get('runUntilFail'): custom_mach_command += ['--run-until-failure'] if input.get('repeat'): custom_mach_command += ['--repeat', str(input['repeat'])] # add any custom gecko preferences for (key, val) in input.get('preferences', {}).iteritems(): custom_mach_command += ['--setpref', '{}={}'.format(key, val)] custom_mach_command += [input['path']] new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join( custom_mach_command) # update environment new_task_definition['payload']['env'].update(input.get('environment', {})) # tweak the treeherder symbol new_task_definition['extra']['treeherder']['symbol'] += '-custom' logging.info("New task definition: %s", new_task_definition) # actually create the new task new_task_id = slugid() logger.info("Creating new mochitest task with id %s", new_task_id) session = requests.Session() create_task(session, new_task_id, 'test-retrigger', new_task_definition)
def derive_index_task(task, taskgraph, label_to_taskid, parameters, graph_config): """Create the shell of a task that depends on `task` and on the given docker image.""" purpose = "index-task" label = f"{purpose}-{task.label}" provisioner_id, worker_type = get_worker_type(graph_config, "misc", parameters["level"]) task_def = { "provisionerId": provisioner_id, "workerType": worker_type, "dependencies": [task.task_id], "created": { "relative-datestamp": "0 seconds" }, "deadline": task.task["deadline"], # no point existing past the parent task's deadline "expires": task.task["deadline"], "metadata": { "name": label, "description": "{} for {}".format(purpose, task.task["metadata"]["description"]), "owner": task.task["metadata"]["owner"], "source": task.task["metadata"]["source"], }, "scopes": [], "payload": { "image": { "path": "public/image.tar.zst", "namespace": "taskgraph.cache.level-3.docker-images.v2.index-task.latest", "type": "indexed-image", }, "features": { "taskclusterProxy": True, }, "maxRunTime": 600, }, } # only include the docker-image dependency here if it is actually in the # taskgraph (has not been optimized). It is included in # task_def['dependencies'] unconditionally. dependencies = {"parent": task.task_id} task = Task( kind="misc", label=label, attributes={}, task=task_def, dependencies=dependencies, ) task.task_id = slugid() return task, taskgraph, label_to_taskid
def handle_custom_retrigger(parameters, graph_config, input, task_group_id, task_id): task = taskcluster.get_task_definition(task_id) decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels( parameters, graph_config) pre_task = full_task_graph.tasks[task['metadata']['name']] # fix up the task's dependencies, similar to how optimization would # have done in the decision dependencies = {name: label_to_taskid[label] for name, label in pre_task.dependencies.iteritems()} new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies) new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues()) # don't want to run mozharness tests, want a custom mach command instead new_task_definition['payload']['command'] += ['--no-run-tests'] custom_mach_command = [task['tags']['test-type']] # mochitests may specify a flavor if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'): custom_mach_command += [ '--keep-open=false', '-f', new_task_definition['payload']['env']['MOCHITEST_FLAVOR'] ] enable_e10s = json.loads(new_task_definition['payload']['env'].get( 'ENABLE_E10S', 'true')) if not enable_e10s: custom_mach_command += ['--disable-e10s'] custom_mach_command += ['--log-tbpl=-', '--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))] if input.get('runUntilFail'): custom_mach_command += ['--run-until-failure'] if input.get('repeat'): custom_mach_command += ['--repeat', str(input.get('repeat', 30))] # add any custom gecko preferences for (key, val) in input.get('preferences', {}).iteritems(): custom_mach_command += ['--setpref', '{}={}'.format(key, val)] custom_mach_command += [input['path']] new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join( custom_mach_command) # update environment new_task_definition['payload']['env'].update(input.get('environment', {})) # tweak the treeherder symbol new_task_definition['extra']['treeherder']['symbol'] += '-custom' logging.info("New task definition: %s", new_task_definition) # actually create the new task new_task_id = slugid() create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def derive_misc_task(task, purpose, image, taskgraph, label_to_taskid): """Create the shell of a task that depends on `task` and on the given docker image.""" label = '{}-{}'.format(purpose, task.label) # this is why all docker image tasks are included in the target task graph: we # need to find them in label_to_taskid, if if nothing else required them image_taskid = label_to_taskid['build-docker-image-' + image] task_def = { 'provisionerId': 'gecko-t', 'workerType': 'misc', 'dependencies': [task.task_id, image_taskid], 'created': { 'relative-datestamp': '0 seconds' }, 'deadline': task.task['deadline'], # no point existing past the parent task's deadline 'expires': task.task['deadline'], 'metadata': { 'name': label, 'description': '{} for {}'.format(purpose, task.task['metadata']['description']), 'owner': task.task['metadata']['owner'], 'source': task.task['metadata']['source'], }, 'scopes': [], 'payload': { 'image': { 'path': 'public/image.tar.zst', 'taskId': image_taskid, 'type': 'task-image', }, 'features': { 'taskclusterProxy': True, }, 'maxRunTime': 600, }, } # only include the docker-image dependency here if it is actually in the # taskgraph (has not been optimized). It is included in # task_def['dependencies'] unconditionally. dependencies = {'parent': task.task_id} if image_taskid in taskgraph.tasks: dependencies['docker-image'] = image_taskid task = Task(kind='misc', label=label, attributes={}, task=task_def, dependencies=dependencies) task.task_id = slugid() return task
def get_subgraph(target_task_graph, removed_tasks, replaced_tasks, label_to_taskid): """ Return the subgraph of target_task_graph consisting only of non-optimized tasks and edges between them. To avoid losing track of taskIds for tasks optimized away, this method simultaneously substitutes real taskIds for task labels in the graph, and populates each task definition's `dependencies` key with the appropriate taskIds. Task references are resolved in the process. """ # check for any dependency edges from included to removed tasks bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges if l not in removed_tasks and r in removed_tasks] if bad_edges: probs = ', '.join( '{} depends on {} as {} but it has been removed'.format(l, r, n) for l, r, n in bad_edges) raise Exception("Optimization error: " + probs) # fill in label_to_taskid for anything not removed or replaced assert replaced_tasks <= set(label_to_taskid) for label in sorted(target_task_graph.graph.nodes - removed_tasks - set(label_to_taskid)): label_to_taskid[label] = slugid() # resolve labels to taskIds and populate task['dependencies'] tasks_by_taskid = {} named_links_dict = target_task_graph.graph.named_links_dict() omit = removed_tasks | replaced_tasks for label, task in target_task_graph.tasks.iteritems(): if label in omit: continue task.task_id = label_to_taskid[label] named_task_dependencies = { name: label_to_taskid[label] for name, label in named_links_dict.get(label, {}).iteritems() } task.task = resolve_task_references(task.label, task.task, named_task_dependencies) deps = task.task.setdefault('dependencies', []) deps.extend(sorted(named_task_dependencies.itervalues())) tasks_by_taskid[task.task_id] = task # resolve edges to taskIds edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right), name) for (left, right, name) in target_task_graph.graph.edges) # ..and drop edges that are no longer entirely in the task graph # (note that this omits edges to replaced tasks, but they are still in task.dependnecies) edges_by_taskid = set( (left, right, name) for (left, right, name) in edges_by_taskid if left in tasks_by_taskid and right in tasks_by_taskid) return TaskGraph(tasks_by_taskid, Graph(set(tasks_by_taskid), edges_by_taskid))
def mochitest_retrigger_action(parameters, input, task_group_id, task_id, task): decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(parameters) pre_task = full_task_graph.tasks[task['metadata']['name']] # fix up the task's dependencies, similar to how optimization would # have done in the decision dependencies = {name: label_to_taskid[label] for name, label in pre_task.dependencies.iteritems()} new_task_definition = resolve_task_references(pre_task.label, pre_task.task, dependencies) new_task_definition.setdefault('dependencies', []).extend(dependencies.itervalues()) # don't want to run mozharness tests, want a custom mach command instead new_task_definition['payload']['command'] += ['--no-run-tests'] custom_mach_command = [task['tags']['test-type']] # mochitests may specify a flavor if new_task_definition['payload']['env'].get('MOCHITEST_FLAVOR'): custom_mach_command += [ '--keep-open=false', '-f', new_task_definition['payload']['env']['MOCHITEST_FLAVOR'] ] enable_e10s = json.loads(new_task_definition['payload']['env'].get( 'ENABLE_E10S', 'true')) if not enable_e10s: custom_mach_command += ['--disable-e10s'] custom_mach_command += ['--log-tbpl=-', '--log-tbpl-level={}'.format(input.get('logLevel', 'debug'))] if input.get('runUntilFail'): custom_mach_command += ['--run-until-failure'] if input.get('repeat'): custom_mach_command += ['--repeat', str(input.get('repeat', 30))] # add any custom gecko preferences for (key, val) in input.get('preferences', {}).iteritems(): custom_mach_command += ['--setpref', '{}={}'.format(key, val)] custom_mach_command += [input['path']] new_task_definition['payload']['env']['CUSTOM_MACH_COMMAND'] = ' '.join( custom_mach_command) # update environment new_task_definition['payload']['env'].update(input.get('environment', {})) # tweak the treeherder symbol new_task_definition['extra']['treeherder']['symbol'] += '-custom' logging.info("New task definition: %s", new_task_definition) # actually create the new task new_task_id = slugid() create_task_from_def(new_task_id, new_task_definition, parameters['level'])
def get_subgraph(target_task_graph, removed_tasks, replaced_tasks, label_to_taskid): """ Return the subgraph of target_task_graph consisting only of non-optimized tasks and edges between them. To avoid losing track of taskIds for tasks optimized away, this method simultaneously substitutes real taskIds for task labels in the graph, and populates each task definition's `dependencies` key with the appropriate taskIds. Task references are resolved in the process. """ # check for any dependency edges from included to removed tasks bad_edges = [(l, r, n) for l, r, n in target_task_graph.graph.edges if l not in removed_tasks and r in removed_tasks] if bad_edges: probs = ', '.join('{} depends on {} as {} but it has been removed'.format(l, r, n) for l, r, n in bad_edges) raise Exception("Optimization error: " + probs) # fill in label_to_taskid for anything not removed or replaced assert replaced_tasks <= set(label_to_taskid) for label in sorted(target_task_graph.graph.nodes - removed_tasks - set(label_to_taskid)): label_to_taskid[label] = slugid() # resolve labels to taskIds and populate task['dependencies'] tasks_by_taskid = {} named_links_dict = target_task_graph.graph.named_links_dict() omit = removed_tasks | replaced_tasks for label, task in target_task_graph.tasks.iteritems(): if label in omit: continue task.task_id = label_to_taskid[label] named_task_dependencies = { name: label_to_taskid[label] for name, label in named_links_dict.get(label, {}).iteritems()} task.task = resolve_task_references(task.label, task.task, named_task_dependencies) deps = task.task.setdefault('dependencies', []) deps.extend(sorted(named_task_dependencies.itervalues())) tasks_by_taskid[task.task_id] = task # resolve edges to taskIds edges_by_taskid = ( (label_to_taskid.get(left), label_to_taskid.get(right), name) for (left, right, name) in target_task_graph.graph.edges ) # ..and drop edges that are no longer entirely in the task graph # (note that this omits edges to replaced tasks, but they are still in task.dependnecies) edges_by_taskid = set( (left, right, name) for (left, right, name) in edges_by_taskid if left in tasks_by_taskid and right in tasks_by_taskid ) return TaskGraph( tasks_by_taskid, Graph(set(tasks_by_taskid), edges_by_taskid))
def retrigger_decision_action(parameters, graph_config, input, task_group_id, task_id): """For a single task, we try to just run exactly the same task once more. It's quite possible that we don't have the scopes to do so (especially for an action), but this is best-effort.""" # make all of the timestamps relative; they will then be turned back into # absolute timestamps relative to the current time. task = taskcluster.get_task_definition(task_id) task = relativize_datestamps(task) create_task_from_def(slugid(), task, parameters['level'])
def __call__(self, name): ''' So this object can easily be passed to mustache we allow it to be called directly... ''' if name in self._names: return self._names[name] self._names[name] = slugid() return self._names[name]
def __call__(self, name): ''' So this object can easily be passed to mustache we allow it to be called directly... ''' if name in self._names: return self._names[name]; self._names[name] = slugid() return self._names[name]
def create_isolate_failure_tasks(task_definition, failures, level): """ Create tasks to re-run the original tasks plus tasks to test each failing test directory and individual path. """ # redo the original task... logger.info("create_isolate_failure_tasks: task_definition: {}," "failures: {}".format(task_definition, failures)) new_task_id = slugid() new_task_definition = copy.deepcopy(task_definition) th_dict = new_task_definition['extra']['treeherder'] th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I' th_dict['tier'] = 3 logger.info('Cloning original task') create_task_from_def(new_task_id, new_task_definition, level) for failure_group in failures: failure_group_suffix = '-id' if failure_group == 'dirs' else '-it' for failure_path in failures[failure_group]: new_task_id = slugid() new_task_definition = copy.deepcopy(task_definition) th_dict = new_task_definition['extra']['treeherder'] th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I' th_dict['symbol'] = th_dict['symbol'] + failure_group_suffix th_dict['tier'] = 3 suite = new_task_definition['extra']['suite'] if '-chunked' in suite: suite = suite[:suite.index('-chunked')] if '-coverage' in suite: suite = suite[:suite.index('-coverage')] env_dict = new_task_definition['payload']['env'] if 'MOZHARNESS_TEST_PATHS' not in env_dict: env_dict['MOZHARNESS_TEST_PATHS'] = {} if 'windows' in th_dict['machine']['platform']: failure_path = '\\'.join(failure_path.split('/')) env_dict['MOZHARNESS_TEST_PATHS'] = json.dumps({suite: [failure_path]}) logger.info('Creating task for {}'.format(failure_path)) create_task_from_def(new_task_id, new_task_definition, level)
def build_callback_action_task(parameters): if not available(parameters): return None match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository']) if not match: raise Exception('Unrecognized head_repository') repo_scope = 'assume:repo:{}/{}:*'.format(match.group(1), match.group(2)) task_group_id = os.environ.get('TASK_ID', slugid()) template = os.path.join(GECKO, '.taskcluster.yml') with open(template, 'r') as f: taskcluster_yml = yaml.safe_load(f) if taskcluster_yml['version'] != 1: raise Exception( 'actions.json must be updated to work with .taskcluster.yml' ) if not isinstance(taskcluster_yml['tasks'], list): raise Exception( '.taskcluster.yml "tasks" must be a list for action tasks' ) return { '$let': { 'tasks_for': 'action', 'repository': { 'url': parameters['head_repository'], 'project': parameters['project'], 'level': parameters['level'], }, 'push': { 'owner': '*****@*****.**', 'pushlog_id': parameters['pushlog_id'], 'revision': parameters['head_rev'], }, 'action': { 'name': name, 'title': title, 'description': description, 'taskGroupId': task_group_id, 'repo_scope': repo_scope, 'cb_name': cb.__name__, 'symbol': symbol, }, }, 'in': taskcluster_yml['tasks'][0] }
def create_task_from_def(task_def, level): """Create a new task from a definition rather than from a label that is already in the full-task-graph. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Use this for entirely new tasks or ones that change internals of the task. It is useful if you want to "edit" the full_task_graph and then hand it to this function. No dependencies will be scheduled. You must handle this yourself. Seeing how create_tasks handles it might prove helpful.""" task_def["schedulerId"] = "gecko-level-{}".format(level) label = task_def["metadata"]["name"] task_id = slugid().decode("ascii") session = get_session() create.create_task(session, task_id, label, task_def)
def derive_misc_task(task, purpose, image, taskgraph, label_to_taskid): """Create the shell of a task that depends on `task` and on the given docker image.""" label = '{}-{}'.format(purpose, task.label) # this is why all docker image tasks are included in the target task graph: we # need to find them in label_to_taskid, if if nothing else required them image_taskid = label_to_taskid['build-docker-image-' + image] task_def = { 'provisionerId': 'aws-provisioner-v1', 'workerType': 'gecko-misc', 'dependencies': [task.task_id, image_taskid], 'created': {'relative-datestamp': '0 seconds'}, 'deadline': task.task['deadline'], # no point existing past the parent task's deadline 'expires': task.task['deadline'], 'metadata': { 'name': label, 'description': '{} for {}'.format(purpose, task.task['metadata']['description']), 'owner': task.task['metadata']['owner'], 'source': task.task['metadata']['source'], }, 'scopes': [], 'payload': { 'image': { 'path': 'public/image.tar.zst', 'taskId': image_taskid, 'type': 'task-image', }, 'features': { 'taskclusterProxy': True, }, 'maxRunTime': 600, } } # only include the docker-image dependency here if it is actually in the # taskgraph (has not been optimized). It is included in # task_def['dependencies'] unconditionally. dependencies = {'parent': task.task_id} if image_taskid in taskgraph.tasks: dependencies['docker-image'] = image_taskid task = Task(kind='misc', label=label, attributes={}, task=task_def, dependencies=dependencies) task.task_id = slugid() return task
def backfill_action(parameters, input, task_group_id, task_id, task): label = task['metadata']['name'] pushes = [] depth = input.get('depth', 5) end_id = int(parameters['pushlog_id']) - 1 while True: start_id = max(end_id - depth, 0) pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id) r = requests.get(pushlog_url) r.raise_for_status() pushes = pushes + r.json()['pushes'].keys() if len(pushes) >= depth: break end_id = start_id - 1 start_id -= depth if start_id < 0: break pushes = sorted(pushes)[-depth:] for push in pushes: full_task_graph = get_artifact_from_index( INDEX_TMPL.format(parameters['project'], push), 'public/full-task-graph.json') _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact_from_index( INDEX_TMPL.format(parameters['project'], push), 'public/label-to-taskid.json') if label in full_task_graph.tasks.keys(): task = full_task_graph.tasks[label] dependencies = { name: label_to_taskid[label] for name, label in task.dependencies.iteritems() } task_def = resolve_task_references(task.label, task.task, dependencies) task_def.setdefault('dependencies', []).extend(dependencies.itervalues()) create_task(slugid(), task_def, parameters['level']) else: logging.info('Could not find {} on {}. Skipping.'.format( label, push))
def build_callback_action_task(parameters): if not available(parameters): return None match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository']) if not match: raise Exception('Unrecognized head_repository') repo_scope = 'assume:repo:{}/{}:*'.format( match.group(1), match.group(2)) task_group_id = os.environ.get('TASK_ID', slugid()) template = os.path.join(GECKO, '.taskcluster.yml') with open(template, 'r') as f: taskcluster_yml = yaml.safe_load(f) if taskcluster_yml['version'] != 1: raise Exception('actions.json must be updated to work with .taskcluster.yml') if not isinstance(taskcluster_yml['tasks'], list): raise Exception('.taskcluster.yml "tasks" must be a list for action tasks') return { '$let': { 'tasks_for': 'action', 'repository': { 'url': parameters['head_repository'], 'project': parameters['project'], 'level': parameters['level'], }, 'push': { 'owner': '*****@*****.**', 'pushlog_id': parameters['pushlog_id'], 'revision': parameters['head_rev'], }, 'action': { 'name': name, 'title': title, 'description': description, 'taskGroupId': task_group_id, 'repo_scope': repo_scope, 'cb_name': cb.__name__, 'symbol': symbol, }, }, 'in': taskcluster_yml['tasks'][0] }
def create_tasks(taskgraph, label_to_taskid): # TODO: use the taskGroupId of the decision task task_group_id = slugid() taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()} session = requests.Session() decision_task_id = os.environ.get('TASK_ID') with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e: fs = {} # We can't submit a task until its dependencies have been submitted. # So our strategy is to walk the graph and submit tasks once all # their dependencies have been submitted. # # Using visit_postorder() here isn't the most efficient: we'll # block waiting for dependencies of task N to submit even though # dependencies for task N+1 may be finished. If we need to optimize # this further, we can build a graph of task dependencies and walk # that. for task_id in taskgraph.graph.visit_postorder(): task_def = taskgraph.tasks[task_id].task # if this task has no dependencies, make it depend on this decision # task so that it does not start immediately; and so that if this loop # fails halfway through, none of the already-created tasks run. if decision_task_id and not task_def.get('dependencies'): task_def['dependencies'] = [decision_task_id] task_def['taskGroupId'] = task_group_id # Wait for dependencies before submitting this. deps_fs = [ fs[dep] for dep in task_def['dependencies'] if dep in fs ] for f in futures.as_completed(deps_fs): f.result() fs[task_id] = e.submit(_create_task, session, task_id, taskid_to_label[task_id], task_def) # Wait for all futures to complete. for f in futures.as_completed(fs.values()): f.result()
def create_tasks(taskgraph, label_to_taskid): # TODO: use the taskGroupId of the decision task task_group_id = slugid() taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()} session = requests.Session() decision_task_id = os.environ.get('TASK_ID') with futures.ThreadPoolExecutor(requests.adapters.DEFAULT_POOLSIZE) as e: fs = {} # We can't submit a task until its dependencies have been submitted. # So our strategy is to walk the graph and submit tasks once all # their dependencies have been submitted. # # Using visit_postorder() here isn't the most efficient: we'll # block waiting for dependencies of task N to submit even though # dependencies for task N+1 may be finished. If we need to optimize # this further, we can build a graph of task dependencies and walk # that. for task_id in taskgraph.graph.visit_postorder(): task_def = taskgraph.tasks[task_id].task # if this task has no dependencies, make it depend on this decision # task so that it does not start immediately; and so that if this loop # fails halfway through, none of the already-created tasks run. if decision_task_id and not task_def.get('dependencies'): task_def['dependencies'] = [decision_task_id] task_def['taskGroupId'] = task_group_id # Wait for dependencies before submitting this. deps_fs = [fs[dep] for dep in task_def.get('dependencies', []) if dep in fs] for f in futures.as_completed(deps_fs): f.result() fs[task_id] = e.submit(_create_task, session, task_id, taskid_to_label[task_id], task_def) # Wait for all futures to complete. for f in futures.as_completed(fs.values()): f.result()
def create_tasks(taskgraph): # TODO: use the taskGroupId of the decision task task_group_id = slugid() label_to_taskid = collections.defaultdict(slugid) session = requests.Session() for label in taskgraph.graph.visit_postorder(): task = taskgraph.tasks[label] deps_by_name = { n: label_to_taskid[r] for (l, r, n) in taskgraph.graph.edges if l == label} task_def = task.kind.get_task_definition(task, deps_by_name) task_def['taskGroupId'] = task_group_id task_def['dependencies'] = deps_by_name.values() task_def['requires'] = 'all-completed' _create_task(session, label_to_taskid[label], label, task_def)
def create_tasks(taskgraph): # TODO: use the taskGroupId of the decision task task_group_id = slugid() label_to_taskid = collections.defaultdict(slugid) session = requests.Session() for label in taskgraph.graph.visit_postorder(): task = taskgraph.tasks[label] deps_by_name = { n: label_to_taskid[r] for (l, r, n) in taskgraph.graph.edges if l == label } task_def = task.kind.get_task_definition(task, deps_by_name) task_def['taskGroupId'] = task_group_id task_def['dependencies'] = deps_by_name.values() task_def['requires'] = 'all-completed' _create_task(session, label_to_taskid[label], label, task_def)
def task_id_for_image(seen_images, project, name): if name in seen_images: return seen_images[name]['taskId'] context_path = os.path.join('testing', 'docker', name) context_hash = generate_context_hash(context_path) task_id = get_task_id_for_namespace(project, name, context_hash) if task_id: seen_images[name] = {'taskId': task_id} return task_id task_id = slugid() seen_images[name] = { 'taskId': task_id, 'path': context_path, 'hash': context_hash } return task_id
def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid): """ Return the subgraph of annotated_task_graph consisting only of non-optimized tasks and edges between them. To avoid losing track of taskIds for tasks optimized away, this method simultaneously substitutes real taskIds for task labels in the graph, and populates each task definition's `dependencies` key with the appropriate taskIds. Task references are resolved in the process. """ # resolve labels to taskIds and populate task['dependencies'] tasks_by_taskid = {} for label in annotated_task_graph.graph.visit_postorder(): task = annotated_task_graph.tasks[label] if task.optimized: continue task.task_id = label_to_taskid[label] = slugid() named_task_dependencies = { name: label_to_taskid[label] for name, label in named_links_dict.get(label, {}).iteritems()} task.task = resolve_task_references(task.label, task.task, named_task_dependencies) task.task.setdefault('dependencies', []).extend(named_task_dependencies.itervalues()) tasks_by_taskid[task.task_id] = task # resolve edges to taskIds edges_by_taskid = ( (label_to_taskid.get(left), label_to_taskid.get(right), name) for (left, right, name) in annotated_task_graph.graph.edges ) # ..and drop edges that are no longer in the task graph edges_by_taskid = set( (left, right, name) for (left, right, name) in edges_by_taskid if left in tasks_by_taskid and right in tasks_by_taskid ) return TaskGraph( tasks_by_taskid, Graph(set(tasks_by_taskid), edges_by_taskid))
def run_missing_tests(parameters, input, task_group_id, task_id, task): decision_task_id = find_decision_task(parameters) full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) target_tasks = get_artifact(decision_task_id, "public/target-tasks.json") label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # The idea here is to schedule all tasks of the `test` kind that were # targetted but did not appear in the final task-graph -- those were the # optimized tasks. to_run = [] already_run = 0 for label in target_tasks: task = full_task_graph.tasks[label] if task.kind != 'test': continue # not a test if label in label_to_taskid: already_run += 1 continue to_run.append(task) for task in to_run: # fix up the task's dependencies, similar to how optimization would # have done in the decision dependencies = { name: label_to_taskid[label] for name, label in task.dependencies.iteritems() } task_def = resolve_task_references(task.label, task.task, dependencies) task_def.setdefault('dependencies', []).extend(dependencies.itervalues()) create_task(slugid(), task_def, parameters['level']) logger.info( 'Out of {} test tasks, {} already existed and the action created {}'. format(already_run + len(to_run), already_run, len(to_run)))
def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid): """ Return the subgraph of annotated_task_graph consisting only of non-optimized tasks and edges between them. To avoid losing track of taskIds for tasks optimized away, this method simultaneously substitutes real taskIds for task labels in the graph, and populates each task definition's `dependencies` key with the appropriate taskIds. Task references are resolved in the process. """ # resolve labels to taskIds and populate task['dependencies'] tasks_by_taskid = {} for label in annotated_task_graph.graph.visit_postorder(): task = annotated_task_graph.tasks[label] if task.optimized: continue task.task_id = label_to_taskid[label] = slugid() named_task_dependencies = { name: label_to_taskid[label] for name, label in named_links_dict.get(label, {}).iteritems() } task.task = resolve_task_references(task.label, task.task, named_task_dependencies) task.task.setdefault('dependencies', []).extend(named_task_dependencies.itervalues()) tasks_by_taskid[task.task_id] = task # resolve edges to taskIds edges_by_taskid = ((label_to_taskid.get(left), label_to_taskid.get(right), name) for (left, right, name) in annotated_task_graph.graph.edges) # ..and drop edges that are no longer in the task graph edges_by_taskid = set( (left, right, name) for (left, right, name) in edges_by_taskid if left in tasks_by_taskid and right in tasks_by_taskid) return TaskGraph(tasks_by_taskid, Graph(set(tasks_by_taskid), edges_by_taskid))
def schedule_tasks(f=None): to_remove = set() for task_id in tasklist: task_def = taskgraph.tasks[task_id].task # If we haven't finished submitting all our dependencies yet, # come back to this later. # Some dependencies aren't in our graph, so make sure to filter # those out deps = set(task_def.get('dependencies', [])) & alltasks if any((d not in fs or not fs[d].done()) for d in deps): continue fs[task_id] = e.submit(create_task, session, task_id, taskid_to_label[task_id], task_def) to_remove.add(task_id) # Schedule tasks as many times as task_duplicates indicates attributes = taskgraph.tasks[task_id].attributes for i in range(1, attributes.get('task_duplicates', 1)): # We use slugid() since we want a distinct task id fs[task_id] = e.submit(create_task, session, slugid(), taskid_to_label[task_id], task_def) tasklist.difference_update(to_remove)
def schedule_tasks(): # bail out early if any futures have failed if any(f.done() and f.exception() for f in fs.values()): return to_remove = set() new = set() def submit(task_id, label, task_def): fut = e.submit(create_task, session, task_id, label, task_def) new.add(fut) fs[task_id] = fut for task_id in tasklist: task_def = taskgraph.tasks[task_id].task # If we haven't finished submitting all our dependencies yet, # come back to this later. # Some dependencies aren't in our graph, so make sure to filter # those out deps = set(task_def.get('dependencies', [])) & alltasks if any((d not in fs or not fs[d].done()) for d in deps): continue submit(task_id, taskid_to_label[task_id], task_def) to_remove.add(task_id) # Schedule tasks as many times as task_duplicates indicates attributes = taskgraph.tasks[task_id].attributes for i in range(1, attributes.get('task_duplicates', 1)): # We use slugid() since we want a distinct task id submit(slugid().decode("ascii"), taskid_to_label[task_id], task_def) tasklist.difference_update(to_remove) # as each of those futures complete, try to schedule more tasks for f in futures.as_completed(new): schedule_tasks()
def build_callback_action_task(parameters): if not available(parameters): return None match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters['head_repository']) if not match: raise Exception('Unrecognized head_repository') repo_scope = 'assume:repo:{}/{}:*'.format(match.group(1), match.group(2)) task_group_id = os.environ.get('TASK_ID', slugid()) return { 'created': { '$fromNow': '' }, 'deadline': { '$fromNow': '12 hours' }, 'expires': { '$fromNow': '14 days' }, 'metadata': { 'owner': '*****@*****.**', 'source': '{}raw-file/{}/{}'.format( parameters['head_repository'], parameters['head_rev'], source_path, ), 'name': 'Action: {}'.format(title), 'description': 'Task executing callback for action.\n\n---\n' + description, }, 'workerType': 'gecko-{}-decision'.format(parameters['level']), 'provisionerId': 'aws-provisioner-v1', 'taskGroupId': task_group_id, 'schedulerId': 'gecko-level-{}'.format(parameters['level']), 'scopes': [ repo_scope, ], 'tags': { 'createdForUser': parameters['owner'], 'kind': 'action-callback', }, 'routes': [ 'tc-treeherder.v2.{}.{}.{}'.format( parameters['project'], parameters['head_rev'], parameters['pushlog_id']), 'tc-treeherder-stage.v2.{}.{}.{}'.format( parameters['project'], parameters['head_rev'], parameters['pushlog_id']), ], 'payload': { 'env': { 'GECKO_BASE_REPOSITORY': 'https://hg.mozilla.org/mozilla-unified', 'GECKO_HEAD_REPOSITORY': parameters['head_repository'], 'GECKO_HEAD_REF': parameters['head_ref'], 'GECKO_HEAD_REV': parameters['head_rev'], 'HG_STORE_PATH': '/home/worker/checkouts/hg-store', 'ACTION_TASK_GROUP_ID': task_group_id, 'ACTION_TASK_ID': { '$json': { '$eval': 'taskId' } }, 'ACTION_TASK': { '$json': { '$eval': 'task' } }, 'ACTION_INPUT': { '$json': { '$eval': 'input' } }, 'ACTION_CALLBACK': cb.__name__, 'ACTION_PARAMETERS': { '$json': { '$eval': 'parameters' } }, }, 'cache': { 'level-{}-checkouts'.format(parameters['level']): '/home/worker/checkouts', }, 'features': { 'taskclusterProxy': True, 'chainOfTrust': True, }, 'image': docker_image('decision'), 'maxRunTime': 1800, 'command': [ '/home/worker/bin/run-task', '--vcs-checkout=/home/worker/checkouts/gecko', '--', 'bash', '-cx', """\ cd /home/worker/checkouts/gecko && ln -s /home/worker/artifacts artifacts && ./mach --log-no-times taskgraph action-callback""", ], }, 'extra': { 'treeherder': { 'groupName': 'action-callback', 'groupSymbol': 'AC', 'symbol': symbol, }, }, }
def action_builder(parameters, graph_config): if not available(parameters): return None actionPerm = 'generic' if generic else cb_name # gather up the common decision-task-supplied data for this action repo_param = '{}head_repository'.format( graph_config['project-repo-param-prefix']) repository = { 'url': parameters[repo_param], 'project': parameters['project'], 'level': parameters['level'], } revision = parameters['{}head_rev'.format( graph_config['project-repo-param-prefix'])] push = { 'owner': '*****@*****.**', 'pushlog_id': parameters['pushlog_id'], 'revision': revision, } task_group_id = os.environ.get('TASK_ID', slugid()) match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters[repo_param]) if not match: raise Exception('Unrecognized {}'.format(repo_param)) action = { 'name': name, 'title': title, 'description': description, # target taskGroupId (the task group this decision task is creating) 'taskGroupId': task_group_id, 'cb_name': cb_name, 'symbol': symbol, } rv = { 'name': name, 'title': title, 'description': description, 'context': context(parameters), } if schema: rv['schema'] = schema( graph_config=graph_config) if callable(schema) else schema # for kind=task, we embed the task from .taskcluster.yml in the action, with # suitable context if kind == 'task': # tasks get all of the scopes the original push did, yuck; this is not # done with kind = hook. repo_scope = 'assume:repo:{}/{}:branch:default'.format( match.group(1), match.group(2)) action['repo_scope'] = repo_scope taskcluster_yml = read_taskcluster_yml( graph_config.taskcluster_yml) if taskcluster_yml['version'] != 1: raise Exception( 'actions.json must be updated to work with .taskcluster.yml' ) tasks = taskcluster_yml['tasks'] if not isinstance(tasks, list): raise Exception( '.taskcluster.yml "tasks" must be a list for action tasks' ) rv.update({ 'kind': 'task', 'task': { '$let': { 'tasks_for': 'action', 'repository': repository, 'push': push, 'action': action, }, 'in': tasks[0], }, }) # for kind=hook elif kind == 'hook': trustDomain = graph_config['trust-domain'] level = parameters['level'] tcyml_hash = hash_taskcluster_yml(graph_config.taskcluster_yml) # the tcyml_hash is prefixed with `/` in the hookId, so users will be granted # hooks:trigger-hook:project-gecko/in-tree-action-3-myaction/*; if another # action was named `myaction/release`, then the `*` in the scope would also # match that action. To prevent such an accident, we prohibit `/` in hook # names. if '/' in actionPerm: raise Exception( '`/` is not allowed in action names; use `-`') rv.update({ 'kind': 'hook', 'hookGroupId': 'project-{}'.format(trustDomain), 'hookId': 'in-tree-action-{}-{}/{}'.format(level, actionPerm, tcyml_hash), 'hookPayload': { # provide the decision-task parameters as context for triggerHook "decision": { 'action': action, 'repository': repository, 'push': push, # parameters is long, so fetch it from the actions.json variables 'parameters': { '$eval': 'parameters' }, }, # and pass everything else through from our own context "user": { 'input': { '$eval': 'input' }, 'taskId': { '$eval': 'taskId' }, # target taskId (or null) 'taskGroupId': { '$eval': 'taskGroupId' }, # target task group } }, 'extra': { 'actionPerm': actionPerm, }, }) return rv
def build_callback_action_task(parameters, graph_config): if not available(parameters): return None repo_param = '{}head_repository'.format( graph_config['project-repo-param-prefix']) revision = parameters['{}head_rev'.format( graph_config['project-repo-param-prefix'])] match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', parameters[repo_param]) if not match: raise Exception('Unrecognized {}'.format(repo_param)) repo_scope = 'assume:repo:{}/{}:branch:default'.format( match.group(1), match.group(2)) task_group_id = os.environ.get('TASK_ID', slugid()) # FIXME: https://bugzilla.mozilla.org/show_bug.cgi?id=1454034 # trust-domain works, but isn't semantically correct here. if graph_config['trust-domain'] == 'comm': template = os.path.join(GECKO, 'comm', '.taskcluster.yml') else: template = os.path.join(GECKO, '.taskcluster.yml') with open(template, 'r') as f: taskcluster_yml = yaml.safe_load(f) if taskcluster_yml['version'] != 1: raise Exception( 'actions.json must be updated to work with .taskcluster.yml' ) if not isinstance(taskcluster_yml['tasks'], list): raise Exception( '.taskcluster.yml "tasks" must be a list for action tasks' ) return { '$let': { 'tasks_for': 'action', 'repository': { 'url': parameters[repo_param], 'project': parameters['project'], 'level': parameters['level'], }, 'push': { 'owner': '*****@*****.**', 'pushlog_id': parameters['pushlog_id'], 'revision': revision, }, 'action': { 'name': name, 'title': title, 'description': description, 'taskGroupId': task_group_id, 'repo_scope': repo_scope, 'cb_name': cb.__name__, 'symbol': symbol, }, }, 'in': taskcluster_yml['tasks'][0] }
def create_tasks(taskgraph, label_to_taskid, params, decision_task_id=None): taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()} session = requests.Session() # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase # that limit. Connections are established as needed, so using a large value # should not negatively impact performance. http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY, pool_maxsize=CONCURRENCY) session.mount('https://', http_adapter) session.mount('http://', http_adapter) decision_task_id = decision_task_id or os.environ.get('TASK_ID') # when running as an actual decision task, we use the decision task's # taskId as the taskGroupId. The process that created the decision task # helpfully placed it in this same taskGroup. If there is no $TASK_ID, # fall back to a slugid task_group_id = decision_task_id or slugid() scheduler_id = 'gecko-level-{}'.format(params['level']) # Add the taskGroupId, schedulerId and optionally the decision task # dependency for task_id in taskgraph.graph.nodes: task_def = taskgraph.tasks[task_id].task # if this task has no dependencies *within* this taskgraph, make it # depend on this decision task. If it has another dependency within # the taskgraph, then it already implicitly depends on the decision # task. The result is that tasks do not start immediately. if this # loop fails halfway through, none of the already-created tasks run. if decision_task_id: if not any(t in taskgraph.tasks for t in task_def.get('dependencies', [])): task_def.setdefault('dependencies', []).append(decision_task_id) task_def['taskGroupId'] = task_group_id task_def['schedulerId'] = scheduler_id with futures.ThreadPoolExecutor(CONCURRENCY) as e: fs = {} # We can't submit a task until its dependencies have been submitted. # So our strategy is to walk the graph and submit tasks once all # their dependencies have been submitted. tasklist = set(taskgraph.graph.visit_postorder()) alltasks = tasklist.copy() def schedule_tasks(f=None): to_remove = set() for task_id in tasklist: task_def = taskgraph.tasks[task_id].task # If we haven't finished submitting all our dependencies yet, # come back to this later. # Some dependencies aren't in our graph, so make sure to filter # those out deps = set(task_def.get('dependencies', [])) & alltasks if any((d not in fs or not fs[d].done()) for d in deps): continue fs[task_id] = e.submit(create_task, session, task_id, taskid_to_label[task_id], task_def) to_remove.add(task_id) # Schedule tasks as many times as task_duplicates indicates attributes = taskgraph.tasks[task_id].attributes for i in range(1, attributes.get('task_duplicates', 1)): # We use slugid() since we want a distinct task id fs[task_id] = e.submit(create_task, session, slugid(), taskid_to_label[task_id], task_def) tasklist.difference_update(to_remove) schedule_tasks() while tasklist: for f in futures.as_completed(fs.values()): f.result() schedule_tasks()
def mklabel(): return TASKID_PLACEHOLDER.format(slugid())
def mklabel(): return slugid()
def add_code_review_task(taskgraph, label_to_taskid, parameters, graph_config): logger.debug("Morphing: adding index tasks") review_config = parameters.get("code-review") if not review_config: return taskgraph, label_to_taskid code_review_tasks = {} for label, task in taskgraph.tasks.items(): if task.attributes.get("code-review"): code_review_tasks[task.label] = task.task_id if code_review_tasks: code_review_task_def = { "provisionerId": "built-in", "workerType": "succeed", "dependencies": sorted(code_review_tasks.values()), # This option permits to run the task # regardless of the dependencies tasks exit status # as we are interested in the task failures "requires": "all-resolved", "created": { "relative-datestamp": "0 seconds" }, "deadline": { "relative-datestamp": "1 day" }, # no point existing past the parent task's deadline "expires": { "relative-datestamp": "1 day" }, "metadata": { "name": "code-review", "description": "List all issues found in static analysis and linting tasks", "owner": parameters["owner"], "source": _get_morph_url(), }, "scopes": [], "payload": {}, "routes": ["project.relman.codereview.v1.try_ending"], "extra": { "code-review": { "phabricator-build-target": review_config["phabricator-build-target"], "repository": parameters["head_repository"], "revision": parameters["head_rev"], } }, } task = Task( kind="misc", label="code-review", attributes={}, task=code_review_task_def, dependencies=code_review_tasks, ) task.task_id = slugid() taskgraph, label_to_taskid = amend_taskgraph(taskgraph, label_to_taskid, [task]) logger.info("Added code review task.") return taskgraph, label_to_taskid
def create_graph(self, **params): from taskcluster_graph.commit_parser import parse_commit from slugid import nice as slugid from taskcluster_graph.from_now import json_time_from_now, current_json_time from taskcluster_graph.templates import Templates import taskcluster_graph.build_task project = params["project"] message = params.get("message", "") if project == "try" else DEFAULT_TRY # Message would only be blank when not created from decision task if project == "try" and not message: sys.stderr.write( "Must supply commit message when creating try graph. " "Example: --message='try: -b do -p all -u all'" ) sys.exit(1) templates = Templates(ROOT) job_path = os.path.join(ROOT, "tasks", "branches", project, "job_flags.yml") job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph = parse_commit(message, jobs) cmdline_interactive = params.get("interactive", False) # Template parameters used when expanding the graph parameters = dict( gaia_info().items() + { "index": "index", "project": project, "pushlog_id": params.get("pushlog_id", 0), "docker_image": docker_image, "base_repository": params["base_repository"] or params["head_repository"], "head_repository": params["head_repository"], "head_ref": params["head_ref"] or params["head_rev"], "head_rev": params["head_rev"], "owner": params["owner"], "from_now": json_time_from_now, "now": current_json_time(), "revision_hash": params["revision_hash"], }.items() ) treeherder_route = "{}.{}".format(params["project"], params.get("revision_hash", "")) routes_file = os.path.join(ROOT, "routes.json") with open(routes_file) as f: contents = json.load(f) json_routes = contents["routes"] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = {"tasks": [], "scopes": []} if params["revision_hash"]: for env in TREEHERDER_ROUTES: graph["scopes"].append("queue:route:{}.{}".format(TREEHERDER_ROUTES[env], treeherder_route)) graph["metadata"] = { "source": "http://todo.com/what/goes/here", "owner": params["owner"], # TODO: Add full mach commands to this example? "description": "Task graph generated via ./mach taskcluster-graph", "name": "task graph local", } all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = dict(parameters) build_parameters["build_slugid"] = slugid() build_task = templates.load(build["task"], build_parameters) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) if params["revision_hash"]: decorate_task_treeherder_routes(build_task["task"], treeherder_route) decorate_task_json_routes(build, build_task["task"], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph["tasks"].append(build_task) test_packages_url, tests_url, mozharness_url = None, None, None if "test_packages" in build_task["task"]["extra"]["locations"]: test_packages_url = ARTIFACT_URL.format( build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["test_packages"] ) if "tests" in build_task["task"]["extra"]["locations"]: tests_url = ARTIFACT_URL.format( build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["tests"] ) if "mozharness" in build_task["task"]["extra"]["locations"]: mozharness_url = ARTIFACT_URL.format( build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["mozharness"] ) build_url = ARTIFACT_URL.format( build_parameters["build_slugid"], build_task["task"]["extra"]["locations"]["build"] ) build_parameters["build_url"] = build_url # img_url is only necessary for device builds img_url = ARTIFACT_URL.format( build_parameters["build_slugid"], build_task["task"]["extra"]["locations"].get("img", "") ) build_parameters["img_url"] = img_url define_task = DEFINE_TASK.format(build_task["task"]["workerType"]) for route in build_task["task"].get("routes", []): if route.startswith("index.gecko.v2") and route in all_routes: raise Exception( "Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % (route, build_task["task"]["metadata"]["name"], all_routes[route]) ) all_routes[route] = build_task["task"]["metadata"]["name"] graph["scopes"].append(define_task) graph["scopes"].extend(build_task["task"].get("scopes", [])) route_scopes = map(lambda route: "queue:route:" + route, build_task["task"].get("routes", [])) graph["scopes"].extend(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task["task"]["extra"]["treeherder"] if "machine" not in build_treeherder_config: message = "({}), extra.treeherder.machine required for all builds" raise ValueError(message.format(build["task"])) if "build" not in build_treeherder_config: build_treeherder_config["build"] = build_treeherder_config["machine"] if "collection" not in build_treeherder_config: build_treeherder_config["collection"] = {"opt": True} if len(build_treeherder_config["collection"].keys()) != 1: message = "({}), extra.treeherder.collection must contain one type" raise ValueError(message.fomrat(build["task"])) for post_build in build["post-build"]: # copy over the old parameters to update the template post_parameters = copy.copy(build_parameters) post_task = configure_dependent_task( post_build["task"], post_parameters, slugid(), templates, build_treeherder_config ) set_interactive_task(post_task, interactive) graph["tasks"].append(post_task) for test in build["dependents"]: test = test["allowed_build_tasks"][build["task"]] test_parameters = copy.copy(build_parameters) if tests_url: test_parameters["tests_url"] = tests_url if test_packages_url: test_parameters["test_packages_url"] = test_packages_url if mozharness_url: test_parameters["mozharness_url"] = mozharness_url test_definition = templates.load(test["task"], {})["task"] chunk_config = test_definition["extra"]["chunks"] # Allow branch configs to override task level chunking... if "chunks" in test: chunk_config["total"] = test["chunks"] test_parameters["total_chunks"] = chunk_config["total"] for chunk in range(1, chunk_config["total"] + 1): if "only_chunks" in test and chunk not in test["only_chunks"]: continue test_parameters["chunk"] = chunk test_task = configure_dependent_task( test["task"], test_parameters, slugid(), templates, build_treeherder_config ) set_interactive_task(test_task, interactive) if params["revision_hash"]: decorate_task_treeherder_routes(test_task["task"], treeherder_route) graph["tasks"].append(test_task) define_task = DEFINE_TASK.format(test_task["task"]["workerType"]) graph["scopes"].append(define_task) graph["scopes"].extend(test_task["task"].get("scopes", [])) graph["scopes"] = list(set(graph["scopes"])) if params["print_names_only"]: tIDs = defaultdict(list) def print_task(task, indent=0): print("{}- {}".format(" " * indent, task["task"]["metadata"]["name"])) for child in tIDs[task["taskId"]]: print_task(child, indent=indent + 2) # build a dependency map for task in graph["tasks"]: if "requires" in task: for tID in task["requires"]: tIDs[tID].append(task) # recursively print root tasks for task in graph["tasks"]: if "requires" not in task: print_task(task) return # When we are extending the graph remove extra fields... if params["ci"] is True: graph.pop("scopes", None) graph.pop("metadata", None) print(json.dumps(graph, indent=4))
def create_graph(self, **params): from functools import partial from slugid import nice as slugid import taskcluster_graph.transform.routes as routes_transform from taskcluster_graph.commit_parser import parse_commit from taskcluster_graph.image_builder import (docker_image, normalize_image_details, task_id_for_image) from taskcluster_graph.from_now import ( json_time_from_now, current_json_time, ) from taskcluster_graph.templates import Templates import taskcluster_graph.build_task if params['dry_run']: from taskcluster_graph.dry_run import ( json_time_from_now, current_json_time, slugid, ) project = params['project'] message = params.get('message', '') if project == 'try' else DEFAULT_TRY # Message would only be blank when not created from decision task if project == 'try' and not message: sys.stderr.write( "Must supply commit message when creating try graph. " \ "Example: --message='try: -b do -p all -u all'" ) sys.exit(1) templates = Templates(ROOT) job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml') job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph = parse_commit(message, jobs) cmdline_interactive = params.get('interactive', False) # Default to current time if querying the head rev fails pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) pushinfo = query_pushinfo(params['head_repository'], params['head_rev']) if pushinfo: pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) # Template parameters used when expanding the graph seen_images = {} parameters = dict(gaia_info().items() + { 'index': 'index', 'project': project, 'pushlog_id': params.get('pushlog_id', 0), 'docker_image': docker_image, 'task_id_for_image': partial(task_id_for_image, seen_images, project), 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'owner': params['owner'], 'from_now': json_time_from_now, 'now': current_json_time(), 'revision_hash': params['revision_hash'] }.items()) treeherder_route = '{}.{}'.format(params['project'], params.get('revision_hash', '')) routes_file = os.path.join(ROOT, 'routes.json') with open(routes_file) as f: contents = json.load(f) json_routes = contents['routes'] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = {'tasks': [], 'scopes': []} if params['revision_hash']: for env in routes_transform.TREEHERDER_ROUTES: route = 'queue:route:{}.{}'.format( routes_transform.TREEHERDER_ROUTES[env], treeherder_route) graph['scopes'].append(route) graph['metadata'] = { 'source': 'http://todo.com/what/goes/here', 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Task graph generated via ./mach taskcluster-graph', 'name': 'task graph local' } all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = merge_dicts(parameters, build['additional-parameters']) build_parameters['build_slugid'] = slugid() build_task = templates.load(build['task'], build_parameters) # Copy build_* attributes to expose them to post-build tasks # as well as json routes and tests task_extra = build_task['task']['extra'] build_parameters['build_name'] = task_extra['build_name'] build_parameters['build_type'] = task_extra['build_type'] build_parameters['build_product'] = task_extra['build_product'] normalize_image_details(graph, build_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) if params['revision_hash']: routes_transform.decorate_task_treeherder_routes( build_task['task'], treeherder_route) routes_transform.decorate_task_json_routes( build_task['task'], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) test_packages_url, tests_url, mozharness_url = None, None, None if 'test_packages' in build_task['task']['extra']['locations']: test_packages_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['test_packages']) if 'tests' in build_task['task']['extra']['locations']: tests_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['tests']) if 'mozharness' in build_task['task']['extra']['locations']: mozharness_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['mozharness']) build_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['build']) build_parameters['build_url'] = build_url # img_url is only necessary for device builds img_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations'].get('img', '')) build_parameters['img_url'] = img_url define_task = DEFINE_TASK.format(build_task['task']['workerType']) for route in build_task['task'].get('routes', []): if route.startswith('index.gecko.v2') and route in all_routes: raise Exception( "Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % ( route, build_task['task']['metadata']['name'], all_routes[route], )) all_routes[route] = build_task['task']['metadata']['name'] graph['scopes'].append(define_task) graph['scopes'].extend(build_task['task'].get('scopes', [])) route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])) graph['scopes'].extend(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task['task']['extra']['treeherder'] if 'machine' not in build_treeherder_config: message = '({}), extra.treeherder.machine required for all builds' raise ValueError(message.format(build['task'])) if 'build' not in build_treeherder_config: build_treeherder_config['build'] = \ build_treeherder_config['machine'] if 'collection' not in build_treeherder_config: build_treeherder_config['collection'] = {'opt': True} if len(build_treeherder_config['collection'].keys()) != 1: message = '({}), extra.treeherder.collection must contain one type' raise ValueError(message.fomrat(build['task'])) for post_build in build['post-build']: # copy over the old parameters to update the template # TODO additional-parameters is currently not an option, only # enabled for build tasks post_parameters = merge_dicts( build_parameters, post_build.get('additional-parameters', {})) post_task = configure_dependent_task(post_build['task'], post_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, post_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(post_task, interactive) graph['tasks'].append(post_task) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] # TODO additional-parameters is currently not an option, only # enabled for build tasks test_parameters = merge_dicts( build_parameters, test.get('additional-parameters', {})) test_parameters = copy.copy(build_parameters) if tests_url: test_parameters['tests_url'] = tests_url if test_packages_url: test_parameters['test_packages_url'] = test_packages_url if mozharness_url: test_parameters['mozharness_url'] = mozharness_url test_definition = templates.load(test['task'], {})['task'] chunk_config = test_definition['extra'].get('chunks', {}) # Allow branch configs to override task level chunking... if 'chunks' in test: chunk_config['total'] = test['chunks'] chunked = 'total' in chunk_config if chunked: test_parameters['total_chunks'] = chunk_config['total'] if 'suite' in test_definition['extra']: suite_config = test_definition['extra']['suite'] test_parameters['suite'] = suite_config['name'] test_parameters['flavor'] = suite_config.get('flavor', '') for chunk in range(1, chunk_config.get('total', 1) + 1): if 'only_chunks' in test and chunked and \ chunk not in test['only_chunks']: continue if chunked: test_parameters['chunk'] = chunk test_task = configure_dependent_task( test['task'], test_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, test_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(test_task, interactive) if params['revision_hash']: routes_transform.decorate_task_treeherder_routes( test_task['task'], treeherder_route) graph['tasks'].append(test_task) define_task = DEFINE_TASK.format( test_task['task']['workerType']) graph['scopes'].append(define_task) graph['scopes'].extend(test_task['task'].get('scopes', [])) graph['scopes'] = list(set(graph['scopes'])) if params['print_names_only']: tIDs = defaultdict(list) def print_task(task, indent=0): print('{}- {}'.format(' ' * indent, task['task']['metadata']['name'])) for child in tIDs[task['taskId']]: print_task(child, indent=indent + 2) # build a dependency map for task in graph['tasks']: if 'requires' in task: for tID in task['requires']: tIDs[tID].append(task) # recursively print root tasks for task in graph['tasks']: if 'requires' not in task: print_task(task) return # When we are extending the graph remove extra fields... if params['ci'] is True: graph.pop('scopes', None) graph.pop('metadata', None) print(json.dumps(graph, indent=4))
def create_graph(self, **params): from taskcluster_graph.commit_parser import parse_commit from slugid import nice as slugid from taskcluster_graph.from_now import ( json_time_from_now, current_json_time, ) from taskcluster_graph.templates import Templates import taskcluster_graph.build_task if params['dry_run']: from taskcluster_graph.dry_run import ( json_time_from_now, current_json_time, slugid, ) project = params['project'] message = params.get('message', '') if project == 'try' else DEFAULT_TRY # Message would only be blank when not created from decision task if project == 'try' and not message: sys.stderr.write( "Must supply commit message when creating try graph. " \ "Example: --message='try: -b do -p all -u all'" ) sys.exit(1) templates = Templates(ROOT) job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml') job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph = parse_commit(message, jobs) cmdline_interactive = params.get('interactive', False) # Default to current time if querying the head rev fails pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) pushinfo = query_pushinfo(params['head_repository'], params['head_rev']) if pushinfo: pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) # Template parameters used when expanding the graph parameters = dict(gaia_info().items() + { 'index': 'index', 'project': project, 'pushlog_id': params.get('pushlog_id', 0), 'docker_image': docker_image, 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'pushdate': pushdate, 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'owner': params['owner'], 'from_now': json_time_from_now, 'now': current_json_time(), 'revision_hash': params['revision_hash'] }.items()) treeherder_route = '{}.{}'.format( params['project'], params.get('revision_hash', '') ) routes_file = os.path.join(ROOT, 'routes.json') with open(routes_file) as f: contents = json.load(f) json_routes = contents['routes'] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = { 'tasks': [], 'scopes': [] } if params['revision_hash']: for env in TREEHERDER_ROUTES: graph['scopes'].append('queue:route:{}.{}'.format(TREEHERDER_ROUTES[env], treeherder_route)) graph['metadata'] = { 'source': 'http://todo.com/what/goes/here', 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Task graph generated via ./mach taskcluster-graph', 'name': 'task graph local' } all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = dict(parameters) build_parameters['build_slugid'] = slugid() build_task = templates.load(build['task'], build_parameters) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) if params['revision_hash']: decorate_task_treeherder_routes(build_task['task'], treeherder_route) decorate_task_json_routes(build, build_task['task'], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) test_packages_url, tests_url, mozharness_url = None, None, None if 'test_packages' in build_task['task']['extra']['locations']: test_packages_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['test_packages'] ) if 'tests' in build_task['task']['extra']['locations']: tests_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['tests'] ) if 'mozharness' in build_task['task']['extra']['locations']: mozharness_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['mozharness'] ) build_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['build'] ) build_parameters['build_url'] = build_url # img_url is only necessary for device builds img_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations'].get('img', '') ) build_parameters['img_url'] = img_url define_task = DEFINE_TASK.format(build_task['task']['workerType']) for route in build_task['task'].get('routes', []): if route.startswith('index.gecko.v2') and route in all_routes: raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % ( route, build_task['task']['metadata']['name'], all_routes[route], )) all_routes[route] = build_task['task']['metadata']['name'] graph['scopes'].append(define_task) graph['scopes'].extend(build_task['task'].get('scopes', [])) route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])) graph['scopes'].extend(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task['task']['extra']['treeherder'] if 'machine' not in build_treeherder_config: message = '({}), extra.treeherder.machine required for all builds' raise ValueError(message.format(build['task'])) if 'build' not in build_treeherder_config: build_treeherder_config['build'] = \ build_treeherder_config['machine'] if 'collection' not in build_treeherder_config: build_treeherder_config['collection'] = { 'opt': True } if len(build_treeherder_config['collection'].keys()) != 1: message = '({}), extra.treeherder.collection must contain one type' raise ValueError(message.fomrat(build['task'])) for post_build in build['post-build']: # copy over the old parameters to update the template post_parameters = copy.copy(build_parameters) post_task = configure_dependent_task(post_build['task'], post_parameters, slugid(), templates, build_treeherder_config) set_interactive_task(post_task, interactive) graph['tasks'].append(post_task) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] test_parameters = copy.copy(build_parameters) if tests_url: test_parameters['tests_url'] = tests_url if test_packages_url: test_parameters['test_packages_url'] = test_packages_url if mozharness_url: test_parameters['mozharness_url'] = mozharness_url test_definition = templates.load(test['task'], {})['task'] chunk_config = test_definition['extra']['chunks'] # Allow branch configs to override task level chunking... if 'chunks' in test: chunk_config['total'] = test['chunks'] test_parameters['total_chunks'] = chunk_config['total'] for chunk in range(1, chunk_config['total'] + 1): if 'only_chunks' in test and \ chunk not in test['only_chunks']: continue test_parameters['chunk'] = chunk test_task = configure_dependent_task(test['task'], test_parameters, slugid(), templates, build_treeherder_config) set_interactive_task(test_task, interactive) if params['revision_hash']: decorate_task_treeherder_routes( test_task['task'], treeherder_route) graph['tasks'].append(test_task) define_task = DEFINE_TASK.format( test_task['task']['workerType'] ) graph['scopes'].append(define_task) graph['scopes'].extend(test_task['task'].get('scopes', [])) graph['scopes'] = list(set(graph['scopes'])) if params['print_names_only']: tIDs = defaultdict(list) def print_task(task, indent=0): print('{}- {}'.format(' ' * indent, task['task']['metadata']['name'])) for child in tIDs[task['taskId']]: print_task(child, indent=indent+2) # build a dependency map for task in graph['tasks']: if 'requires' in task: for tID in task['requires']: tIDs[tID].append(task) # recursively print root tasks for task in graph['tasks']: if 'requires' not in task: print_task(task) return # When we are extending the graph remove extra fields... if params['ci'] is True: graph.pop('scopes', None) graph.pop('metadata', None) print(json.dumps(graph, indent=4))
def create_graph(self, **params): from functools import partial from slugid import nice as slugid import taskcluster_graph.transform.routes as routes_transform from taskcluster_graph.commit_parser import parse_commit from taskcluster_graph.image_builder import ( docker_image, normalize_image_details, task_id_for_image ) from taskcluster_graph.from_now import ( json_time_from_now, current_json_time, ) from taskcluster_graph.templates import Templates import taskcluster_graph.build_task if params['dry_run']: from taskcluster_graph.dry_run import ( json_time_from_now, current_json_time, slugid, ) project = params['project'] message = params.get('message', '') if project == 'try' else DEFAULT_TRY # Message would only be blank when not created from decision task if project == 'try' and not message: sys.stderr.write( "Must supply commit message when creating try graph. " \ "Example: --message='try: -b do -p all -u all'" ) sys.exit(1) templates = Templates(ROOT) job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml') job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph = parse_commit(message, jobs) cmdline_interactive = params.get('interactive', False) # Default to current time if querying the head rev fails pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) pushinfo = query_pushinfo(params['head_repository'], params['head_rev']) if pushinfo: pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) # Template parameters used when expanding the graph seen_images = {} parameters = dict(gaia_info().items() + { 'index': 'index', 'project': project, 'pushlog_id': params.get('pushlog_id', 0), 'docker_image': docker_image, 'task_id_for_image': partial(task_id_for_image, seen_images, project), 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'owner': params['owner'], 'level': params['level'], 'from_now': json_time_from_now, 'now': current_json_time(), 'revision_hash': params['revision_hash'] }.items()) treeherder_route = '{}.{}'.format( params['project'], params.get('revision_hash', '') ) routes_file = os.path.join(ROOT, 'routes.json') with open(routes_file) as f: contents = json.load(f) json_routes = contents['routes'] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = { 'tasks': [], 'scopes': set(), } if params['revision_hash']: for env in routes_transform.TREEHERDER_ROUTES: route = 'queue:route:{}.{}'.format( routes_transform.TREEHERDER_ROUTES[env], treeherder_route) graph['scopes'].add(route) graph['metadata'] = { 'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']), 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Task graph generated via ./mach taskcluster-graph', 'name': 'task graph local' } all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = merge_dicts(parameters, build['additional-parameters']); build_parameters['build_slugid'] = slugid() build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task']) build_task = templates.load(build['task'], build_parameters) # Copy build_* attributes to expose them to post-build tasks # as well as json routes and tests task_extra = build_task['task']['extra'] build_parameters['build_name'] = task_extra['build_name'] build_parameters['build_type'] = task_extra['build_type'] build_parameters['build_product'] = task_extra['build_product'] normalize_image_details(graph, build_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) if params['revision_hash']: routes_transform.decorate_task_treeherder_routes(build_task['task'], treeherder_route) routes_transform.decorate_task_json_routes(build_task['task'], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) for location in build_task['task']['extra'].get('locations', {}): build_parameters['{}_url'.format(location)] = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations'][location] ) for url in build_task['task']['extra'].get('url', {}): build_parameters['{}_url'.format(url)] = \ build_task['task']['extra']['url'][url] define_task = DEFINE_TASK.format(build_task['task']['workerType']) for route in build_task['task'].get('routes', []): if route.startswith('index.gecko.v2') and route in all_routes: raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % ( route, build_task['task']['metadata']['name'], all_routes[route], )) all_routes[route] = build_task['task']['metadata']['name'] graph['scopes'].add(define_task) graph['scopes'] |= set(build_task['task'].get('scopes', [])) route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])) graph['scopes'] |= set(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task['task']['extra']['treeherder'] if 'machine' not in build_treeherder_config: message = '({}), extra.treeherder.machine required for all builds' raise ValueError(message.format(build['task'])) if 'build' not in build_treeherder_config: build_treeherder_config['build'] = \ build_treeherder_config['machine'] if 'collection' not in build_treeherder_config: build_treeherder_config['collection'] = { 'opt': True } if len(build_treeherder_config['collection'].keys()) != 1: message = '({}), extra.treeherder.collection must contain one type' raise ValueError(message.fomrat(build['task'])) for post_build in build['post-build']: # copy over the old parameters to update the template # TODO additional-parameters is currently not an option, only # enabled for build tasks post_parameters = merge_dicts(build_parameters, post_build.get('additional-parameters', {})) post_task = configure_dependent_task(post_build['task'], post_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, post_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(post_task, interactive) graph['tasks'].append(post_task) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] # TODO additional-parameters is currently not an option, only # enabled for build tasks test_parameters = merge_dicts(build_parameters, test.get('additional-parameters', {})) test_parameters = copy.copy(build_parameters) test_definition = templates.load(test['task'], {})['task'] chunk_config = test_definition['extra'].get('chunks', {}) # Allow branch configs to override task level chunking... if 'chunks' in test: chunk_config['total'] = test['chunks'] chunked = 'total' in chunk_config if chunked: test_parameters['total_chunks'] = chunk_config['total'] if 'suite' in test_definition['extra']: suite_config = test_definition['extra']['suite'] test_parameters['suite'] = suite_config['name'] test_parameters['flavor'] = suite_config.get('flavor', '') for chunk in range(1, chunk_config.get('total', 1) + 1): if 'only_chunks' in test and chunked and \ chunk not in test['only_chunks']: continue if chunked: test_parameters['chunk'] = chunk test_task = configure_dependent_task(test['task'], test_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, test_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(test_task, interactive) if params['revision_hash']: routes_transform.decorate_task_treeherder_routes( test_task['task'], treeherder_route ) graph['tasks'].append(test_task) define_task = DEFINE_TASK.format( test_task['task']['workerType'] ) graph['scopes'].add(define_task) graph['scopes'] |= set(test_task['task'].get('scopes', [])) graph['scopes'] = sorted(graph['scopes']) if params['print_names_only']: tIDs = defaultdict(list) def print_task(task, indent=0): print('{}- {}'.format(' ' * indent, task['task']['metadata']['name'])) for child in tIDs[task['taskId']]: print_task(child, indent=indent+2) # build a dependency map for task in graph['tasks']: if 'requires' in task: for tID in task['requires']: tIDs[tID].append(task) # recursively print root tasks for task in graph['tasks']: if 'requires' not in task: print_task(task) return # When we are extending the graph remove extra fields... if params['ci'] is True: graph.pop('scopes', None) graph.pop('metadata', None) print(json.dumps(graph, indent=4, sort_keys=True))
def create_isolate_failure_tasks(task_definition, failures, level, times): """ Create tasks to re-run the original task plus tasks to test each failing test directory and individual path. """ logger.info("Isolate task:\n{}".format( json.dumps(task_definition, indent=2))) task_name = task_definition['metadata']['name'] repeatable_task = False if ('crashtest' in task_name or 'mochitest' in task_name or 'reftest' in task_name and 'jsreftest' not in task_name): repeatable_task = True th_dict = task_definition['extra']['treeherder'] symbol = th_dict['symbol'] is_windows = 'windows' in th_dict['machine']['platform'] suite = task_definition['extra']['suite'] if '-coverage' in suite: suite = suite[:suite.index('-coverage')] is_wpt = 'web-platform-tests' in suite # command is a copy of task_definition['payload']['command'] from the original task. # It is used to create the new version containing the # task_definition['payload']['command'] with repeat_args which is updated every time # through the failure_group loop. command = copy.deepcopy(task_definition['payload']['command']) th_dict['groupSymbol'] = th_dict['groupSymbol'] + '-I' th_dict['tier'] = 3 for i in range(times): create_task_from_def(slugid(), task_definition, level) if repeatable_task: task_definition['payload']['maxRunTime'] = 3600 * 3 for failure_group in failures: if failure_group == 'dirs': failure_group_suffix = '-id' # execute 5 total loops repeat_args = ['--repeat=4'] if repeatable_task else [] else: failure_group_suffix = '-it' # execute 20 total loops repeat_args = ['--repeat=19'] if repeatable_task else [] if repeat_args: task_definition['payload']['command'] = add_args_to_command( command, extra_args=repeat_args) else: task_definition['payload']['command'] = command # saved_command is a saved version of the # task_definition['payload']['command'] with the repeat # arguments. We need to save it since # task_definition['payload']['command'] will be modified in the failure_path loop # when we are isolating web-platform-tests. saved_command = copy.deepcopy(task_definition['payload']['command']) for failure_path in failures[failure_group]: th_dict['symbol'] = symbol + failure_group_suffix if is_windows and not is_wpt: failure_path = '\\'.join(failure_path.split('/')) if is_wpt: include_args = ['--include={}'.format(failure_path)] task_definition['payload']['command'] = add_args_to_command( saved_command, extra_args=include_args) else: task_definition['payload']['env'][ 'MOZHARNESS_TEST_PATHS'] = six.ensure_text( json.dumps({suite: [failure_path]})) logger.info("Creating task for path {} with command {}".format( failure_path, task_definition['payload']['command'])) for i in range(times): create_task_from_def(slugid(), task_definition, level)
def create_graph(self, **params): from functools import partial from mozpack.path import match as mozpackmatch from slugid import nice as slugid import taskcluster_graph.transform.routes as routes_transform import taskcluster_graph.transform.treeherder as treeherder_transform from taskcluster_graph.commit_parser import parse_commit from taskcluster_graph.image_builder import (docker_image, normalize_image_details, task_id_for_image) from taskcluster_graph.from_now import ( json_time_from_now, current_json_time, ) from taskcluster_graph.templates import Templates import taskcluster_graph.build_task if params['dry_run']: from taskcluster_graph.dry_run import ( json_time_from_now, current_json_time, slugid, ) project = params['project'] message = params.get('message', '') if project == 'try' else DEFAULT_TRY templates = Templates(ROOT) job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml') job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph, trigger_tests = parse_commit(message, jobs) cmdline_interactive = params.get('interactive', False) # Default to current time if querying the head rev fails pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) vcs_info = query_vcs_info(params['head_repository'], params['head_rev']) changed_files = set() if vcs_info: pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate)) sys.stderr.write('%d commits influencing task scheduling:\n' % len(vcs_info.changesets)) for c in vcs_info.changesets: sys.stderr.write( '%s %s\n' % (c['node'][0:12], c['desc'].splitlines()[0].encode( 'ascii', 'ignore'))) changed_files |= set(c['files']) # Template parameters used when expanding the graph seen_images = {} parameters = dict(gaia_info().items() + { 'index': 'index', 'project': project, 'pushlog_id': params.get('pushlog_id', 0), 'docker_image': docker_image, 'task_id_for_image': partial(task_id_for_image, seen_images, project), 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'owner': params['owner'], 'level': params['level'], 'from_now': json_time_from_now, 'now': current_json_time(), 'revision_hash': params['revision_hash'] }.items()) treeherder_route = '{}.{}'.format(params['project'], params.get('revision_hash', '')) routes_file = os.path.join(ROOT, 'routes.json') with open(routes_file) as f: contents = json.load(f) json_routes = contents['routes'] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = { 'tasks': [], 'scopes': set(), } if params['revision_hash']: for env in routes_transform.TREEHERDER_ROUTES: route = 'queue:route:{}.{}'.format( routes_transform.TREEHERDER_ROUTES[env], treeherder_route) graph['scopes'].add(route) graph['metadata'] = { 'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format( repo=params['head_repository'], rev=params['head_rev']), 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Task graph generated via ./mach taskcluster-graph', 'name': 'task graph local' } # Filter the job graph according to conditions met by this invocation run. def should_run(task): # Old style build or test task that doesn't define conditions. Always runs. if 'when' not in task: return True # Command line override to not filter. if params['ignore_conditions']: return True when = task['when'] # If the task defines file patterns and we have a set of changed # files to compare against, only run if a file pattern matches one # of the changed files. file_patterns = when.get('file_patterns', None) if file_patterns and changed_files: for pattern in file_patterns: for path in changed_files: if mozpackmatch(path, pattern): sys.stderr.write( 'scheduling %s because pattern %s ' 'matches %s\n' % (task['task'], pattern, path)) return True # No file patterns matched. Discard task. sys.stderr.write( 'discarding %s because no relevant files changed\n' % task['task']) return False return True job_graph = filter(should_run, job_graph) all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = merge_dicts(parameters, build['additional-parameters']) build_parameters['build_slugid'] = slugid() build_parameters[ 'source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format( repo=params['head_repository'], rev=params['head_rev'], file=build['task']) build_task = templates.load(build['task'], build_parameters) # Copy build_* attributes to expose them to post-build tasks # as well as json routes and tests task_extra = build_task['task']['extra'] build_parameters['build_name'] = task_extra['build_name'] build_parameters['build_type'] = task_extra['build_type'] build_parameters['build_product'] = task_extra['build_product'] normalize_image_details(graph, build_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) if params['revision_hash']: treeherder_transform.add_treeherder_revision_info( build_task['task'], params['head_rev'], params['revision_hash']) routes_transform.decorate_task_treeherder_routes( build_task['task'], treeherder_route) routes_transform.decorate_task_json_routes( build_task['task'], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) for location in build_task['task']['extra'].get('locations', {}): build_parameters['{}_url'.format( location)] = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations'][location]) for url in build_task['task']['extra'].get('url', {}): build_parameters['{}_url'.format(url)] = \ build_task['task']['extra']['url'][url] define_task = DEFINE_TASK.format(build_task['task']['workerType']) for route in build_task['task'].get('routes', []): if route.startswith('index.gecko.v2') and route in all_routes: raise Exception( "Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % ( route, build_task['task']['metadata']['name'], all_routes[route], )) all_routes[route] = build_task['task']['metadata']['name'] graph['scopes'].add(define_task) graph['scopes'] |= set(build_task['task'].get('scopes', [])) route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])) graph['scopes'] |= set(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task['task']['extra']['treeherder'] if 'machine' not in build_treeherder_config: message = '({}), extra.treeherder.machine required for all builds' raise ValueError(message.format(build['task'])) if 'build' not in build_treeherder_config: build_treeherder_config['build'] = \ build_treeherder_config['machine'] if 'collection' not in build_treeherder_config: build_treeherder_config['collection'] = {'opt': True} if len(build_treeherder_config['collection'].keys()) != 1: message = '({}), extra.treeherder.collection must contain one type' raise ValueError(message.fomrat(build['task'])) for post_build in build['post-build']: # copy over the old parameters to update the template # TODO additional-parameters is currently not an option, only # enabled for build tasks post_parameters = merge_dicts( build_parameters, post_build.get('additional-parameters', {})) post_task = configure_dependent_task(post_build['task'], post_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, post_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(post_task, interactive) treeherder_transform.add_treeherder_revision_info( post_task['task'], params['head_rev'], params['revision_hash']) graph['tasks'].append(post_task) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] # TODO additional-parameters is currently not an option, only # enabled for build tasks test_parameters = merge_dicts( build_parameters, test.get('additional-parameters', {})) test_parameters = copy.copy(build_parameters) test_definition = templates.load(test['task'], {})['task'] chunk_config = test_definition['extra'].get('chunks', {}) # Allow branch configs to override task level chunking... if 'chunks' in test: chunk_config['total'] = test['chunks'] chunked = 'total' in chunk_config if chunked: test_parameters['total_chunks'] = chunk_config['total'] if 'suite' in test_definition['extra']: suite_config = test_definition['extra']['suite'] test_parameters['suite'] = suite_config['name'] test_parameters['flavor'] = suite_config.get('flavor', '') for chunk in range(1, chunk_config.get('total', 1) + 1): if 'only_chunks' in test and chunked and \ chunk not in test['only_chunks']: continue if chunked: test_parameters['chunk'] = chunk test_task = configure_dependent_task( test['task'], test_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, test_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(test_task, interactive) if params['revision_hash']: treeherder_transform.add_treeherder_revision_info( test_task['task'], params['head_rev'], params['revision_hash']) routes_transform.decorate_task_treeherder_routes( test_task['task'], treeherder_route) # This will schedule test jobs N times for i in range(0, trigger_tests): graph['tasks'].append(test_task) # If we're scheduling more tasks each have to be unique test_task = copy.deepcopy(test_task) test_task['taskId'] = slugid() define_task = DEFINE_TASK.format( test_task['task']['workerType']) graph['scopes'].add(define_task) graph['scopes'] |= set(test_task['task'].get('scopes', [])) graph['scopes'] = sorted(graph['scopes']) if params['print_names_only']: tIDs = defaultdict(list) def print_task(task, indent=0): print('{}- {}'.format(' ' * indent, task['task']['metadata']['name'])) for child in tIDs[task['taskId']]: print_task(child, indent=indent + 2) # build a dependency map for task in graph['tasks']: if 'requires' in task: for tID in task['requires']: tIDs[tID].append(task) # recursively print root tasks for task in graph['tasks']: if 'requires' not in task: print_task(task) return # When we are extending the graph remove extra fields... if params['ci'] is True: graph.pop('scopes', None) graph.pop('metadata', None) print(json.dumps(graph, indent=4, sort_keys=True))
def create_graph(self, **params): from functools import partial from mozpack.path import match as mozpackmatch from slugid import nice as slugid from taskcluster_graph.mach_util import ( merge_dicts, gaia_info, configure_dependent_task, set_interactive_task, remove_caches_from_task, query_vcs_info ) import taskcluster_graph.transform.routes as routes_transform import taskcluster_graph.transform.treeherder as treeherder_transform from taskcluster_graph.commit_parser import parse_commit from taskcluster_graph.image_builder import ( docker_image, normalize_image_details, task_id_for_image ) from taskcluster_graph.from_now import ( json_time_from_now, current_json_time, ) from taskcluster_graph.templates import Templates import taskcluster_graph.build_task if params['dry_run']: from taskcluster_graph.dry_run import ( json_time_from_now, current_json_time, slugid, ) project = params['project'] message = params.get('message', '') if project == 'try' else DEFAULT_TRY templates = Templates(ROOT) job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml') job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH jobs = templates.load(job_path, {}) job_graph, trigger_tests = parse_commit(message, jobs) cmdline_interactive = params.get('interactive', False) # Default to current time if querying the head rev fails pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime()) vcs_info = query_vcs_info(params['head_repository'], params['head_rev']) changed_files = set() if vcs_info: pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate)) sys.stderr.write('%d commits influencing task scheduling:\n' % len(vcs_info.changesets)) for c in vcs_info.changesets: sys.stderr.write('%s %s\n' % ( c['node'][0:12], c['desc'].splitlines()[0].encode('ascii', 'ignore'))) changed_files |= set(c['files']) # Template parameters used when expanding the graph seen_images = {} parameters = dict(gaia_info().items() + { 'index': 'index', 'project': project, 'pushlog_id': params.get('pushlog_id', 0), 'docker_image': docker_image, 'task_id_for_image': partial(task_id_for_image, seen_images, project), 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'pushdate': pushdate, 'pushtime': pushdate[8:], 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'owner': params['owner'], 'level': params['level'], 'from_now': json_time_from_now, 'now': current_json_time(), 'revision_hash': params['revision_hash'] }.items()) treeherder_route = '{}.{}'.format( params['project'], params.get('revision_hash', '') ) routes_file = os.path.join(ROOT, 'routes.json') with open(routes_file) as f: contents = json.load(f) json_routes = contents['routes'] # TODO: Nightly and/or l10n routes # Task graph we are generating for taskcluster... graph = { 'tasks': [], 'scopes': set(), } if params['revision_hash']: for env in routes_transform.TREEHERDER_ROUTES: route = 'queue:route:{}.{}'.format( routes_transform.TREEHERDER_ROUTES[env], treeherder_route) graph['scopes'].add(route) graph['metadata'] = { 'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(repo=params['head_repository'], rev=params['head_rev']), 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Task graph generated via ./mach taskcluster-graph', 'name': 'task graph local' } # Filter the job graph according to conditions met by this invocation run. def should_run(task): # Old style build or test task that doesn't define conditions. Always runs. if 'when' not in task: return True # Command line override to not filter. if params['ignore_conditions']: return True when = task['when'] # If the task defines file patterns and we have a set of changed # files to compare against, only run if a file pattern matches one # of the changed files. file_patterns = when.get('file_patterns', None) if file_patterns and changed_files: # Always consider changes to the task definition itself file_patterns.append('testing/taskcluster/{task}'.format(task=task['task'])) for pattern in file_patterns: for path in changed_files: if mozpackmatch(path, pattern): sys.stderr.write('scheduling %s because pattern %s ' 'matches %s\n' % (task['task'], pattern, path)) return True # No file patterns matched. Discard task. sys.stderr.write('discarding %s because no relevant files changed\n' % task['task']) return False return True job_graph = filter(should_run, job_graph) all_routes = {} for build in job_graph: interactive = cmdline_interactive or build["interactive"] build_parameters = merge_dicts(parameters, build['additional-parameters']); build_parameters['build_slugid'] = slugid() build_parameters['source'] = '{repo}file/{rev}/testing/taskcluster/{file}'.format(repo=params['head_repository'], rev=params['head_rev'], file=build['task']) build_task = templates.load(build['task'], build_parameters) # Copy build_* attributes to expose them to post-build tasks # as well as json routes and tests task_extra = build_task['task']['extra'] build_parameters['build_name'] = task_extra['build_name'] build_parameters['build_type'] = task_extra['build_type'] build_parameters['build_product'] = task_extra['build_product'] normalize_image_details(graph, build_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(build_task, interactive) # try builds don't use cache if project == "try": remove_caches_from_task(build_task) set_expiration(build_task, json_time_from_now(TRY_EXPIRATION)) if params['revision_hash']: treeherder_transform.add_treeherder_revision_info(build_task['task'], params['head_rev'], params['revision_hash']) routes_transform.decorate_task_treeherder_routes(build_task['task'], treeherder_route) routes_transform.decorate_task_json_routes(build_task['task'], json_routes, build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) for location in build_task['task']['extra'].get('locations', {}): build_parameters['{}_url'.format(location)] = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations'][location] ) for url in build_task['task']['extra'].get('url', {}): build_parameters['{}_url'.format(url)] = \ build_task['task']['extra']['url'][url] define_task = DEFINE_TASK.format(build_task['task']['workerType']) for route in build_task['task'].get('routes', []): if route.startswith('index.gecko.v2') and route in all_routes: raise Exception("Error: route '%s' is in use by multiple tasks: '%s' and '%s'" % ( route, build_task['task']['metadata']['name'], all_routes[route], )) all_routes[route] = build_task['task']['metadata']['name'] graph['scopes'].add(define_task) graph['scopes'] |= set(build_task['task'].get('scopes', [])) route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])) graph['scopes'] |= set(route_scopes) # Treeherder symbol configuration for the graph required for each # build so tests know which platform they belong to. build_treeherder_config = build_task['task']['extra']['treeherder'] if 'machine' not in build_treeherder_config: message = '({}), extra.treeherder.machine required for all builds' raise ValueError(message.format(build['task'])) if 'build' not in build_treeherder_config: build_treeherder_config['build'] = \ build_treeherder_config['machine'] if 'collection' not in build_treeherder_config: build_treeherder_config['collection'] = { 'opt': True } if len(build_treeherder_config['collection'].keys()) != 1: message = '({}), extra.treeherder.collection must contain one type' raise ValueError(message.fomrat(build['task'])) for post_build in build['post-build']: # copy over the old parameters to update the template # TODO additional-parameters is currently not an option, only # enabled for build tasks post_parameters = merge_dicts(build_parameters, post_build.get('additional-parameters', {})) post_task = configure_dependent_task(post_build['task'], post_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, post_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(post_task, interactive) treeherder_transform.add_treeherder_revision_info(post_task['task'], params['head_rev'], params['revision_hash']) if project == "try": set_expiration(post_task, json_time_from_now(TRY_EXPIRATION)) graph['tasks'].append(post_task) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] # TODO additional-parameters is currently not an option, only # enabled for build tasks test_parameters = merge_dicts(build_parameters, test.get('additional-parameters', {})) test_parameters = copy.copy(build_parameters) test_definition = templates.load(test['task'], {})['task'] chunk_config = test_definition['extra'].get('chunks', {}) # Allow branch configs to override task level chunking... if 'chunks' in test: chunk_config['total'] = test['chunks'] chunked = 'total' in chunk_config if chunked: test_parameters['total_chunks'] = chunk_config['total'] if 'suite' in test_definition['extra']: suite_config = test_definition['extra']['suite'] test_parameters['suite'] = suite_config['name'] test_parameters['flavor'] = suite_config.get('flavor', '') for chunk in range(1, chunk_config.get('total', 1) + 1): if 'only_chunks' in test and chunked and \ chunk not in test['only_chunks']: continue if chunked: test_parameters['chunk'] = chunk test_task = configure_dependent_task(test['task'], test_parameters, slugid(), templates, build_treeherder_config) normalize_image_details(graph, test_task, seen_images, build_parameters, os.environ.get('TASK_ID', None)) set_interactive_task(test_task, interactive) if params['revision_hash']: treeherder_transform.add_treeherder_revision_info(test_task['task'], params['head_rev'], params['revision_hash']) routes_transform.decorate_task_treeherder_routes( test_task['task'], treeherder_route ) if project == "try": set_expiration(test_task, json_time_from_now(TRY_EXPIRATION)) # This will schedule test jobs N times for i in range(0, trigger_tests): graph['tasks'].append(test_task) # If we're scheduling more tasks each have to be unique test_task = copy.deepcopy(test_task) test_task['taskId'] = slugid() define_task = DEFINE_TASK.format( test_task['task']['workerType'] ) graph['scopes'].add(define_task) graph['scopes'] |= set(test_task['task'].get('scopes', [])) graph['scopes'] = sorted(graph['scopes']) if params['print_names_only']: tIDs = defaultdict(list) def print_task(task, indent=0): print('{}- {}'.format(' ' * indent, task['task']['metadata']['name'])) for child in tIDs[task['taskId']]: print_task(child, indent=indent+2) # build a dependency map for task in graph['tasks']: if 'requires' in task: for tID in task['requires']: tIDs[tID].append(task) # recursively print root tasks for task in graph['tasks']: if 'requires' not in task: print_task(task) return # When we are extending the graph remove extra fields... if params['ci'] is True: graph.pop('scopes', None) graph.pop('metadata', None) print(json.dumps(graph, indent=4, sort_keys=True))