def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', description='Task A', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimization={'skip-unless-has-relevant-tests': None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'description': 'Task A', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'soft_dependencies': [], 'optimization': None, }, 'b': { 'kind': 'test', 'label': 'b', 'description': '', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'soft_dependencies': [], 'optimization': { 'skip-unless-has-relevant-tests': None }, } })
def test_round_trip(self): graph = TaskGraph( tasks={ "a": Task( kind="fancy", label="a", description="Task A", attributes={}, dependencies={"prereq": "b"}, # must match edges, below optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def"}, ), "b": Task( kind="pre", label="b", attributes={}, dependencies={}, optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def2"}, ), }, graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}), ) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. If you wish to create the tasks in a new group, leave out decision_task_id.""" to_run = set(to_run) target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: full_task_graph[l] for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) optimized_task_graph, label_to_taskid = optimize_task_graph( target_task_graph, params, to_run, label_to_taskid) write_artifact('task-graph.json', optimized_task_graph.to_json()) write_artifact('label-to-taskid.json', label_to_taskid) write_artifact('to-run.json', list(to_run)) create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task( kind='test', label='b', attributes={}, task={'task': 'def'}, optimizations=[['seta']], # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { 'a': { 'kind': 'test', 'label': 'a', 'attributes': { 'attr': 'a-task', 'kind': 'test' }, 'task': { 'taskdef': True }, 'dependencies': { 'edgelabel': 'b' }, 'optimizations': [], }, 'b': { 'kind': 'test', 'label': 'b', 'attributes': { 'kind': 'test' }, 'task': { 'task': 'def' }, 'dependencies': {}, 'optimizations': [['seta']], } })
def create_tasks(graph_config, to_run, full_task_graph, label_to_taskid, params, decision_task_id=None, suffix='', modifier=lambda t: t): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. Optionally this function takes a `modifier` function that is passed in each task before it is put into a new graph. It should return a valid task. Note that this is passed _all_ tasks in the graph, not just the set in to_run. You may want to skip modifying tasks not in your to_run list. If `suffix` is given, then it is used to give unique names to the resulting artifacts. If you call this function multiple times in the same action, pass a different suffix each time to avoid overwriting artifacts. If you wish to create the tasks in a new group, leave out decision_task_id. Returns an updated label_to_taskid containing the new tasks""" if suffix != '': suffix = '-{}'.format(suffix) to_run = set(to_run) # Copy to avoid side-effects later full_task_graph = copy.deepcopy(full_task_graph) label_to_taskid = label_to_taskid.copy() target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: modifier(full_task_graph[l]) for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) optimized_task_graph, label_to_taskid = optimize_task_graph( target_task_graph, params, to_run, label_to_taskid) write_artifact('task-graph{}.json'.format(suffix), optimized_task_graph.to_json()) write_artifact('label-to-taskid{}.json'.format(suffix), label_to_taskid) write_artifact('to-run{}.json'.format(suffix), list(to_run)) create.create_tasks( graph_config, optimized_task_graph, label_to_taskid, params, decision_task_id, ) return label_to_taskid
def test_taskgraph_to_json(self): tasks = { "a": Task( kind="test", label="a", description="Task A", attributes={"attr": "a-task"}, task={"taskdef": True}, ), "b": Task( kind="test", label="b", attributes={}, task={"task": "def"}, optimization={"skip-unless-has-relevant-tests": None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={"first": "a"}, ), } graph = Graph(nodes=set("ab"), edges={("a", "b", "edgelabel")}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual( res, { "a": { "kind": "test", "label": "a", "description": "Task A", "attributes": {"attr": "a-task", "kind": "test"}, "task": {"taskdef": True}, "dependencies": {"edgelabel": "b"}, "soft_dependencies": [], "if_dependencies": [], "optimization": None, }, "b": { "kind": "test", "label": "b", "description": "", "attributes": {"kind": "test"}, "task": {"task": "def"}, "dependencies": {}, "soft_dependencies": [], "if_dependencies": [], "optimization": {"skip-unless-has-relevant-tests": None}, }, }, )
def test_create_tasks(self): tasks = { "tid-a": Task(kind="test", label="a", attributes={}, task={"payload": "hello world"}), "tid-b": Task(kind="test", label="b", attributes={}, task={"payload": "hello world"}), } label_to_taskid = {"a": "tid-a", "b": "tid-b"} graph = Graph(nodes={"tid-a", "tid-b"}, edges={("tid-a", "tid-b", "edge")}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.items(): self.assertEqual(task["payload"], "hello world") self.assertEqual(task["schedulerId"], "domain-level-4") # make sure the dependencies exist, at least for depid in task.get("dependencies", []): if depid == "decisiontask": # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def generate_tasks(params=None, full=False, disable_target_task_filter=False): cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph") attr = "full_task_set" if full else "target_task_set" cache = os.path.join(cache_dir, attr) invalidate(cache) if os.path.isfile(cache): with open(cache, "r") as fh: return TaskGraph.from_json(json.load(fh))[1] if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format( attr.replace("_", " "))) taskgraph.fast = True cwd = os.getcwd() os.chdir(build.topsrcdir) root = os.path.join(build.topsrcdir, "taskcluster", "ci") target_tasks_method = ("try_select_tasks" if not disable_target_task_filter else "try_select_tasks_uncommon") params = parameters_loader( params, strict=False, overrides={ "try_mode": "try_select", "target_tasks_method": target_tasks_method, }, ) # Cache both full_task_set and target_task_set regardless of whether or not # --full was requested. Caching is cheap and can potentially save a lot of # time. generator = TaskGraphGenerator(root_dir=root, parameters=params) def generate(attr): try: tg = getattr(generator, attr) except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) # write cache with open(os.path.join(cache_dir, attr), "w") as fh: json.dump(tg.to_json(), fh) return tg tg_full = generate("full_task_set") tg_target = generate("target_task_set") # discard results from these, we only need cache. if full: generate("full_task_graph") generate("target_task_graph") os.chdir(cwd) if full: return tg_full return tg_target
def run_missing_tests(parameters, input, task_group_id, task_id, task): decision_task_id = find_decision_task(parameters) full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) target_tasks = get_artifact(decision_task_id, "public/target-tasks.json") label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # The idea here is to schedule all tasks of the `test` kind that were # targetted but did not appear in the final task-graph -- those were the # optimized tasks. to_run = [] already_run = 0 for label in target_tasks: task = full_task_graph.tasks[label] if task.kind != 'test': continue # not a test if label in label_to_taskid: already_run += 1 continue to_run.append(label) create_tasks(to_run, full_task_graph, label_to_taskid, parameters, decision_task_id) logger.info( 'Out of {} test tasks, {} already existed and the action created {}'. format(already_run + len(to_run), already_run, len(to_run)))
def retrigger_action(parameters, input, task_group_id, task_id, task): decision_task_id = find_decision_task(parameters) full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") label = task['metadata']['name'] with_downstream = ' ' to_run = [label] if input.get('downstream'): to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes to_run = to_run & set(label_to_taskid.keys()) with_downstream = ' (with downstream) ' times = input.get('times', 1) for i in xrange(times): create_tasks(to_run, full_task_graph, label_to_taskid, parameters, decision_task_id) logger.info('Scheduled {}{}(time {}/{})'.format( label, with_downstream, i + 1, times))
def test_create_tasks(self): tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), 'tid-b': Task(kind='test', label='b', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) taskgraph = TaskGraph(tasks, graph) create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", ) for tid, task in self.created_tasks.iteritems(): self.assertEqual(task['payload'], 'hello world') self.assertEqual(task['schedulerId'], 'domain-level-4') # make sure the dependencies exist, at least for depid in task.get('dependencies', []): if depid == 'decisiontask': # Don't look for decisiontask here continue self.assertIn(depid, self.created_tasks)
def add_new_jobs_action(parameters, input, task_group_id, task_id, task): decision_task_id = find_decision_task(parameters) full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") for elem in input['tasks']: if elem in full_task_graph.tasks: task = full_task_graph.tasks[elem] # fix up the task's dependencies, similar to how optimization would # have done in the decision dependencies = { name: label_to_taskid[label] for name, label in task.dependencies.iteritems() } task_def = resolve_task_references(task.label, task.task, dependencies) task_def.setdefault('dependencies', []).extend(dependencies.itervalues()) # actually create the new task create_task(slugid(), task_def, parameters['level']) else: raise Exception('{} was not found in the task-graph'.format(elem))
def inner(tasks): label_to_taskid = {k: k + '-tid' for k in tasks} for label, task_id in label_to_taskid.iteritems(): tasks[label].task_id = task_id graph = Graph(nodes=set(tasks), edges=set()) taskgraph = TaskGraph(tasks, graph) return taskgraph, label_to_taskid
def make_taskgraph(self, tasks): label_to_taskid = {k: k + "-tid" for k in tasks} for label, task_id in label_to_taskid.items(): tasks[label].task_id = task_id graph = Graph(nodes=set(tasks), edges=set()) taskgraph = TaskGraph(tasks, graph) return taskgraph, label_to_taskid
def fetch_graph_and_labels(parameters): decision_task_id = find_decision_task(parameters) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # Now fetch any modifications made by action tasks and swap out new tasks # for old ones namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format( parameters['project'], parameters['pushlog_id']) for action in list_tasks(namespace): try: run_label_to_id = get_artifact(action, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.info( 'Skipping {} due to missing artifact! Error: {}'.format( action, e)) continue return (decision_task_id, full_task_graph, label_to_taskid)
def get_filtered_taskgraph(self, taskgraph, tasksregex): from taskgraph.graph import Graph from taskgraph.taskgraph import TaskGraph """ This class method filters all the tasks on basis of a regular expression and returns a new TaskGraph object """ # return original taskgraph if no regular expression is passed if not tasksregex: return taskgraph named_links_dict = taskgraph.graph.named_links_dict() filteredtasks = {} filterededges = set() regexprogram = re.compile(tasksregex) for key in taskgraph.graph.visit_postorder(): task = taskgraph.tasks[key] if regexprogram.match(task.label): filteredtasks[key] = task for depname, dep in named_links_dict[key].iteritems(): if regexprogram.match(dep): filterededges.add((key, dep, depname)) filtered_taskgraph = TaskGraph( filteredtasks, Graph(set(filteredtasks), filterededges)) return filtered_taskgraph
def test_create_tasks_fails_if_create_fails(self, create_task): "creat_tasks fails if a single create_task call fails" tasks = { 'tid-a': Task(kind='test', label='a', attributes={}, task={'payload': 'hello world'}), } label_to_taskid = {'a': 'tid-a'} graph = Graph(nodes={'tid-a'}, edges=set()) taskgraph = TaskGraph(tasks, graph) def fail(*args): print("UHOH") raise RuntimeError('oh noes!') create_task.side_effect = fail with self.assertRaises(RuntimeError): create.create_tasks( GRAPH_CONFIG, taskgraph, label_to_taskid, {"level": "4"}, decision_task_id="decisiontask", )
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): release_promotion_flavor = input['release_promotion_flavor'] promotion_config = graph_config['release-promotion']['flavors'][ release_promotion_flavor] target_tasks_method = promotion_config['target-tasks-method'].format( project=parameters['project']) rebuild_kinds = input.get('rebuild_kinds') or promotion_config.get( 'rebuild-kinds', []) do_not_optimize = input.get('do_not_optimize') or promotion_config.get( 'do-not-optimize', []) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids`` or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: previous_graph_ids = [find_decision_task(parameters, graph_config)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters['optimize_target_tasks'] = True parameters['xpi_name'] = input['xpi_name'] # TODO # - require this is a specific revision # - possibly also check that this is on a reviewed PR or merged into # a trusted branch. this will require an oauth token parameters['xpi_revision'] = input.get('revision', 'master') parameters['shipping_phase'] = input['release_promotion_flavor'] # We blow away `tasks_for` when we load the on-push decision task's # parameters.yml. Let's set this back to `action`. parameters['tasks_for'] = "action" if input.get('version'): parameters['version'] = input['version'] # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def default_matches(self, attributes, parameters): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes=attributes, task={}), }, graph=Graph(nodes={'a'}, edges=set())) return 'a' in method(graph, parameters, {})
def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id): task = taskcluster.get_task_definition(task_id) label = task['metadata']['name'] pushes = [] depth = 2 end_id = int(parameters['pushlog_id']) while True: start_id = max(end_id - depth, 0) pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], start_id, end_id) r = requests.get(pushlog_url) r.raise_for_status() pushes = pushes + list(r.json()['pushes'].keys()) if len(pushes) >= depth: break end_id = start_id - 1 start_id -= depth if start_id < 0: break pushes = sorted(pushes)[-depth:] backfill_pushes = [] for push in pushes: try: full_task_graph = get_artifact_from_index( INDEX_TMPL.format(parameters['project'], push), 'public/full-task-graph.json') _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact_from_index( INDEX_TMPL.format(parameters['project'], push), 'public/label-to-taskid.json') push_params = get_artifact_from_index( INDEX_TMPL.format(parameters['project'], push), 'public/parameters.yml') push_decision_task_id = find_decision_task(push_params, graph_config) except HTTPError as e: logger.info('Skipping {} due to missing index artifacts! Error: {}'.format(push, e)) continue if label in full_task_graph.tasks.keys(): def modifier(task): if task.label != label: return task cmd = task.task['payload']['command'] task.task['payload']['command'] = add_args_to_perf_command( cmd, ['--gecko-profile']) task.task['extra']['treeherder']['symbol'] += '-p' return task create_tasks(graph_config, [label], full_task_graph, label_to_taskid, push_params, push_decision_task_id, push, modifier=modifier) backfill_pushes.append(push) else: logging.info('Could not find {} on {}. Skipping.'.format(label, push)) combine_task_graph_files(backfill_pushes)
def generate_tasks(params=None, full=False): # TODO: Remove after January 1st, 2020. # Try to delete the old taskgraph cache directories. root = build.topsrcdir root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest() old_cache_dirs = [ os.path.join(get_state_dir(), 'cache', 'taskgraph'), os.path.join(get_state_dir(), 'cache', root_hash, 'taskgraph'), ] for cache_dir in old_cache_dirs: if os.path.isdir(cache_dir): shutil.rmtree(cache_dir) cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph') attr = 'full_task_set' if full else 'target_task_set' cache = os.path.join(cache_dir, attr) invalidate(cache, root) if os.path.isfile(cache): with open(cache, 'r') as fh: return TaskGraph.from_json(json.load(fh))[1] if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format(attr.replace('_', ' '))) taskgraph.fast = True cwd = os.getcwd() os.chdir(root) root = os.path.join(root, 'taskcluster', 'ci') params = parameters_loader(params, strict=False, overrides={'try_mode': 'try_select'}) # Cache both full_task_set and target_task_set regardless of whether or not # --full was requested. Caching is cheap and can potentially save a lot of # time. generator = TaskGraphGenerator(root_dir=root, parameters=params) def generate(attr): try: tg = getattr(generator, attr) except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) # write cache with open(os.path.join(cache_dir, attr), 'w') as fh: json.dump(tg.to_json(), fh) return tg tg_full = generate('full_task_set') tg_target = generate('target_task_set') os.chdir(cwd) if full: return tg_full return tg_target
def fetch_graph_and_labels(parameters, graph_config): decision_task_id = find_decision_task(parameters, graph_config) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # fetch everything in parallel; this avoids serializing any delay in downloading # each artifact (such as waiting for the artifact to be mirrored locally) with futures.ThreadPoolExecutor(CONCURRENCY) as e: fetches = [] # fetch any modifications made by action tasks and swap out new tasks # for old ones def fetch_action(task_id): logger.info(f"fetching label-to-taskid.json for action task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") namespace = "{}.v2.{}.pushlog-id.{}.actions".format( graph_config["trust-domain"], parameters["project"], parameters["pushlog_id"], ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_action, task_id)) # Similarly for cron tasks.. def fetch_cron(task_id): logger.info(f"fetching label-to-taskid.json for cron task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") namespace = "{}.v2.{}.revision.{}.cron".format( graph_config["trust-domain"], parameters["project"], parameters["head_rev"] ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_cron, task_id)) # now wait for each fetch to complete, raising an exception if there # were any issues for f in futures.as_completed(fetches): f.result() return (decision_task_id, full_task_graph, label_to_taskid)
def make_graph(*tasks_and_edges, **kwargs): tasks = {t.label: t for t in tasks_and_edges if isinstance(t, Task)} edges = {e for e in tasks_and_edges if not isinstance(e, Task)} tg = TaskGraph(tasks, graph.Graph(set(tasks), edges)) if kwargs.get("deps", True): # set dependencies based on edges for l, r, name in tg.graph.edges: tg.tasks[l].dependencies[name] = r return tg
def tg(request): if not hasattr(request.module, 'TASKS'): pytest.fail("'tg' fixture used from a module that didn't define the TASKS variable") tasks = request.module.TASKS for task in tasks: task.setdefault('task', {}) task['task'].setdefault('tags', {}) tasks = {t['label']: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def test_round_trip(self): graph = TaskGraph(tasks={ 'a': Task( kind='fancy', label='a', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimization={'seta': None}, task={'task': 'def'}), 'b': Task( kind='pre', label='b', attributes={}, dependencies={}, optimization={'seta': None}, task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def test_round_trip(self): graph = TaskGraph(tasks={ 'a': Task( kind='fancy', label='a', description='Task A', attributes={}, dependencies={'prereq': 'b'}, # must match edges, below optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def'}), 'b': Task( kind='pre', label='b', attributes={}, dependencies={}, optimization={'skip-unless-has-relevant-tests': None}, task={'task': 'def2'}), }, graph=Graph(nodes={'a', 'b'}, edges={('a', 'b', 'prereq')})) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def default_matches(self, run_on_projects, project): method = target_tasks.get_method('default') graph = TaskGraph(tasks={ 'a': Task(kind='build', label='a', attributes={'run_on_projects': run_on_projects}, task={}), }, graph=Graph(nodes={'a'}, edges=set())) parameters = {'project': project} return 'a' in method(graph, parameters)
def default_matches(self, attributes, parameters): method = target_tasks.get_method("default") graph = TaskGraph( tasks={ "a": Task(kind="build", label="a", attributes=attributes, task={}), }, graph=Graph(nodes={"a"}, edges=set()), ) return "a" in method(graph, parameters, {})
def create_tasks(to_run, full_task_graph, label_to_taskid, params, decision_task_id=None): """Create new tasks. The task definition will have {relative-datestamp': '..'} rendered just like in a decision task. Action callbacks should use this function to create new tasks, allowing easy debugging with `mach taskgraph action-callback --test`. This builds up all required tasks to run in order to run the tasks requested. If you wish to create the tasks in a new group, leave out decision_task_id.""" to_run = set(to_run) target_graph = full_task_graph.graph.transitive_closure(to_run) target_task_graph = TaskGraph( {l: full_task_graph[l] for l in target_graph.nodes}, target_graph) target_task_graph.for_each_task(update_parent) optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, params, to_run, label_to_taskid) write_artifact('task-graph.json', optimized_task_graph.to_json()) write_artifact('label-to-taskid.json', label_to_taskid) write_artifact('to-run.json', list(to_run)) create.create_tasks(optimized_task_graph, label_to_taskid, params, decision_task_id)
def tg(request): if not hasattr(request.module, "TASKS"): pytest.fail( "'tg' fixture used from a module that didn't define the TASKS variable" ) tasks = request.module.TASKS for task in tasks: task.setdefault("task", {}) task["task"].setdefault("tags", {}) tasks = {t["label"]: Task(**t) for t in tasks} return TaskGraph(tasks, Graph(tasks.keys(), set()))
def make_task_graph(self): tasks = { 'a': Task(kind=None, label='a', attributes={}, task={}), 'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}, task={}), 'c': Task(kind=None, label='c', attributes={'run_on_projects': ['try']}, task={}), } graph = Graph(nodes=set('abc'), edges=set()) return TaskGraph(tasks, graph)
def make_task_graph(self): tasks = { "a": Task(kind=None, label="a", attributes={}, task={}), "b": Task(kind=None, label="b", attributes={"at-at": "yep"}, task={}), "c": Task(kind=None, label="c", attributes={"run_on_projects": ["try"]}, task={}), } graph = Graph(nodes=set("abc"), edges=set()) return TaskGraph(tasks, graph)
def test_taskgraph_to_json(self): tasks = { 'a': Task(kind='test', label='a', attributes={'attr': 'a-task'}, task={'taskdef': True}), 'b': Task(kind='test', label='b', attributes={}, task={'task': 'def'}, optimization={'seta': None}, # note that this dep is ignored, superseded by that # from the taskgraph's edges dependencies={'first': 'a'}), } graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) taskgraph = TaskGraph(tasks, graph) res = taskgraph.to_json() self.assertEqual(res, { 'a': { 'kind': 'test', 'label': 'a', 'attributes': {'attr': 'a-task', 'kind': 'test'}, 'task': {'taskdef': True}, 'dependencies': {'edgelabel': 'b'}, 'optimization': None, }, 'b': { 'kind': 'test', 'label': 'b', 'attributes': {'kind': 'test'}, 'task': {'task': 'def'}, 'dependencies': {}, 'optimization': {'seta': None}, } })
def fetch_graph_and_labels(parameters): decision_task_id = find_decision_task(parameters) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # Now fetch any modifications made by action tasks and swap out new tasks # for old ones namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format( parameters['project'], parameters['pushlog_id']) for action in list_tasks(namespace): try: run_label_to_id = get_artifact(action, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.info('Skipping {} due to missing artifact! Error: {}'.format(action, e)) continue return (decision_task_id, full_task_graph, label_to_taskid)
def release_promotion_action(parameters, input, task_group_id, task_id, task): release_promotion_flavor = input['release_promotion_flavor'] release_history = {} desktop_release_type = None next_version = str(input.get('next_version') or '') if release_promotion_flavor in VERSION_BUMP_FLAVORS: # We force str() the input, hence the 'None' if next_version in ['', 'None']: raise Exception( "`next_version` property needs to be provided for %s " "targets." % ', '.join(VERSION_BUMP_FLAVORS) ) if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS: desktop_release_type = input.get('desktop_release_type', None) if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES: raise Exception("`desktop_release_type` must be one of: %s" % ", ".join(VALID_DESKTOP_RELEASE_TYPES)) if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS: partial_updates = json.dumps(input.get('partial_updates', {})) if partial_updates == "{}": raise Exception( "`partial_updates` property needs to be provided for %s " "targets." % ', '.join(PARTIAL_UPDATES_FLAVORS) ) balrog_prefix = 'Firefox' if desktop_release_type == 'devedition': balrog_prefix = 'Devedition' os.environ['PARTIAL_UPDATES'] = partial_updates release_history = populate_release_history( balrog_prefix, parameters['project'], partial_updates=input['partial_updates'] ) if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS: uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', [])) if partial_updates == "[]": raise Exception( "`uptake_monitoring_platforms` property needs to be provided for %s " "targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS) ) os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor] target_tasks_method = input.get( 'target_tasks_method', promotion_config['target_tasks_method'].format(project=parameters['project']) ) rebuild_kinds = input.get( 'rebuild_kinds', promotion_config.get('rebuild_kinds', []) ) do_not_optimize = input.get( 'do_not_optimize', promotion_config.get('do_not_optimize', []) ) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``, # or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: revision = input.get('revision') parameters['pushlog_id'] = parameters['pushlog_id'] or \ find_hg_revision_pushlog_id(parameters, revision) previous_graph_ids = [find_decision_task(parameters)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds ) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) parameters['next_version'] = next_version parameters['release_history'] = release_history parameters['desktop_release_type'] = desktop_release_type parameters['release_eta'] = input.get('release_eta', '') # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({}, parameters=parameters)