def test_round_trip(self): graph = TaskGraph( tasks={ "a": Task( kind="fancy", label="a", description="Task A", attributes={}, dependencies={"prereq": "b"}, # must match edges, below optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def"}, ), "b": Task( kind="pre", label="b", attributes={}, dependencies={}, optimization={"skip-unless-has-relevant-tests": None}, task={"task": "def2"}, ), }, graph=Graph(nodes={"a", "b"}, edges={("a", "b", "prereq")}), ) tasks, new_graph = TaskGraph.from_json(graph.to_json()) self.assertEqual(graph, new_graph)
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): release_promotion_flavor = input["release_promotion_flavor"] promotion_config = graph_config["release-promotion"]["flavors"][ release_promotion_flavor ] release_history = {} product = promotion_config["product"] next_version = str(input.get("next_version") or "") if promotion_config.get("version-bump", False): # We force str() the input, hence the 'None' if next_version in ["", "None"]: raise Exception( "`next_version` property needs to be provided for `{}` " "target.".format(release_promotion_flavor) ) if promotion_config.get("partial-updates", False): partial_updates = input.get("partial_updates", {}) if not partial_updates and release_level(parameters["project"]) == "production": raise Exception( "`partial_updates` property needs to be provided for `{}`" "target.".format(release_promotion_flavor) ) balrog_prefix = product.title() os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True) release_history = populate_release_history( balrog_prefix, parameters["project"], partial_updates=partial_updates ) target_tasks_method = promotion_config["target-tasks-method"].format( project=parameters["project"] ) rebuild_kinds = input.get( "rebuild_kinds", promotion_config.get("rebuild-kinds", []) ) do_not_optimize = input.get( "do_not_optimize", promotion_config.get("do-not-optimize", []) ) # Build previous_graph_ids from ``previous_graph_ids``, ``revision``, # or the action parameters. previous_graph_ids = input.get("previous_graph_ids") if not previous_graph_ids: revision = input.get("revision") if revision: head_rev_param = "{}head_rev".format( graph_config["project-repo-param-prefix"] ) push_parameters = { head_rev_param: revision, "project": parameters["project"], } else: push_parameters = parameters previous_graph_ids = [find_decision_task(push_parameters, graph_config)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds ) parameters["do_not_optimize"] = do_not_optimize parameters["target_tasks_method"] = target_tasks_method parameters["build_number"] = int(input["build_number"]) parameters["next_version"] = next_version parameters["release_history"] = release_history if promotion_config.get("is-rc"): parameters["release_type"] += "-rc" parameters["release_eta"] = input.get("release_eta", "") parameters["release_product"] = product # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters["optimize_target_tasks"] = True if release_promotion_flavor == "promote_firefox_partner_repack": release_enable_partner_repack = True release_enable_partner_attribution = False release_enable_emefree = False elif release_promotion_flavor == "promote_firefox_partner_attribution": release_enable_partner_repack = False release_enable_partner_attribution = True release_enable_emefree = False else: # for promotion or ship phases, we use the action input to turn the repacks/attribution off release_enable_partner_repack = input.get("release_enable_partner_repack", True) release_enable_partner_attribution = input.get( "release_enable_partner_attribution", True ) release_enable_emefree = input.get("release_enable_emefree", True) partner_url_config = get_partner_url_config(parameters, graph_config) if ( release_enable_partner_repack and not partner_url_config["release-partner-repack"] ): raise Exception("Can't enable partner repacks when no config url found") if ( release_enable_partner_attribution and not partner_url_config["release-partner-attribution"] ): raise Exception("Can't enable partner attribution when no config url found") if release_enable_emefree and not partner_url_config["release-eme-free-repack"]: raise Exception("Can't enable EMEfree repacks when no config url found") parameters["release_enable_partner_repack"] = release_enable_partner_repack parameters[ "release_enable_partner_attribution" ] = release_enable_partner_attribution parameters["release_enable_emefree"] = release_enable_emefree partner_config = input.get("release_partner_config") if not partner_config and any( [ release_enable_partner_repack, release_enable_partner_attribution, release_enable_emefree, ] ): github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if partner_config: parameters["release_partner_config"] = fix_partner_config(partner_config) parameters["release_partners"] = input.get("release_partners") if input.get("release_partner_build_number"): parameters["release_partner_build_number"] = input[ "release_partner_build_number" ] if input["version"]: parameters["version"] = input["version"] parameters["required_signoffs"] = get_required_signoffs(input, parameters) parameters["signoff_urls"] = get_signoff_urls(input, parameters) # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def fetch_graph_and_labels(parameters, graph_config): decision_task_id = find_decision_task(parameters, graph_config) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") logger.info("Load taskgraph from JSON.") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") logger.info("Fetching additional tasks from action and cron tasks.") # fetch everything in parallel; this avoids serializing any delay in downloading # each artifact (such as waiting for the artifact to be mirrored locally) with futures.ThreadPoolExecutor(CONCURRENCY) as e: fetches = [] # fetch any modifications made by action tasks and swap out new tasks # for old ones def fetch_action(task_id): logger.info( f"fetching label-to-taskid.json for action task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug( f"No label-to-taskid.json found for {task_id}: {e}") head_rev_param = "{}head_rev".format( graph_config["project-repo-param-prefix"]) namespace = "{}.v2.{}.revision.{}.taskgraph.actions".format( graph_config["trust-domain"], parameters["project"], parameters[head_rev_param], ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_action, task_id)) # Similarly for cron tasks.. def fetch_cron(task_id): logger.info( f"fetching label-to-taskid.json for cron task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug( f"No label-to-taskid.json found for {task_id}: {e}") namespace = "{}.v2.{}.revision.{}.cron".format( graph_config["trust-domain"], parameters["project"], parameters[head_rev_param], ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_cron, task_id)) # now wait for each fetch to complete, raising an exception if there # were any issues for f in futures.as_completed(fetches): f.result() return (decision_task_id, full_task_graph, label_to_taskid)
def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id): task = taskcluster.get_task_definition(task_id) label = task["metadata"]["name"] pushes = [] depth = 2 end_id = int(parameters["pushlog_id"]) while True: start_id = max(end_id - depth, 0) pushlog_url = PUSHLOG_TMPL.format(parameters["head_repository"], start_id, end_id) r = requests.get(pushlog_url) r.raise_for_status() pushes = pushes + list(r.json()["pushes"].keys()) if len(pushes) >= depth: break end_id = start_id - 1 start_id -= depth if start_id < 0: break pushes = sorted(pushes)[-depth:] backfill_pushes = [] for push in pushes: try: full_task_graph = get_artifact_from_index( INDEX_TMPL.format(parameters["project"], push), "public/full-task-graph.json", ) _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact_from_index( INDEX_TMPL.format(parameters["project"], push), "public/label-to-taskid.json", ) push_params = get_artifact_from_index( INDEX_TMPL.format(parameters["project"], push), "public/parameters.yml") push_decision_task_id = find_decision_task(push_params, graph_config) except HTTPError as e: logger.info( f"Skipping {push} due to missing index artifacts! Error: {e}") continue if label in full_task_graph.tasks.keys(): def modifier(task): if task.label != label: return task cmd = task.task["payload"]["command"] task.task["payload"]["command"] = add_args_to_perf_command( cmd, ["--gecko-profile"]) task.task["extra"]["treeherder"]["symbol"] += "-p" task.task["extra"]["treeherder"]["groupName"] += " (profiling)" return task create_tasks( graph_config, [label], full_task_graph, label_to_taskid, push_params, push_decision_task_id, push, modifier=modifier, ) backfill_pushes.append(push) else: logging.info(f"Could not find {label} on {push}. Skipping.") combine_task_graph_files(backfill_pushes)
def generate_tasks(params=None, full=False, disable_target_task_filter=False): attr = "full_task_set" if full else "target_task_set" target_tasks_method = ( "try_select_tasks" if not disable_target_task_filter else "try_select_tasks_uncommon" ) params = parameters_loader( params, strict=False, overrides={ "try_mode": "try_select", "target_tasks_method": target_tasks_method, }, ) root = os.path.join(build.topsrcdir, "taskcluster", "ci") gecko_taskgraph.fast = True generator = TaskGraphGenerator(root_dir=root, parameters=params) cache_dir = os.path.join(get_state_dir(srcdir=True), "cache", "taskgraph") key = cache_key(attr, generator.parameters, disable_target_task_filter) cache = os.path.join(cache_dir, key) invalidate(cache) if os.path.isfile(cache): with open(cache) as fh: return TaskGraph.from_json(json.load(fh))[1] if not os.path.isdir(cache_dir): os.makedirs(cache_dir) print("Task configuration changed, generating {}".format(attr.replace("_", " "))) cwd = os.getcwd() os.chdir(build.topsrcdir) def generate(attr): try: tg = getattr(generator, attr) except ParameterMismatch as e: print(PARAMETER_MISMATCH.format(e.args[0])) sys.exit(1) # write cache key = cache_key(attr, generator.parameters, disable_target_task_filter) with open(os.path.join(cache_dir, key), "w") as fh: json.dump(tg.to_json(), fh) return tg # Cache both full_task_set and target_task_set regardless of whether or not # --full was requested. Caching is cheap and can potentially save a lot of # time. tg_full = generate("full_task_set") tg_target = generate("target_task_set") # discard results from these, we only need cache. if full: generate("full_task_graph") generate("target_task_graph") os.chdir(cwd) if full: return tg_full return tg_target