def test_Parameters_immutable(self): p = Parameters(**self.vals) def assign(): p["head_ref"] = 20 self.assertRaises(Exception, assign)
def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS projects = ("autoland", "mozilla-central") if parameters["project"] not in projects: projects = (parameters["project"], ) tasks = set() for project in projects: params = dict(parameters) params["project"] = project parameters = Parameters(**params) try: target_tasks_method = PER_PROJECT_PARAMETERS[project][ "target_tasks_method"] except KeyError: target_tasks_method = "default" tasks.update( get_method(target_tasks_method)(full_task_graph, parameters, graph_config)) return sorted(tasks)
def target_tasks_try_auto(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on autoland (rather than try) via the `run_on_projects` attributes. Should do the same thing as the `default` target tasks method. """ params = dict(parameters) params["project"] = "autoland" parameters = Parameters(**params) regex_filters = parameters["try_task_config"].get("tasks-regex") include_regexes = exclude_regexes = [] if regex_filters: include_regexes = [ re.compile(r) for r in regex_filters.get("include", []) ] exclude_regexes = [ re.compile(r) for r in regex_filters.get("exclude", []) ] return [ l for l, t in full_task_graph.tasks.items() if standard_filter(t, parameters) and filter_out_shipping_phase( t, parameters) and filter_out_devedition(t, parameters) and filter_by_uncommon_try_tasks(t.label) and filter_by_regex(t.label, include_regexes, mode="include") and filter_by_regex(t.label, exclude_regexes, mode="exclude") and filter_unsupported_artifact_builds(t, parameters) and filter_out_shippable(t) ]
def trigger_action_callback(task_group_id, task_id, input, callback, parameters, root, test=False): """ Trigger action callback with the given inputs. If `test` is true, then run the action callback in testing mode, without actually creating tasks. """ graph_config = load_graph_config(root) graph_config.register() callbacks = _get_callbacks(graph_config) cb = callbacks.get(callback, None) if not cb: raise Exception("Unknown callback: {}. Known callbacks: {}".format( callback, ", ".join(callbacks))) if test: create.testing = True taskcluster.testing = True if not test: sanity_check_task_scope(callback, parameters, graph_config) cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)
def merge_automation_action(parameters, graph_config, input, task_group_id, task_id): # make parameters read-write parameters = dict(parameters) parameters["target_tasks_method"] = "merge_automation" parameters["merge_config"] = { "force-dry-run": input.get("force-dry-run", False), "behavior": input["behavior"], } for field in [ "from-repo", "from-branch", "to-repo", "to-branch", "ssh-user-alias", "push", "fetch-version-from", ]: if input.get(field): parameters["merge_config"][field] = input[field] parameters["tasks_for"] = "action" # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def logfile(spec): """Determine logfile given a parameters specification.""" if logdir is None: return None return os.path.join( logdir, "{}_{}.log".format(options["graph_attr"], Parameters.format_spec(spec)), )
def get_image_digest(image_name): params = Parameters( level=os.environ.get("MOZ_SCM_LEVEL", "3"), strict=False, ) tasks = load_tasks_for_kind(params, "docker-image") task = tasks[f"docker-image-{image_name}"] return task.attributes["cached_task"]["digest"]
def scriptworker_canary(parameters, graph_config, input, task_group_id, task_id): scriptworkers = input["scriptworkers"] # make parameters read-write parameters = dict(parameters) parameters["target_tasks_method"] = "scriptworker_canary" parameters["try_task_config"] = { "scriptworker-canary-workers": scriptworkers, } parameters["tasks_for"] = "action" # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def dump_output(out, path=None, params_spec=None): from gecko_taskgraph.parameters import Parameters params_name = Parameters.format_spec(params_spec) fh = None if path: # Substitute params name into file path if necessary if params_spec and "{params}" not in path: name, ext = os.path.splitext(path) name += "_{params}" path = name + ext path = path.format(params=params_name) fh = open(path, "w") else: print( "Dumping result with parameters from {}:".format(params_name), file=sys.stderr, ) print(out + "\n", file=fh)
def test_Parameters_check_missing(self): p = Parameters() self.assertRaises(Exception, lambda: p.check()) p = Parameters(strict=False) p.check() # should not raise
def actions_json(graph_config): decision_task_id = "abcdef" return render_actions_json(Parameters(strict=False), graph_config, decision_task_id)
def run( task_type, release_type, try_config=None, push=True, message="{msg}", closed_tree=False, ): if task_type == "list": print_available_task_types() sys.exit(0) if release_type == "nightly": previous_graph = get_nightly_graph() else: release = get_releases(RELEASE_TO_BRANCH[release_type])[-1] previous_graph = get_release_graph(release) existing_tasks = find_existing_tasks([previous_graph]) previous_parameters = Parameters(strict=False, **get_artifact(previous_graph, "public/parameters.yml")) # Copy L10n configuration from the commit the release we are using was # based on. This *should* ensure that the chunking of L10n tasks is the # same between graphs. files_to_change = { path: get_hg_file(previous_parameters, path) for path in [ "browser/locales/l10n-changesets.json", "browser/locales/shipped-locales", ] } try_config = try_config or {} task_config = { "version": 2, "parameters": { "existing_tasks": existing_tasks, "try_task_config": try_config, "try_mode": "try_task_config", }, } for param in ( "app_version", "build_number", "next_version", "release_history", "release_product", "release_type", "version", ): task_config["parameters"][param] = previous_parameters[param] try_config["tasks"] = TASK_TYPES[task_type] for label in try_config["tasks"]: if label in existing_tasks: del existing_tasks[label] msg = "scriptworker tests: {}".format(task_type) return push_to_try( "scriptworker", message.format(msg=msg), push=push, closed_tree=closed_tree, try_task_config=task_config, files_to_change=files_to_change, )
def test_make_index_tasks(make_taskgraph, graph_config): task_def = { "routes": [ "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.pushdate." "2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk", "index.gecko.v2.mozilla-central.revision." "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN", ], "deadline": "soon", "metadata": { "description": "desc", "owner": "*****@*****.**", "source": "https://source", }, "extra": { "index": { "rank": 1540722354 }, }, } task = Task(kind="test", label="a", attributes={}, task=task_def) docker_task = Task(kind="docker-image", label="docker-image-index-task", attributes={}, task={}) taskgraph, label_to_taskid = make_taskgraph({ task.label: task, docker_task.label: docker_task, }) index_paths = [ r.split(".", 1)[1] for r in task_def["routes"] if r.startswith("index.") ] index_task = morph.make_index_task( task, taskgraph, label_to_taskid, Parameters(strict=False), graph_config, index_paths=index_paths, index_rank=1540722354, purpose="index-task", dependencies={}, ) assert index_task.task["payload"]["command"][0] == "insert-indexes.js" assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid" assert index_task.task["payload"]["env"]["INDEX_RANK"] == 1540722354 # check the scope summary assert index_task.task["scopes"] == [ "index:insert-task:gecko.v2.mozilla-central.*" ]
def show_taskgraph(options): from mozversioncontrol import get_repository_object as get_repository from gecko_taskgraph.parameters import Parameters if options.pop("verbose", False): logging.root.setLevel(logging.DEBUG) repo = None cur_ref = None diffdir = None output_file = options["output_file"] if options["diff"]: repo = get_repository(os.getcwd()) if not repo.working_directory_clean(): print( "abort: can't diff taskgraph with dirty working directory", file=sys.stderr, ) return 1 # We want to return the working directory to the current state # as best we can after we're done. In all known cases, using # branch or bookmark (which are both available on the VCS object) # as `branch` is preferable to a specific revision. cur_ref = repo.branch or repo.head_ref[:12] diffdir = tempfile.mkdtemp() atexit.register( shutil.rmtree, diffdir ) # make sure the directory gets cleaned up options["output_file"] = os.path.join( diffdir, f"{options['graph_attr']}_{cur_ref}" ) print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr) parameters: List[Any[str, Parameters]] = options.pop("parameters") if not parameters: kwargs = { "target-kind": options.get("target_kind"), } parameters = [Parameters(strict=False, **kwargs)] # will use default values for param in parameters[:]: if isinstance(param, str) and os.path.isdir(param): parameters.remove(param) parameters.extend( [ p.as_posix() for p in Path(param).iterdir() if p.suffix in (".yml", ".json") ] ) logdir = None if len(parameters) > 1: # Log to separate files for each process instead of stderr to # avoid interleaving. basename = os.path.basename(os.getcwd()) logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename) if not os.path.isdir(logdir): os.makedirs(logdir) else: # Only setup logging if we have a single parameter spec. Otherwise # logging will go to files. This is also used as a hook for Gecko # to setup its `mach` based logging. setup_logging() generate_taskgraph(options, parameters, logdir) if options["diff"]: assert diffdir is not None assert repo is not None # Some transforms use global state for checks, so will fail # when running taskgraph a second time in the same session. # Reload all taskgraph modules to avoid this. for mod in sys.modules.copy(): if mod != __name__ and mod.startswith("taskgraph"): del sys.modules[mod] if options["diff"] == "default": base_ref = repo.base_ref else: base_ref = options["diff"] try: repo.update(base_ref) base_ref = repo.head_ref[:12] options["output_file"] = os.path.join( diffdir, f"{options['graph_attr']}_{base_ref}" ) print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr) generate_taskgraph(options, parameters, logdir) finally: repo.update(cur_ref) # Generate diff(s) diffcmd = [ "diff", "-U20", "--report-identical-files", f"--label={options['graph_attr']}@{base_ref}", f"--label={options['graph_attr']}@{cur_ref}", ] non_fatal_failures = [] for spec in parameters: base_path = os.path.join(diffdir, f"{options['graph_attr']}_{base_ref}") cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_ref}") params_name = None if len(parameters) > 1: params_name = Parameters.format_spec(spec) base_path += f"_{params_name}" cur_path += f"_{params_name}" # We only capture errors when the 'base' generation fails. This is # because if the 'current' generation passed, the failure is likely # due to a difference in the set of revisions being tested and # harmless. We'll still log a warning to notify that the diff is # not available. But if the current generation failed, the error # needs to be addressed. if not os.path.isfile(base_path): non_fatal_failures.append(os.path.basename(base_path)) continue try: proc = subprocess.run( diffcmd + [base_path, cur_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=True, ) diff_output = proc.stdout returncode = 0 except subprocess.CalledProcessError as e: # returncode 1 simply means diffs were found if e.returncode != 1: print(e.stderr, file=sys.stderr) raise diff_output = e.output returncode = e.returncode dump_output( diff_output, # Don't bother saving file if no diffs were found. Log to # console in this case instead. path=None if returncode == 0 else output_file, params_spec=spec if len(parameters) > 1 else None, ) if non_fatal_failures: failstr = "\n ".join(sorted(non_fatal_failures)) print( "WARNING: Diff skipped for the following generation{s} " "due to failures:\n {failstr}".format( s="s" if len(non_fatal_failures) > 1 else "", failstr=failstr ), file=sys.stderr, ) if options["format"] != "json": print( "If you were expecting differences in task bodies " 'you should pass "-J"\n', file=sys.stderr, ) if len(parameters) > 1: print("See '{}' for logs".format(logdir), file=sys.stderr)
def test_Parameters_missing_KeyError(self): p = Parameters(**self.vals) self.assertRaises(KeyError, lambda: p["z"])
def test_Parameters_invalid_KeyError(self): """even if the value is present, if it's not a valid property, raise KeyError""" p = Parameters(xyz=10, strict=True, **self.vals) self.assertRaises(Exception, lambda: p.check())
def test_Parameters_get(self): p = Parameters(head_ref=10, level=20) self.assertEqual(p["head_ref"], 10)
def test_Parameters_check(self): p = Parameters(**self.vals) p.check() # should not raise
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): release_promotion_flavor = input["release_promotion_flavor"] promotion_config = graph_config["release-promotion"]["flavors"][ release_promotion_flavor ] release_history = {} product = promotion_config["product"] next_version = str(input.get("next_version") or "") if promotion_config.get("version-bump", False): # We force str() the input, hence the 'None' if next_version in ["", "None"]: raise Exception( "`next_version` property needs to be provided for `{}` " "target.".format(release_promotion_flavor) ) if promotion_config.get("partial-updates", False): partial_updates = input.get("partial_updates", {}) if not partial_updates and release_level(parameters["project"]) == "production": raise Exception( "`partial_updates` property needs to be provided for `{}`" "target.".format(release_promotion_flavor) ) balrog_prefix = product.title() os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True) release_history = populate_release_history( balrog_prefix, parameters["project"], partial_updates=partial_updates ) target_tasks_method = promotion_config["target-tasks-method"].format( project=parameters["project"] ) rebuild_kinds = input.get( "rebuild_kinds", promotion_config.get("rebuild-kinds", []) ) do_not_optimize = input.get( "do_not_optimize", promotion_config.get("do-not-optimize", []) ) # Build previous_graph_ids from ``previous_graph_ids``, ``revision``, # or the action parameters. previous_graph_ids = input.get("previous_graph_ids") if not previous_graph_ids: revision = input.get("revision") if revision: head_rev_param = "{}head_rev".format( graph_config["project-repo-param-prefix"] ) push_parameters = { head_rev_param: revision, "project": parameters["project"], } else: push_parameters = parameters previous_graph_ids = [find_decision_task(push_parameters, graph_config)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds ) parameters["do_not_optimize"] = do_not_optimize parameters["target_tasks_method"] = target_tasks_method parameters["build_number"] = int(input["build_number"]) parameters["next_version"] = next_version parameters["release_history"] = release_history if promotion_config.get("is-rc"): parameters["release_type"] += "-rc" parameters["release_eta"] = input.get("release_eta", "") parameters["release_product"] = product # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters["optimize_target_tasks"] = True if release_promotion_flavor == "promote_firefox_partner_repack": release_enable_partner_repack = True release_enable_partner_attribution = False release_enable_emefree = False elif release_promotion_flavor == "promote_firefox_partner_attribution": release_enable_partner_repack = False release_enable_partner_attribution = True release_enable_emefree = False else: # for promotion or ship phases, we use the action input to turn the repacks/attribution off release_enable_partner_repack = input.get("release_enable_partner_repack", True) release_enable_partner_attribution = input.get( "release_enable_partner_attribution", True ) release_enable_emefree = input.get("release_enable_emefree", True) partner_url_config = get_partner_url_config(parameters, graph_config) if ( release_enable_partner_repack and not partner_url_config["release-partner-repack"] ): raise Exception("Can't enable partner repacks when no config url found") if ( release_enable_partner_attribution and not partner_url_config["release-partner-attribution"] ): raise Exception("Can't enable partner attribution when no config url found") if release_enable_emefree and not partner_url_config["release-eme-free-repack"]: raise Exception("Can't enable EMEfree repacks when no config url found") parameters["release_enable_partner_repack"] = release_enable_partner_repack parameters[ "release_enable_partner_attribution" ] = release_enable_partner_attribution parameters["release_enable_emefree"] = release_enable_emefree partner_config = input.get("release_partner_config") if not partner_config and any( [ release_enable_partner_repack, release_enable_partner_attribution, release_enable_emefree, ] ): github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if partner_config: parameters["release_partner_config"] = fix_partner_config(partner_config) parameters["release_partners"] = input.get("release_partners") if input.get("release_partner_build_number"): parameters["release_partner_build_number"] = input[ "release_partner_build_number" ] if input["version"]: parameters["version"] = input["version"] parameters["required_signoffs"] = get_required_signoffs(input, parameters) parameters["signoff_urls"] = get_signoff_urls(input, parameters) # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def test_Parameters_check_extra(self): p = Parameters(xyz=10, **self.vals) self.assertRaises(Exception, lambda: p.check()) p = Parameters(strict=False, xyz=10, **self.vals) p.check() # should not raise