def test_Parameters_immutable(self):
p = Parameters(**self.vals)
def assign():
p["head_ref"] = 20
self.assertRaises(Exception, assign)
def target_tasks_try_select_uncommon(full_task_graph, parameters,
graph_config):
from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS
projects = ("autoland", "mozilla-central")
if parameters["project"] not in projects:
projects = (parameters["project"], )
tasks = set()
for project in projects:
params = dict(parameters)
params["project"] = project
parameters = Parameters(**params)
try:
target_tasks_method = PER_PROJECT_PARAMETERS[project][
"target_tasks_method"]
except KeyError:
target_tasks_method = "default"
tasks.update(
get_method(target_tasks_method)(full_task_graph, parameters,
graph_config))
return sorted(tasks)
def target_tasks_try_auto(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on autoland
(rather than try) via the `run_on_projects` attributes.
Should do the same thing as the `default` target tasks method.
"""
params = dict(parameters)
params["project"] = "autoland"
parameters = Parameters(**params)
regex_filters = parameters["try_task_config"].get("tasks-regex")
include_regexes = exclude_regexes = []
if regex_filters:
include_regexes = [
re.compile(r) for r in regex_filters.get("include", [])
]
exclude_regexes = [
re.compile(r) for r in regex_filters.get("exclude", [])
]
return [
l for l, t in full_task_graph.tasks.items()
if standard_filter(t, parameters) and filter_out_shipping_phase(
t, parameters) and filter_out_devedition(t, parameters)
and filter_by_uncommon_try_tasks(t.label)
and filter_by_regex(t.label, include_regexes, mode="include")
and filter_by_regex(t.label, exclude_regexes, mode="exclude")
and filter_unsupported_artifact_builds(t, parameters)
and filter_out_shippable(t)
]
def trigger_action_callback(task_group_id,
task_id,
input,
callback,
parameters,
root,
test=False):
"""
Trigger action callback with the given inputs. If `test` is true, then run
the action callback in testing mode, without actually creating tasks.
"""
graph_config = load_graph_config(root)
graph_config.register()
callbacks = _get_callbacks(graph_config)
cb = callbacks.get(callback, None)
if not cb:
raise Exception("Unknown callback: {}. Known callbacks: {}".format(
callback, ", ".join(callbacks)))
if test:
create.testing = True
taskcluster.testing = True
if not test:
sanity_check_task_scope(callback, parameters, graph_config)
cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)
def merge_automation_action(parameters, graph_config, input, task_group_id,
task_id):
# make parameters read-write
parameters = dict(parameters)
parameters["target_tasks_method"] = "merge_automation"
parameters["merge_config"] = {
"force-dry-run": input.get("force-dry-run", False),
"behavior": input["behavior"],
}
for field in [
"from-repo",
"from-branch",
"to-repo",
"to-branch",
"ssh-user-alias",
"push",
"fetch-version-from",
]:
if input.get(field):
parameters["merge_config"][field] = input[field]
parameters["tasks_for"] = "action"
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def logfile(spec):
"""Determine logfile given a parameters specification."""
if logdir is None:
return None
return os.path.join(
logdir,
"{}_{}.log".format(options["graph_attr"], Parameters.format_spec(spec)),
)
def get_image_digest(image_name):
params = Parameters(
level=os.environ.get("MOZ_SCM_LEVEL", "3"),
strict=False,
)
tasks = load_tasks_for_kind(params, "docker-image")
task = tasks[f"docker-image-{image_name}"]
return task.attributes["cached_task"]["digest"]
def scriptworker_canary(parameters, graph_config, input, task_group_id,
task_id):
scriptworkers = input["scriptworkers"]
# make parameters read-write
parameters = dict(parameters)
parameters["target_tasks_method"] = "scriptworker_canary"
parameters["try_task_config"] = {
"scriptworker-canary-workers": scriptworkers,
}
parameters["tasks_for"] = "action"
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def dump_output(out, path=None, params_spec=None):
from gecko_taskgraph.parameters import Parameters
params_name = Parameters.format_spec(params_spec)
fh = None
if path:
# Substitute params name into file path if necessary
if params_spec and "{params}" not in path:
name, ext = os.path.splitext(path)
name += "_{params}"
path = name + ext
path = path.format(params=params_name)
fh = open(path, "w")
else:
print(
"Dumping result with parameters from {}:".format(params_name),
file=sys.stderr,
)
print(out + "\n", file=fh)
def test_Parameters_check_missing(self):
p = Parameters()
self.assertRaises(Exception, lambda: p.check())
p = Parameters(strict=False)
p.check() # should not raise
def actions_json(graph_config):
decision_task_id = "abcdef"
return render_actions_json(Parameters(strict=False), graph_config,
decision_task_id)
def run(
task_type,
release_type,
try_config=None,
push=True,
message="{msg}",
closed_tree=False,
):
if task_type == "list":
print_available_task_types()
sys.exit(0)
if release_type == "nightly":
previous_graph = get_nightly_graph()
else:
release = get_releases(RELEASE_TO_BRANCH[release_type])[-1]
previous_graph = get_release_graph(release)
existing_tasks = find_existing_tasks([previous_graph])
previous_parameters = Parameters(strict=False,
**get_artifact(previous_graph,
"public/parameters.yml"))
# Copy L10n configuration from the commit the release we are using was
# based on. This *should* ensure that the chunking of L10n tasks is the
# same between graphs.
files_to_change = {
path: get_hg_file(previous_parameters, path)
for path in [
"browser/locales/l10n-changesets.json",
"browser/locales/shipped-locales",
]
}
try_config = try_config or {}
task_config = {
"version": 2,
"parameters": {
"existing_tasks": existing_tasks,
"try_task_config": try_config,
"try_mode": "try_task_config",
},
}
for param in (
"app_version",
"build_number",
"next_version",
"release_history",
"release_product",
"release_type",
"version",
):
task_config["parameters"][param] = previous_parameters[param]
try_config["tasks"] = TASK_TYPES[task_type]
for label in try_config["tasks"]:
if label in existing_tasks:
del existing_tasks[label]
msg = "scriptworker tests: {}".format(task_type)
return push_to_try(
"scriptworker",
message.format(msg=msg),
push=push,
closed_tree=closed_tree,
try_task_config=task_config,
files_to_change=files_to_change,
)
def test_make_index_tasks(make_taskgraph, graph_config):
task_def = {
"routes": [
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.pushdate."
"2017.04.04.20170404100210.firefox-l10n.linux64-opt.zh-CN",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.es-MX",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
"index.gecko.v2.mozilla-central.revision."
"b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN",
],
"deadline":
"soon",
"metadata": {
"description": "desc",
"owner": "*****@*****.**",
"source": "https://source",
},
"extra": {
"index": {
"rank": 1540722354
},
},
}
task = Task(kind="test", label="a", attributes={}, task=task_def)
docker_task = Task(kind="docker-image",
label="docker-image-index-task",
attributes={},
task={})
taskgraph, label_to_taskid = make_taskgraph({
task.label: task,
docker_task.label: docker_task,
})
index_paths = [
r.split(".", 1)[1] for r in task_def["routes"]
if r.startswith("index.")
]
index_task = morph.make_index_task(
task,
taskgraph,
label_to_taskid,
Parameters(strict=False),
graph_config,
index_paths=index_paths,
index_rank=1540722354,
purpose="index-task",
dependencies={},
)
assert index_task.task["payload"]["command"][0] == "insert-indexes.js"
assert index_task.task["payload"]["env"]["TARGET_TASKID"] == "a-tid"
assert index_task.task["payload"]["env"]["INDEX_RANK"] == 1540722354
# check the scope summary
assert index_task.task["scopes"] == [
"index:insert-task:gecko.v2.mozilla-central.*"
]
def show_taskgraph(options):
from mozversioncontrol import get_repository_object as get_repository
from gecko_taskgraph.parameters import Parameters
if options.pop("verbose", False):
logging.root.setLevel(logging.DEBUG)
repo = None
cur_ref = None
diffdir = None
output_file = options["output_file"]
if options["diff"]:
repo = get_repository(os.getcwd())
if not repo.working_directory_clean():
print(
"abort: can't diff taskgraph with dirty working directory",
file=sys.stderr,
)
return 1
# We want to return the working directory to the current state
# as best we can after we're done. In all known cases, using
# branch or bookmark (which are both available on the VCS object)
# as `branch` is preferable to a specific revision.
cur_ref = repo.branch or repo.head_ref[:12]
diffdir = tempfile.mkdtemp()
atexit.register(
shutil.rmtree, diffdir
) # make sure the directory gets cleaned up
options["output_file"] = os.path.join(
diffdir, f"{options['graph_attr']}_{cur_ref}"
)
print(f"Generating {options['graph_attr']} @ {cur_ref}", file=sys.stderr)
parameters: List[Any[str, Parameters]] = options.pop("parameters")
if not parameters:
kwargs = {
"target-kind": options.get("target_kind"),
}
parameters = [Parameters(strict=False, **kwargs)] # will use default values
for param in parameters[:]:
if isinstance(param, str) and os.path.isdir(param):
parameters.remove(param)
parameters.extend(
[
p.as_posix()
for p in Path(param).iterdir()
if p.suffix in (".yml", ".json")
]
)
logdir = None
if len(parameters) > 1:
# Log to separate files for each process instead of stderr to
# avoid interleaving.
basename = os.path.basename(os.getcwd())
logdir = os.path.join(appdirs.user_log_dir("taskgraph"), basename)
if not os.path.isdir(logdir):
os.makedirs(logdir)
else:
# Only setup logging if we have a single parameter spec. Otherwise
# logging will go to files. This is also used as a hook for Gecko
# to setup its `mach` based logging.
setup_logging()
generate_taskgraph(options, parameters, logdir)
if options["diff"]:
assert diffdir is not None
assert repo is not None
# Some transforms use global state for checks, so will fail
# when running taskgraph a second time in the same session.
# Reload all taskgraph modules to avoid this.
for mod in sys.modules.copy():
if mod != __name__ and mod.startswith("taskgraph"):
del sys.modules[mod]
if options["diff"] == "default":
base_ref = repo.base_ref
else:
base_ref = options["diff"]
try:
repo.update(base_ref)
base_ref = repo.head_ref[:12]
options["output_file"] = os.path.join(
diffdir, f"{options['graph_attr']}_{base_ref}"
)
print(f"Generating {options['graph_attr']} @ {base_ref}", file=sys.stderr)
generate_taskgraph(options, parameters, logdir)
finally:
repo.update(cur_ref)
# Generate diff(s)
diffcmd = [
"diff",
"-U20",
"--report-identical-files",
f"--label={options['graph_attr']}@{base_ref}",
f"--label={options['graph_attr']}@{cur_ref}",
]
non_fatal_failures = []
for spec in parameters:
base_path = os.path.join(diffdir, f"{options['graph_attr']}_{base_ref}")
cur_path = os.path.join(diffdir, f"{options['graph_attr']}_{cur_ref}")
params_name = None
if len(parameters) > 1:
params_name = Parameters.format_spec(spec)
base_path += f"_{params_name}"
cur_path += f"_{params_name}"
# We only capture errors when the 'base' generation fails. This is
# because if the 'current' generation passed, the failure is likely
# due to a difference in the set of revisions being tested and
# harmless. We'll still log a warning to notify that the diff is
# not available. But if the current generation failed, the error
# needs to be addressed.
if not os.path.isfile(base_path):
non_fatal_failures.append(os.path.basename(base_path))
continue
try:
proc = subprocess.run(
diffcmd + [base_path, cur_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
check=True,
)
diff_output = proc.stdout
returncode = 0
except subprocess.CalledProcessError as e:
# returncode 1 simply means diffs were found
if e.returncode != 1:
print(e.stderr, file=sys.stderr)
raise
diff_output = e.output
returncode = e.returncode
dump_output(
diff_output,
# Don't bother saving file if no diffs were found. Log to
# console in this case instead.
path=None if returncode == 0 else output_file,
params_spec=spec if len(parameters) > 1 else None,
)
if non_fatal_failures:
failstr = "\n ".join(sorted(non_fatal_failures))
print(
"WARNING: Diff skipped for the following generation{s} "
"due to failures:\n {failstr}".format(
s="s" if len(non_fatal_failures) > 1 else "", failstr=failstr
),
file=sys.stderr,
)
if options["format"] != "json":
print(
"If you were expecting differences in task bodies "
'you should pass "-J"\n',
file=sys.stderr,
)
if len(parameters) > 1:
print("See '{}' for logs".format(logdir), file=sys.stderr)
def test_Parameters_missing_KeyError(self):
p = Parameters(**self.vals)
self.assertRaises(KeyError, lambda: p["z"])
def test_Parameters_invalid_KeyError(self):
"""even if the value is present, if it's not a valid property, raise KeyError"""
p = Parameters(xyz=10, strict=True, **self.vals)
self.assertRaises(Exception, lambda: p.check())
def test_Parameters_get(self):
p = Parameters(head_ref=10, level=20)
self.assertEqual(p["head_ref"], 10)
def test_Parameters_check(self):
p = Parameters(**self.vals)
p.check() # should not raise
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
release_promotion_flavor = input["release_promotion_flavor"]
promotion_config = graph_config["release-promotion"]["flavors"][
release_promotion_flavor
]
release_history = {}
product = promotion_config["product"]
next_version = str(input.get("next_version") or "")
if promotion_config.get("version-bump", False):
# We force str() the input, hence the 'None'
if next_version in ["", "None"]:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor)
)
if promotion_config.get("partial-updates", False):
partial_updates = input.get("partial_updates", {})
if not partial_updates and release_level(parameters["project"]) == "production":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor)
)
balrog_prefix = product.title()
os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True)
release_history = populate_release_history(
balrog_prefix, parameters["project"], partial_updates=partial_updates
)
target_tasks_method = promotion_config["target-tasks-method"].format(
project=parameters["project"]
)
rebuild_kinds = input.get(
"rebuild_kinds", promotion_config.get("rebuild-kinds", [])
)
do_not_optimize = input.get(
"do_not_optimize", promotion_config.get("do-not-optimize", [])
)
# Build previous_graph_ids from ``previous_graph_ids``, ``revision``,
# or the action parameters.
previous_graph_ids = input.get("previous_graph_ids")
if not previous_graph_ids:
revision = input.get("revision")
if revision:
head_rev_param = "{}head_rev".format(
graph_config["project-repo-param-prefix"]
)
push_parameters = {
head_rev_param: revision,
"project": parameters["project"],
}
else:
push_parameters = parameters
previous_graph_ids = [find_decision_task(push_parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters["do_not_optimize"] = do_not_optimize
parameters["target_tasks_method"] = target_tasks_method
parameters["build_number"] = int(input["build_number"])
parameters["next_version"] = next_version
parameters["release_history"] = release_history
if promotion_config.get("is-rc"):
parameters["release_type"] += "-rc"
parameters["release_eta"] = input.get("release_eta", "")
parameters["release_product"] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters["optimize_target_tasks"] = True
if release_promotion_flavor == "promote_firefox_partner_repack":
release_enable_partner_repack = True
release_enable_partner_attribution = False
release_enable_emefree = False
elif release_promotion_flavor == "promote_firefox_partner_attribution":
release_enable_partner_repack = False
release_enable_partner_attribution = True
release_enable_emefree = False
else:
# for promotion or ship phases, we use the action input to turn the repacks/attribution off
release_enable_partner_repack = input.get("release_enable_partner_repack", True)
release_enable_partner_attribution = input.get(
"release_enable_partner_attribution", True
)
release_enable_emefree = input.get("release_enable_emefree", True)
partner_url_config = get_partner_url_config(parameters, graph_config)
if (
release_enable_partner_repack
and not partner_url_config["release-partner-repack"]
):
raise Exception("Can't enable partner repacks when no config url found")
if (
release_enable_partner_attribution
and not partner_url_config["release-partner-attribution"]
):
raise Exception("Can't enable partner attribution when no config url found")
if release_enable_emefree and not partner_url_config["release-eme-free-repack"]:
raise Exception("Can't enable EMEfree repacks when no config url found")
parameters["release_enable_partner_repack"] = release_enable_partner_repack
parameters[
"release_enable_partner_attribution"
] = release_enable_partner_attribution
parameters["release_enable_emefree"] = release_enable_emefree
partner_config = input.get("release_partner_config")
if not partner_config and any(
[
release_enable_partner_repack,
release_enable_partner_attribution,
release_enable_emefree,
]
):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters["release_partner_config"] = fix_partner_config(partner_config)
parameters["release_partners"] = input.get("release_partners")
if input.get("release_partner_build_number"):
parameters["release_partner_build_number"] = input[
"release_partner_build_number"
]
if input["version"]:
parameters["version"] = input["version"]
parameters["required_signoffs"] = get_required_signoffs(input, parameters)
parameters["signoff_urls"] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def test_Parameters_check_extra(self):
p = Parameters(xyz=10, **self.vals)
self.assertRaises(Exception, lambda: p.check())
p = Parameters(strict=False, xyz=10, **self.vals)
p.check() # should not raise
