def trigger_action(action_name, decision_task_id, task_id=None, input={}):
if not decision_task_id:
raise ValueError(
"No decision task. We can't find the actions artifact.")
actions_json = get_artifact(decision_task_id, "public/actions.json")
if actions_json["version"] != 1:
raise RuntimeError("Wrong version of actions.json, unable to continue")
# These values substitute $eval in the template
context = {
"input": input,
"taskId": task_id,
"taskGroupId": decision_task_id,
}
# https://docs.taskcluster.net/docs/manual/design/conventions/actions/spec#variables
context.update(actions_json["variables"])
action = _extract_applicable_action(actions_json, action_name,
decision_task_id, task_id)
kind = action["kind"]
if kind == "hook":
hook_payload = jsone.render(action["hookPayload"], context)
trigger_hook(action["hookGroupId"], action["hookId"], hook_payload)
else:
raise NotImplementedError(
f"Unable to submit actions with {kind} kind.")
def run_missing_tests(parameters, graph_config, input, task_group_id, task_id):
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
parameters, graph_config
)
target_tasks = get_artifact(decision_task_id, "public/target-tasks.json")
# The idea here is to schedule all tasks of the `test` kind that were
# targetted but did not appear in the final task-graph -- those were the
# optimized tasks.
to_run = []
already_run = 0
for label in target_tasks:
task = full_task_graph.tasks[label]
if task.kind != "test":
continue # not a test
if label in label_to_taskid:
already_run += 1
continue
to_run.append(label)
create_tasks(
graph_config,
to_run,
full_task_graph,
label_to_taskid,
parameters,
decision_task_id,
)
logger.info(
"Out of {} test tasks, {} already existed and the action created {}".format(
already_run + len(to_run), already_run, len(to_run)
)
)
def find_existing_tasks(previous_graph_ids):
existing_tasks = {}
for previous_graph_id in previous_graph_ids:
label_to_taskid = get_artifact(previous_graph_id,
"public/label-to-taskid.json")
existing_tasks.update(label_to_taskid)
return existing_tasks
def fetch_cron(task_id):
logger.info(
f"fetching label-to-taskid.json for cron task {task_id}")
try:
run_label_to_id = get_artifact(task_id,
"public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
if e.response.status_code != 404:
raise
logger.debug(
f"No label-to-taskid.json found for {task_id}: {e}")
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
release_promotion_flavor = input["release_promotion_flavor"]
promotion_config = graph_config["release-promotion"]["flavors"][
release_promotion_flavor
]
release_history = {}
product = promotion_config["product"]
next_version = str(input.get("next_version") or "")
if promotion_config.get("version-bump", False):
# We force str() the input, hence the 'None'
if next_version in ["", "None"]:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor)
)
if promotion_config.get("partial-updates", False):
partial_updates = input.get("partial_updates", {})
if not partial_updates and release_level(parameters["project"]) == "production":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor)
)
balrog_prefix = product.title()
os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True)
release_history = populate_release_history(
balrog_prefix, parameters["project"], partial_updates=partial_updates
)
target_tasks_method = promotion_config["target-tasks-method"].format(
project=parameters["project"]
)
rebuild_kinds = input.get(
"rebuild_kinds", promotion_config.get("rebuild-kinds", [])
)
do_not_optimize = input.get(
"do_not_optimize", promotion_config.get("do-not-optimize", [])
)
# Build previous_graph_ids from ``previous_graph_ids``, ``revision``,
# or the action parameters.
previous_graph_ids = input.get("previous_graph_ids")
if not previous_graph_ids:
revision = input.get("revision")
if revision:
head_rev_param = "{}head_rev".format(
graph_config["project-repo-param-prefix"]
)
push_parameters = {
head_rev_param: revision,
"project": parameters["project"],
}
else:
push_parameters = parameters
previous_graph_ids = [find_decision_task(push_parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters["do_not_optimize"] = do_not_optimize
parameters["target_tasks_method"] = target_tasks_method
parameters["build_number"] = int(input["build_number"])
parameters["next_version"] = next_version
parameters["release_history"] = release_history
if promotion_config.get("is-rc"):
parameters["release_type"] += "-rc"
parameters["release_eta"] = input.get("release_eta", "")
parameters["release_product"] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters["optimize_target_tasks"] = True
if release_promotion_flavor == "promote_firefox_partner_repack":
release_enable_partner_repack = True
release_enable_partner_attribution = False
release_enable_emefree = False
elif release_promotion_flavor == "promote_firefox_partner_attribution":
release_enable_partner_repack = False
release_enable_partner_attribution = True
release_enable_emefree = False
else:
# for promotion or ship phases, we use the action input to turn the repacks/attribution off
release_enable_partner_repack = input.get("release_enable_partner_repack", True)
release_enable_partner_attribution = input.get(
"release_enable_partner_attribution", True
)
release_enable_emefree = input.get("release_enable_emefree", True)
partner_url_config = get_partner_url_config(parameters, graph_config)
if (
release_enable_partner_repack
and not partner_url_config["release-partner-repack"]
):
raise Exception("Can't enable partner repacks when no config url found")
if (
release_enable_partner_attribution
and not partner_url_config["release-partner-attribution"]
):
raise Exception("Can't enable partner attribution when no config url found")
if release_enable_emefree and not partner_url_config["release-eme-free-repack"]:
raise Exception("Can't enable EMEfree repacks when no config url found")
parameters["release_enable_partner_repack"] = release_enable_partner_repack
parameters[
"release_enable_partner_attribution"
] = release_enable_partner_attribution
parameters["release_enable_emefree"] = release_enable_emefree
partner_config = input.get("release_partner_config")
if not partner_config and any(
[
release_enable_partner_repack,
release_enable_partner_attribution,
release_enable_emefree,
]
):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters["release_partner_config"] = fix_partner_config(partner_config)
parameters["release_partners"] = input.get("release_partners")
if input.get("release_partner_build_number"):
parameters["release_partner_build_number"] = input[
"release_partner_build_number"
]
if input["version"]:
parameters["version"] = input["version"]
parameters["required_signoffs"] = get_required_signoffs(input, parameters)
parameters["signoff_urls"] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def is_backstop(
params,
push_interval=BACKSTOP_PUSH_INTERVAL,
time_interval=BACKSTOP_TIME_INTERVAL,
trust_domain="gecko",
integration_projects=INTEGRATION_PROJECTS,
):
"""Determines whether the given parameters represent a backstop push.
Args:
push_interval (int): Number of pushes
time_interval (int): Minutes between forced schedules.
Use 0 to disable.
trust_domain (str): "gecko" for Firefox, "comm" for Thunderbird
integration_projects (set): project that uses backstop optimization
Returns:
bool: True if this is a backstop, otherwise False.
"""
# In case this is being faked on try.
if params.get("backstop", False):
return True
project = params["project"]
pushid = int(params["pushlog_id"])
pushdate = int(params["pushdate"])
if project in TRY_PROJECTS:
return False
elif project not in integration_projects:
return True
# On every Nth push, want to run all tasks.
if pushid % push_interval == 0:
return True
if time_interval <= 0:
return False
# We also want to ensure we run all tasks at least once per N minutes.
subs = {"trust-domain": trust_domain, "project": project}
index = BACKSTOP_INDEX.format(**subs)
try:
last_backstop_id = find_task_id(index)
except KeyError:
# Index wasn't found, implying there hasn't been a backstop push yet.
return True
if state_task(last_backstop_id) in ("failed", "exception"):
# If the last backstop failed its decision task, make this a backstop.
return True
try:
last_pushdate = get_artifact(last_backstop_id,
"public/parameters.yml")["pushdate"]
except HTTPError as e:
# If the last backstop decision task exists in the index, but
# parameters.yml isn't available yet, it means the decision task is
# still running. If that's the case, we can be pretty sure the time
# component will not cause a backstop, so just return False.
if e.response.status_code == 404:
return False
raise
if (pushdate - last_pushdate) / 60 >= time_interval:
return True
return False
def get_failures(task_id):
"""Returns a dict containing properties containing a list of
directories containing test failures and a separate list of
individual test failures from the errorsummary.log artifact for
the task.
Calls the helper function munge_test_path to attempt to find an
appropriate test path to pass to the task in
MOZHARNESS_TEST_PATHS. If no appropriate test path can be
determined, nothing is returned.
"""
def re_compile_list(*lst):
# Ideally we'd just use rb"" literals and avoid the encode, but
# this file needs to be importable in python2 for now.
return [re.compile(s.encode("utf-8")) for s in lst]
re_bad_tests = re_compile_list(
r"Last test finished",
r"LeakSanitizer",
r"Main app process exited normally",
r"ShutdownLeaks",
r"[(]SimpleTest/TestRunner.js[)]",
r"automation.py",
r"https?://localhost:\d+/\d+/\d+/.*[.]html",
r"jsreftest",
r"leakcheck",
r"mozrunner-startup",
r"pid: ",
r"RemoteProcessMonitor",
r"unknown test url",
)
re_extract_tests = re_compile_list(
r'"test": "(?:[^:]+:)?(?:https?|file):[^ ]+/reftest/tests/([^ "]+)',
r'"test": "(?:[^:]+:)?(?:https?|file):[^:]+:[0-9]+/tests/([^ "]+)',
r'xpcshell-?[^ "]*\.ini:([^ "]+)',
r'/tests/([^ "]+) - finished .*',
r'"test": "([^ "]+)"',
r'"message": "Error running command run_test with arguments '
r"[(]<wptrunner[.]wpttest[.]TestharnessTest ([^>]+)>",
r'"message": "TEST-[^ ]+ [|] ([^ "]+)[^|]*[|]',
)
def munge_test_path(line):
test_path = None
for r in re_bad_tests:
if r.search(line):
return None
for r in re_extract_tests:
m = r.search(line)
if m:
test_path = m.group(1)
break
return test_path
dirs = set()
tests = set()
artifacts = list_artifacts(task_id)
for artifact in artifacts:
if "name" not in artifact or not artifact["name"].endswith("errorsummary.log"):
continue
stream = get_artifact(task_id, artifact["name"])
if not stream:
continue
# The number of tasks created is determined by the
# `times` value and the number of distinct tests and
# directories as: times * (1 + len(tests) + len(dirs)).
# Since the maximum value of `times` specifiable in the
# Treeherder UI is 100, the number of tasks created can
# reach a very large value depending on the number of
# unique tests. During testing, it was found that 10
# distinct tests were sufficient to cause the action task
# to exceed the maxRunTime of 1800 seconds resulting in it
# being aborted. We limit the number of distinct tests
# and thereby the number of distinct test directories to a
# maximum of 5 to keep the action task from timing out.
# We handle the stream as raw bytes because it may contain invalid
# UTF-8 characters in portions other than those containing the error
# messages we're looking for.
for line in stream.read().split(b"\n"):
test_path = munge_test_path(line.strip())
if test_path:
tests.add(test_path.decode("utf-8"))
test_dir = os.path.dirname(test_path)
if test_dir:
dirs.add(test_dir.decode("utf-8"))
if len(tests) > 4:
break
return {"dirs": sorted(dirs), "tests": sorted(tests)}
def run_perfdocs(config, logger=None, paths=None, generate=True):
"""
Build up performance testing documentation dynamically by combining
text data from YAML files that reside in `perfdoc` folders
across the `testing` directory. Each directory is expected to have
an `index.rst` file along with `config.yml` YAMLs defining what needs
to be added to the documentation.
The YAML must also define the name of the "framework" that should be
used in the main index.rst for the performance testing documentation.
The testing documentation list will be ordered alphabetically once
it's produced (to avoid unwanted shifts because of unordered dicts
and path searching).
Note that the suite name headings will be given the H4 (---) style so it
is suggested that you use H3 (===) style as the heading for your
test section. H5 will be used be used for individual tests within each
suite.
Usage for verification: ./mach lint -l perfdocs
Usage for generation: ./mach lint -l perfdocs --fix
For validation, see the Verifier class for a description of how
it works.
The run will fail if the valid result from validate_tree is not
False, implying some warning/problem was logged.
:param dict config: The configuration given by mozlint.
:param StructuredLogger logger: The StructuredLogger instance to be used to
output the linting warnings/errors.
:param list paths: The paths that are being tested. Used to filter
out errors from files outside of these paths.
:param bool generate: If true, the docs will be (re)generated.
"""
from perfdocs.logger import PerfDocLogger
top_dir = os.environ.get("WORKSPACE", None)
if not top_dir:
floc = os.path.abspath(__file__)
top_dir = floc.split("tools")[0]
top_dir = top_dir.replace("\\", "\\\\")
PerfDocLogger.LOGGER = logger
PerfDocLogger.TOP_DIR = top_dir
# Convert all the paths to relative ones
rel_paths = [re.sub(top_dir, "", path) for path in paths]
PerfDocLogger.PATHS = rel_paths
target_dir = [os.path.join(top_dir, i) for i in rel_paths]
for path in target_dir:
if not os.path.exists(path):
raise Exception("Cannot locate directory at %s" % path)
decision_task_id = os.environ.get("DECISION_TASK_ID", None)
if decision_task_id:
from gecko_taskgraph.util.taskcluster import get_artifact
task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
else:
from tryselect.tasks import generate_tasks
task_graph = generate_tasks(params=None,
full=True,
disable_target_task_filter=True).tasks
# Late import because logger isn't defined until later
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
# Run the verifier first
verifier = Verifier(top_dir, task_graph)
verifier.validate_tree()
if not PerfDocLogger.FAILED:
# Even if the tree is valid, we need to check if the documentation
# needs to be regenerated, and if it does, we throw a linting error.
# `generate` dictates whether or not the documentation is generated.
generator = Generator(verifier, generate=generate, workspace=top_dir)
generator.generate_perfdocs()
def run(
task_type,
release_type,
try_config=None,
push=True,
message="{msg}",
closed_tree=False,
):
if task_type == "list":
print_available_task_types()
sys.exit(0)
if release_type == "nightly":
previous_graph = get_nightly_graph()
else:
release = get_releases(RELEASE_TO_BRANCH[release_type])[-1]
previous_graph = get_release_graph(release)
existing_tasks = find_existing_tasks([previous_graph])
previous_parameters = Parameters(strict=False,
**get_artifact(previous_graph,
"public/parameters.yml"))
# Copy L10n configuration from the commit the release we are using was
# based on. This *should* ensure that the chunking of L10n tasks is the
# same between graphs.
files_to_change = {
path: get_hg_file(previous_parameters, path)
for path in [
"browser/locales/l10n-changesets.json",
"browser/locales/shipped-locales",
]
}
try_config = try_config or {}
task_config = {
"version": 2,
"parameters": {
"existing_tasks": existing_tasks,
"try_task_config": try_config,
"try_mode": "try_task_config",
},
}
for param in (
"app_version",
"build_number",
"next_version",
"release_history",
"release_product",
"release_type",
"version",
):
task_config["parameters"][param] = previous_parameters[param]
try_config["tasks"] = TASK_TYPES[task_type]
for label in try_config["tasks"]:
if label in existing_tasks:
del existing_tasks[label]
msg = "scriptworker tests: {}".format(task_type)
return push_to_try(
"scriptworker",
message.format(msg=msg),
push=push,
closed_tree=closed_tree,
try_task_config=task_config,
files_to_change=files_to_change,
)
def fetch_graph_and_labels(parameters, graph_config):
decision_task_id = find_decision_task(parameters, graph_config)
# First grab the graph and labels generated during the initial decision task
full_task_graph = get_artifact(decision_task_id,
"public/full-task-graph.json")
logger.info("Load taskgraph from JSON.")
_, full_task_graph = TaskGraph.from_json(full_task_graph)
label_to_taskid = get_artifact(decision_task_id,
"public/label-to-taskid.json")
logger.info("Fetching additional tasks from action and cron tasks.")
# fetch everything in parallel; this avoids serializing any delay in downloading
# each artifact (such as waiting for the artifact to be mirrored locally)
with futures.ThreadPoolExecutor(CONCURRENCY) as e:
fetches = []
# fetch any modifications made by action tasks and swap out new tasks
# for old ones
def fetch_action(task_id):
logger.info(
f"fetching label-to-taskid.json for action task {task_id}")
try:
run_label_to_id = get_artifact(task_id,
"public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
if e.response.status_code != 404:
raise
logger.debug(
f"No label-to-taskid.json found for {task_id}: {e}")
head_rev_param = "{}head_rev".format(
graph_config["project-repo-param-prefix"])
namespace = "{}.v2.{}.revision.{}.taskgraph.actions".format(
graph_config["trust-domain"],
parameters["project"],
parameters[head_rev_param],
)
for task_id in list_tasks(namespace):
fetches.append(e.submit(fetch_action, task_id))
# Similarly for cron tasks..
def fetch_cron(task_id):
logger.info(
f"fetching label-to-taskid.json for cron task {task_id}")
try:
run_label_to_id = get_artifact(task_id,
"public/label-to-taskid.json")
label_to_taskid.update(run_label_to_id)
except HTTPError as e:
if e.response.status_code != 404:
raise
logger.debug(
f"No label-to-taskid.json found for {task_id}: {e}")
namespace = "{}.v2.{}.revision.{}.cron".format(
graph_config["trust-domain"],
parameters["project"],
parameters[head_rev_param],
)
for task_id in list_tasks(namespace):
fetches.append(e.submit(fetch_cron, task_id))
# now wait for each fetch to complete, raising an exception if there
# were any issues
for f in futures.as_completed(fetches):
f.result()
return (decision_task_id, full_task_graph, label_to_taskid)
def get_parameters(decision_task_id):
return get_artifact(decision_task_id, "public/parameters.yml")
