def make_task_description(config, jobs): for job in jobs: if len(job["dependencies"]) != 1: raise Exception("Exactly 1 dependency is required") job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( job["dependencies"] ) resolve_keyed_by( job, "worker.channel", item_name=job["name"], **{"release-type": config.params["release_type"]}, ) resolve_keyed_by( job, "worker-type", item_name=job["name"], **{"release-level": config.params.release_level()}, ) if config.params.release_level() == "production": job.setdefault("scopes", []).append( add_scope_prefix( config, "flathub:firefox:{}".format(job["worker"]["channel"]), ) ) yield job
def populate_repack_manifests_url(config, tasks): for task in tasks: partner_url_config = get_partner_url_config(config.params, config.graph_config) for k in partner_url_config: if config.kind.startswith(k): task["worker"].setdefault( "env", {})["REPACK_MANIFESTS_URL"] = partner_url_config[k] break else: raise Exception("Can't find partner REPACK_MANIFESTS_URL") for property in ("limit-locales", ): property = f"extra.{property}" resolve_keyed_by( task, property, property, **{"release-level": config.params.release_level()}, ) if task["worker"]["env"]["REPACK_MANIFESTS_URL"].startswith("git@"): task.setdefault("scopes", []).append( "secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh" .format(**config.params)) yield task
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [ "locales-file", "locales-per-chunk", "worker-type", "description", "run-time", "docker-image", "secrets", "fetches.toolchain", "fetches.fetch", "tooltool", "env", "ignore-locales", "mozharness.config", "mozharness.options", "mozharness.actions", "mozharness.script", "treeherder.tier", "treeherder.platform", "index.type", "index.product", "index.job-name", "when.files-changed", ] for job in jobs: job = copy.deepcopy(job) # don't overwrite dict values here for field in fields: resolve_keyed_by(item=job, field=field, item_name=job["name"]) yield job
def fill_email_data(config, tasks): format_kwargs = { "head_rev": config.params["head_rev"], "project": config.params["project"], "th_root": "https://treeherder.mozilla.org/#/", "tiers": "&tier=1%2C2%2C3", } for task in tasks: format_kwargs["task_name"] = task["name"] format_kwargs["filterstring"] = "&searchStr=iris%20{}".format( task["name"]) format_kwargs["chunk"] = task["worker"]["env"]["CURRENT_TEST_DIR"] resolve_keyed_by( task, "notify.email", item_name=task["name"], **{ "project": config.params["project"], }, ) email = task["notify"].get("email") if email: email["link"]["href"] = email["link"]["href"].format( **format_kwargs) email["subject"] = email["subject"].format(**format_kwargs) yield task
def format(config, tasks): """Apply format substitution to worker.env and worker.command.""" format_params = { "release_config": get_release_config(config), "config_params": config.params, } for task in tasks: format_params["task"] = task command = task.get("worker", {}).get("command", []) task["worker"]["command"] = [x.format(**format_params) for x in command] env = task.get("worker", {}).get("env", {}) for k in env.keys(): resolve_keyed_by( env, k, "flatpak envs", **{ "release-level": config.params.release_level(), "project": config.params["project"], } ) task["worker"]["env"][k] = env[k].format(**format_params) yield task
def resolve_keys(config, jobs): for job in jobs: resolve_keyed_by( job, "partner-bucket-scope", item_name=job["label"], **{"release-level": config.params.release_level()}, ) yield job
def resolve_keys(config, jobs): for job in jobs: resolve_keyed_by( job, "use-sccache", item_name=job["name"], **{"release-level": config.params.release_level()}, ) yield job
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = ["fetches.toolchain"] for job in jobs: job = copy.deepcopy(job) # don't overwrite dict values here for field in fields: resolve_keyed_by(item=job, field=field, item_name=job["name"]) yield job
def handle_keyed_by_prereqs(config, tests): """ Only resolve keys for prerequisite fields here since the these keyed-by options might have keyed-by fields as well. """ for test in tests: resolve_keyed_by(test, "raptor-subtests", item_name=test["test-name"]) yield test
def make_task_description(config, jobs): for job in jobs: for key in ["worker-type", "scopes"]: resolve_keyed_by( job, key, item_name=job["name"], **{"release-level": config.params.release_level()}) yield job
def resolve_shipping_product(config, jobs): for job in jobs: resolve_keyed_by( job, "shipping-product", item_name=job["name"], **{ "release-type": config.params["release_type"], }, ) yield job
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [ "mozharness.config", "package-formats", ] for job in jobs: job = copy.deepcopy(job) # don't overwrite dict values here for field in fields: resolve_keyed_by(item=job, field=field, item_name="?") yield job
def resolve_keys(config, tasks): for task in tasks: env = task["worker"].setdefault("env", {}) env["BRANCH"] = config.params["project"] for envvar in env: resolve_keyed_by(env, envvar, envvar, **config.params) for envvar in list(env.keys()): if not env.get(envvar): del env[envvar] yield task
def handle_keyed_by_perftest(config, jobs): fields = [ "perftest-metrics", "perftest-extra-options", "perftest-btime-variants" ] for job in jobs: if job.get("perftest") is None: yield job continue for field in fields: resolve_keyed_by(job, field, item_name=job["name"]) job.pop("perftest") yield job
def resolve_keys(config, jobs): for job in jobs: for field in ("worker-type", "attributes.artifact_map"): resolve_keyed_by( job, field, item_name=job["label"], **{ "release-level": config.params.release_level(), "release-type": config.params["release_type"], "project": config.params["project"], }, ) yield job
def make_iris_tasks(config, jobs): # Each platform will get a copy of the test categories platforms = config.config.get("iris-build-platforms") # The fields needing to be resolve_keyed_by'd fields = [ "dependencies.build", "fetches.build", "run.command", "run-on-projects", "treeherder.platform", "worker.docker-image", "worker.artifacts", "worker.env.PATH", "worker.max-run-time", "worker-type", ] for job in jobs: for platform in platforms: # Make platform-specific clones of each iris task clone = deepcopy(job) basename = clone["name"] clone["description"] = clone["description"].format(basename) clone["name"] = clone["name"] + "-" + platform # resolve_keyed_by picks the correct values based on # the `by-platform` keys in the task definitions for field in fields: resolve_keyed_by( clone, field, clone["name"], **{ "platform": platform, }, ) # iris uses this to select the tests to run in this chunk clone["worker"]["env"]["CURRENT_TEST_DIR"] = basename # Clean up some entries when they aren't needed if clone["worker"]["docker-image"] is None: del clone["worker"]["docker-image"] if clone["worker"]["env"]["PATH"] is None: del clone["worker"]["env"]["PATH"] yield clone
def mozconfig(config, jobs): for job in jobs: resolve_keyed_by( job, "run.mozconfig-variant", item_name=job["name"], **{ "release-type": config.params["release_type"], }, ) mozconfig_variant = job["run"].pop("mozconfig-variant", None) if mozconfig_variant: job["run"].setdefault("extra-config", {})["mozconfig_variant"] = mozconfig_variant yield job
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by project, etc.""" fields = [ "run.config", "run.extra-config", ] for job in jobs: job = copy.deepcopy(job) for field in fields: resolve_keyed_by( item=job, field=field, item_name=job["name"], **{"release-level": config.params.release_level()}) yield job
def split_raptor_subtests(config, tests): for test in tests: # For tests that have 'raptor-subtests' listed, we want to create a separate # test job for every subtest (i.e. split out each page-load URL into its own job) subtests = test.pop("raptor-subtests", None) if not subtests: yield test continue chunk_number = 0 for subtest in subtests: chunk_number += 1 # Create new test job chunked = deepcopy(test) chunked["chunk-number"] = chunk_number chunked["subtest"] = subtest chunked["subtest-symbol"] = subtest if isinstance(chunked["subtest"], list): chunked["subtest"] = subtest[0] chunked["subtest-symbol"] = subtest[1] chunked = resolve_keyed_by(chunked, "tier", chunked["subtest"], defer=["variant"]) yield chunked
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by project, etc.""" fields = [ "run.config", "run.product-field", "run.extra-config", ] release_config = get_release_config(config) version = release_config["version"] for job in jobs: for field in fields: resolve_keyed_by( item=job, field=field, item_name=job["name"], **{ "project": config.params["project"], "release-level": config.params.release_level(), "release-type": config.params["release_type"], }, ) for cfg in job["run"]["config"]: job["run"]["mach"].extend(["--config", cfg]) if config.kind == "cron-bouncer-check": job["run"]["mach"].extend([ "--product-field={}".format(job["run"]["product-field"]), "--products-url={}".format(job["run"]["products-url"]), ]) del job["run"]["product-field"] del job["run"]["products-url"] elif config.kind == "release-bouncer-check": job["run"]["mach"].append(f"--version={version}") del job["run"]["config"] if "extra-config" in job["run"]: env = job["worker"].setdefault("env", {}) env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps( job["run"]["extra-config"], sort_keys=True) del job["run"]["extra-config"] yield job
def stub_installer(config, jobs): for job in jobs: resolve_keyed_by( job, "stub-installer", item_name=job["name"], project=config.params["project"], **{ "release-type": config.params["release_type"], }, ) job.setdefault("attributes", {}) if job.get("stub-installer"): job["attributes"]["stub-installer"] = job["stub-installer"] job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"}) if "stub-installer" in job: del job["stub-installer"] yield job
def add_notifications(config, jobs): for job in jobs: label = "{}-{}".format(config.kind, job["name"]) resolve_keyed_by(job, "emails", label, project=config.params["project"]) emails = [ email.format(config=config.__dict__) for email in job.pop("emails") ] command = [ "release", "send-buglist-email", "--version", config.params["version"], "--product", job["shipping-product"], "--revision", config.params["head_rev"], "--build-number", str(config.params["build_number"]), "--repo", config.params["head_repository"], ] for address in emails: command += ["--address", address] command += [ # We wrap this in `{'task-reference': ...}` below "--task-group-id", "<decision>", ] job["scopes"] = [f"notify:email:{address}" for address in emails] job["run"] = { "using": "mach", "sparse-profile": "mach", "mach": { "task-reference": " ".join(map(shell_quote, command)) }, } yield job
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [ "update-no-wnp", ] for job in jobs: label = job.get("dependent-task", object).__dict__.get("label", "?no-label?") for field in fields: resolve_keyed_by( item=job, field=field, item_name=label, **{ "project": config.params["project"], "release-type": config.params["release_type"], }, ) yield job
def handle_shell(config, jobs): """ Handle the 'shell' property. """ fields = [ "run-on-projects", "worker.env", ] for job in jobs: if not job.get("shell"): yield job continue for field in fields: resolve_keyed_by(job, field, item_name=job["name"]) del job["shell"] yield job
def add_notifications(config, jobs): release_config = get_release_config(config) for job in jobs: label = "{}-{}".format(config.kind, job["name"]) notifications = job.pop("notifications", None) if notifications: resolve_keyed_by(notifications, "emails", label, project=config.params["project"]) emails = notifications["emails"] format_kwargs = dict( task=job, config=config.__dict__, release_config=release_config, ) subject = titleformatter.format(notifications["subject"], **format_kwargs) message = titleformatter.format(notifications["message"], **format_kwargs) emails = [email.format(**format_kwargs) for email in emails] # By default, we only send mail on success to avoid messages like 'blah is in the # candidates dir' when cancelling graphs, dummy job failure, etc status_types = notifications.get("status-types", ["on-completed"]) for s in status_types: job.setdefault("routes", []).extend( [f"notify.email.{email}.{s}" for email in emails]) # Customize the email subject to include release name and build number job.setdefault("extra", {}).update( {"notify": { "email": { "subject": subject, } }}) if message: job["extra"]["notify"]["email"]["content"] = message yield job
def test_no_by_not_only_by(self): self.assertEqual( resolve_keyed_by({"x": { "by-y": True, "a": 10 }}, "x", "n"), {"x": { "by-y": True, "a": 10 }}, )
def make_task_description(config, jobs): release_config = get_release_config(config) for job in jobs: resolve_keyed_by(job, "worker-type", item_name=job["name"], **{"release-level": config.params.release_level()}) resolve_keyed_by(job, "scopes", item_name=job["name"], **{"release-level": config.params.release_level()}) job["worker"][ "release-name"] = "{product}-{version}-build{build_number}".format( product=job["shipping-product"].capitalize(), version=release_config["version"], build_number=release_config["build_number"], ) yield job
def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc, but not `msix.*` fields that can be keyed by `package-format`. Such fields are handled specially below. """ fields = [ "mozharness.config", "package-formats", ] for job in jobs: job = copy.deepcopy(job) # don't overwrite dict values here for field in fields: resolve_keyed_by( item=job, field=field, item_name="?", **{ "release-type": config.params["release_type"], "level": config.params["level"], }, ) yield job
def handle_keyed_by(config, tests): fields = [ "test-url-param", "variants", "limit-platforms", "activity", "binary-path", "fetches.fetch", "max-run-time", "run-on-projects", "target", "tier", "run-visual-metrics", ] for test in tests: for field in fields: resolve_keyed_by(test, field, item_name=test["test-name"], defer=["variant"]) yield test
def test_nested(self): x = { "by-foo": { "F1": { "by-bar": { "B1": 11, "B2": 12, }, }, "F2": 20, "default": 0, }, } self.assertEqual( resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B1"), {"x": 11}) self.assertEqual( resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B2"), {"x": 12}) self.assertEqual(resolve_keyed_by({"x": x}, "x", "x", foo="F2"), {"x": 20}) self.assertEqual( resolve_keyed_by({"x": x}, "x", "x", foo="F99", bar="B1"), {"x": 0}) # bar is deferred self.assertEqual( resolve_keyed_by({"x": x}, "x", "x", defer=["bar"], foo="F1", bar="B1"), {"x": { "by-bar": { "B1": 11, "B2": 12 } }}, )