def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", "langpack(BM{})".format(attributes.get("l10n_chunk", ""))) job["attributes"].update(copy_attributes_from_dependent_job(dep_job)) job["attributes"]["chunk_locales"] = dep_job.attributes.get( "chunk_locales", ["en-US"]) job["description"] = job["description"].format( locales="/".join(job["attributes"]["chunk_locales"]), platform=job["attributes"]["build_platform"], ) job["scopes"] = [ get_beetmover_bucket_scope(config), get_beetmover_action_scope(config), ] job["dependencies"] = {"langpack-copy": dep_job.label} job["run-on-projects"] = job.get("run_on_projects", dep_job.attributes["run_on_projects"]) job["treeherder"] = treeherder job["shipping-phase"] = job.get("shipping-phase", dep_job.attributes["shipping_phase"]) job["shipping-product"] = dep_job.attributes["shipping_product"] yield job
def define_upstream_artifacts(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) if not partner_configs: return for job in jobs: dep_job = job["primary-dependency"] job["depname"] = dep_job.label job["attributes"] = copy_attributes_from_dependent_job(dep_job) repack_ids = job["extra"]["repack_ids"] artifacts_specifications = generate_specifications_of_artifacts_to_sign( config, job, keep_locale_template=True, kind=config.kind, ) task_type = "build" if "notarization" in job["depname"]: task_type = "scriptworker" job["upstream-artifacts"] = [ { "taskId": {"task-reference": f"<{dep_job.kind}>"}, "taskType": task_type, "paths": [ path_template.format(locale=repack_id) for path_template in spec["artifacts"] for repack_id in repack_ids ], "formats": spec["formats"], } for spec in artifacts_specifications ] yield job
def copy_attributes(config, jobs): for job in jobs: dep_job = job["primary-dependency"] job["attributes"] = copy_attributes_from_dependent_job(dep_job) job["attributes"]["chunk_locales"] = dep_job.attributes.get( "chunk_locales", ["en-US"] ) yield job
def copy_in_useful_magic(config, jobs): for job in jobs: dep = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep) attributes.update(job.get("attributes", {})) # build-platform is needed on `job` for by-build-platform job["build-platform"] = attributes.get("build_platform") job["attributes"] = attributes yield job
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault( "symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM")) dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)) treeherder.setdefault("kind", "build") label = job["label"] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) dependencies = {dep_job.kind: dep_job.label} # XXX release snap-repackage has a variable number of dependencies, depending on how many # "post-beetmover-dummy" jobs there are in the graph. if dep_job.kind != "release-snap-repackage" and len( dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if job.get("locale"): attributes["locale"] = job["locale"] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job["shipping-phase"], } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes build_platform = attributes.get("build_platform") if not build_platform: raise Exception("Cannot find build platform!") repack_id = dep_job.task.get("extra", {}).get("repack_id") if not repack_id: raise Exception("Cannot find repack id!") label = dep_job.label.replace("beetmover-", "beetmover-checksums-") description = ( "Beetmove checksums for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=build_platform, build_type=attributes.get("build_type"), ) ) extra = {} extra["partner_path"] = dep_job.task["payload"]["upstreamArtifacts"][0][ "locale" ] extra["repack_id"] = repack_id dependencies = {dep_job.kind: dep_job.label} for k, v in dep_job.dependencies.items(): if k.startswith("beetmover"): dependencies[k] = v attributes = copy_attributes_from_dependent_job(dep_job) task = { "label": label, "description": description, "worker-type": "{}/{}".format( dep_job.task["provisionerId"], dep_job.task["workerType"], ), "scopes": dep_job.task["scopes"], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "extra": extra, } if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] if "shipping-product" in job: task["shipping-product"] = job["shipping-product"] yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "BMcss(N)") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") label = job["label"] build_platform = attributes.get("build_platform") description = "Beetmover submission of checksums for source file" extra = {} if "devedition" in build_platform: extra["product"] = "devedition" else: extra["product"] = "firefox" dependencies = {dep_job.kind: dep_job.label} for k, v in dep_job.dependencies.items(): if k.startswith("beetmover"): dependencies[k] = v attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, } if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] if "shipping-product" in job: task["shipping-product"] = job["shipping-product"] yield task
def make_release_generate_checksums_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "SGenChcks") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") job_template = "{}-{}".format(dep_job.label, "signing") label = job.get("label", job_template) description = "Signing of the overall release-related checksums" dependencies = {dep_job.kind: dep_job.label} upstream_artifacts = [{ "taskId": { "task-reference": f"<{str(dep_job.kind)}>" }, "taskType": "build", "paths": [ get_artifact_path(dep_job, "SHA256SUMS"), get_artifact_path(dep_job, "SHA512SUMS"), ], "formats": ["autograph_gpg"], }] signing_cert_scope = get_signing_cert_scope(config) task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": [ signing_cert_scope, ], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", "")) ) label = job["label"] build_platform = attributes.get("build_platform") description = "Beetmover submission of checksums for langpack files" extra = {} if "devedition" in build_platform: extra["product"] = "devedition" else: extra["product"] = "firefox" dependencies = {dep_job.kind: dep_job.label} for k, v in dep_job.dependencies.items(): if k.startswith("beetmover"): dependencies[k] = v attributes = copy_attributes_from_dependent_job(dep_job) if "chunk_locales" in dep_job.attributes: attributes["chunk_locales"] = dep_job.attributes["chunk_locales"] attributes.update(job.get("attributes", {})) bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, } if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] if "shipping-product" in job: task["shipping-product"] = job["shipping-product"] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "BM-gv") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 2) treeherder.setdefault("kind", "build") label = job["label"] description = ("Beetmover submission for geckoview" "{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) dependencies = deepcopy(dep_job.dependencies) dependencies[dep_job.kind] = dep_job.label if job.get("locale"): attributes["locale"] = job["locale"] attributes["run_on_hg_branches"] = job["run-on-hg-branches"] task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [ job["bucket-scope"], "project:releng:beetmover:action:push-to-maven", ], "dependencies": dependencies, "attributes": attributes, "run-on-projects": job["run-on-projects"], "treeherder": treeherder, "shipping-phase": job["shipping-phase"], } yield task
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "css(N)") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") label = job["label"] description = "Signing of release-source checksums file" dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target-source.checksums", ], "formats": ["autograph_gpg"], }] signing_cert_scope = get_signing_cert_scope(config) task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": [ signing_cert_scope, ], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, } yield task
def fill_template(config, tasks): for task in tasks: dep = task["primary-dependency"] task.pop("dependent-tasks", None) # Fill out the dynamic fields in the task description task["label"] = dep.label + "-upload-symbols" # Skip tasks where we don't have the full crashsymbols enabled if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get( "skip-upload-crashsymbols" ): logger.debug("Skipping upload symbols task for %s", task["label"]) continue task["dependencies"] = {"build": dep.label} task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[ "head_repository" ] task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"] task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][ "SYMBOL_SECRET" ].format(level=config.params["level"]) attributes = copy_attributes_from_dependent_job(dep) attributes.update(task.get("attributes", {})) task["attributes"] = attributes treeherder = inherit_treeherder_from_dep(task, dep) th = dep.task.get("extra")["treeherder"] th_symbol = th.get("symbol") th_groupsymbol = th.get("groupSymbol", "?") # Disambiguate the treeherder symbol. sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol) treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym)) task["treeherder"] = treeherder # We only want to run these tasks if the build is run. # XXX Better to run this on promote phase instead? task["run-on-projects"] = dep.attributes.get("run_on_projects") task["optimization"] = {"upload-symbols": None} task["if-dependencies"] = ["build"] # clear out the stuff that's not part of a task description del task["primary-dependency"] yield task
def define_upstream_artifacts(config, jobs): for job in jobs: dep_job = job["primary-dependency"] upstream_artifact_task = job.pop("upstream-artifact-task", dep_job) job["attributes"] = copy_attributes_from_dependent_job(dep_job) if dep_job.attributes.get("chunk_locales"): # Used for l10n attribute passthrough job["attributes"]["chunk_locales"] = dep_job.attributes.get( "chunk_locales") locale_specifications = generate_specifications_of_artifacts_to_sign( config, job, keep_locale_template=True, dep_kind=upstream_artifact_task.kind, ) upstream_artifacts = [] for spec in locale_specifications: task_type = "l10n" if "notarization" in upstream_artifact_task.kind: task_type = "scriptworker" upstream_artifacts.append({ "taskId": { "task-reference": f"<{upstream_artifact_task.kind}>" }, "taskType": task_type, # Set paths based on artifacts in the specs (above) one per # locale present in the chunk this is signing stuff for. # Pass paths through set and sorted() so we get a list back # and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack) "paths": sorted({ path_template.format(locale=locale) for locale in upstream_artifact_task.attributes.get( "chunk_locales", []) for path_template in spec["artifacts"] }), "formats": spec["formats"], }) job["upstream-artifacts"] = upstream_artifacts yield job
def one_task_per_product_and_platform(config, jobs): unique_products_and_platforms = set() for job in jobs: dep_task = job["primary-dependency"] if "primary-dependency" in job: del job["primary-dependency"] product = dep_task.attributes.get("shipping_product") platform = dep_task.attributes.get("build_platform") if (product, platform) not in unique_products_and_platforms: attr_denylist = ("l10n_chunk", "locale", "artifact_map", "artifact_prefix") attributes = copy_attributes_from_dependent_job( dep_task, denylist=attr_denylist) attributes.update(job.get("attributes", {})) job["attributes"] = attributes job["name"] = f"{product}-{platform}" yield job unique_products_and_platforms.add((product, platform))
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "BM-SGenChcks") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") job_template = f"{dep_job.label}" label = job_template.replace("signing", "beetmover") description = "Transfer *SUMS and *SUMMARY checksums file to S3." # first dependency is the signing task for the *SUMS files dependencies = {dep_job.kind: dep_job.label} if len(dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") # update the dependencies with the dependencies of the signing task dependencies.update(dep_job.dependencies) bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": "promote", } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes build_platform = attributes.get("build_platform") if not build_platform: raise Exception("Cannot find build platform!") label = config.kind description = "Beetmover for partner attribution" if job["partner_public"]: label = f"{label}-public" description = f"{description} public" else: label = f"{label}-private" description = f"{description} private" attributes = copy_attributes_from_dependent_job(dep_job) task = { "label": label, "description": description, "dependencies": { dep_job.kind: dep_job.label }, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "shipping-phase": job["shipping-phase"], "shipping-product": job.get("shipping-product"), "partner_public": job["partner_public"], "worker": job["worker"], "scopes": job["scopes"], } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault( "symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"), ) dep_th_platform = ( dep_job.task.get("extra", {}) .get("treeherder", {}) .get("machine", {}) .get("platform", "") ) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) ) treeherder.setdefault("kind", "build") label = job["label"] build_platform = attributes.get("build_platform") description = ( "Beetmover submission of checksums for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), ) ) extra = {} if "devedition" in build_platform: extra["product"] = "devedition" else: extra["product"] = "firefox" dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if dep_job.attributes.get("locale"): treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale")) attributes["locale"] = dep_job.attributes.get("locale") bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, } if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] if "shipping-product" in job: task["shipping-product"] = job["shipping-product"] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes["build_platform"] if job["build-platform"].startswith("win"): if dep_job.kind.endswith("signing"): continue if job["build-platform"].startswith("macosx"): if dep_job.kind.endswith("repack"): continue dependencies = {dep_job.attributes.get("kind"): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith("macosx") and dependency.endswith("signing"): signing_task = dependency elif build_platform.startswith("win") and dependency.endswith("repack"): signing_task = dependency attributes["repackage_type"] = "repackage" repack_id = job["extra"]["repack_id"] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer" ) if build_platform.startswith("win32") and repack_stub_installer: job["package-formats"].append("installer-stub") repackage_config = [] for format in job.get("package-formats"): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "executable_extension": executable_extension(build_platform), } command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } repackage_config.append(command) run = job.get("mozharness", {}) run.update( { "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, } ) worker = { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, "env": { "REPACK_ID": repack_id, }, # Don't add generic artifact directory. "skip-artifacts": True, } worker_type = "b-linux" worker["docker-image"] = {"in-tree": "debian11-amd64-build"} worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ( "Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job["extra"]["repack_id"], build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "attributes": attributes, "scopes": ["queue:get-artifact:releng/partner/*"], "run-on-projects": dep_job.attributes.get("run_on_projects"), "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer, ), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend( [ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ] ) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get("release_history"): return for job in jobs: dep_job = job["primary-dependency"] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault("symbol", "p(N)") label = job.get("label", f"partials-{dep_job.label}") dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get("locale") if locale: attributes["locale"] = locale treeherder["symbol"] = f"p({locale})" attributes["shipping_phase"] = job["shipping-phase"] build_locale = locale or "en-US" build_platform = attributes["build_platform"] builds = get_builds(config.params["release_history"], build_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {"funsize": {"partials": list()}} update_number = 1 locale_suffix = "" if locale: locale_suffix = f"{locale}/" artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { "locale": build_locale, "from_mar": builds[build]["mar_url"], "to_mar": { "artifact-reference": artifact_path }, "branch": config.params["project"], "update_number": update_number, "dest_mar": build, } if "product" in builds[build]: partial_info["product"] = builds[build]["product"] if "previousVersion" in builds[build]: partial_info["previousVersion"] = builds[build][ "previousVersion"] if "previousBuildNumber" in builds[build]: partial_info["previousBuildNumber"] = builds[build][ "previousBuildNumber"] extra["funsize"]["partials"].append(partial_info) update_number += 1 level = config.params["level"] worker = { "artifacts": _generate_task_output_files(dep_job, builds.keys(), locale), "implementation": "docker-worker", "docker-image": { "in-tree": "funsize-update-generator" }, "os": "linux", "max-run-time": 3600 if "asan" in dep_job.label else 900, "chain-of-trust": True, "taskcluster-proxy": True, "env": { "SIGNING_CERT": identify_desired_signing_keys( config.params["project"], config.params["release_product"]), "EXTRA_PARAMS": f"--arch={architecture(build_platform)}", "MAR_CHANNEL_ID": attributes["mar-channel-id"], }, } if config.params.release_level() == "staging": worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true" task = { "label": label, "description": f"{dep_job.description} Partials", "worker-type": "b-linux", "dependencies": dependencies, "scopes": [], "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, "worker": worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 and any( [build_platform.startswith(prefix) for prefix in ["linux", "win"]]): task["scopes"].append( "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" ) yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get("locale", dep_job.attributes.get("locale")) attributes["repackage_type"] = "repackage-signing" treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "rs(B)") dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") treeherder.setdefault("platform", dep_th_platform) treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)) treeherder.setdefault("kind", "build") if locale: treeherder["symbol"] = f"rs({locale})" if config.kind == "repackage-signing-msi": treeherder["symbol"] = "MSIs({})".format(locale or "N") elif config.kind in ( "repackage-signing-msix", "repackage-signing-shippable-l10n-msix", ): # Like "MSIXs(Bs-multi)". treeherder["symbol"] = "MSIXs({})".format( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B")) label = job["label"] dep_kind = dep_job.kind if "l10n" in dep_kind: dep_kind = "repackage" dependencies = {dep_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to have better beetmover # support. But for multi-locale MSIX packages, we don't want the signing task to directly # depend on the langpack tasks. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != "docker-image" and not k.startswith("shippable-l10n-signing-linux64") }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) scopes = [signing_cert_scope] upstream_artifacts = [] for artifact in sorted(dep_job.release_artifacts): basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: upstream_artifacts.append({ "taskId": { "task-reference": f"<{dep_kind}>" }, "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "treeherder": treeherder, } yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage" locale = attributes.get("locale", job.get("locale")) if locale: attributes["locale"] = locale description = ( "Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "Rpk") dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") treeherder.setdefault("platform", dep_th_platform) treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") # Search dependencies before adding langpack dependencies. signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if "repackage-signing" in dependency: repackage_signing_task = dependency elif "signing" in dependency: signing_task = dependency if config.kind == "repackage-msi": treeherder["symbol"] = "MSI({})".format(locale or "N") elif config.kind == "repackage-msix": assert not locale # Like "MSIXs(Bs)". treeherder["symbol"] = "MSIX({})".format( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B") ) elif config.kind == "repackage-shippable-l10n-msix": assert not locale if attributes.get("l10n_chunk") or attributes.get("chunk_locales"): # We don't want to produce MSIXes for single-locale repack builds. continue description = ( "Repackaging with multiple locales for build '" "{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) # Like "MSIXs(Bs-multi)". treeherder["symbol"] = "MSIX({}-multi)".format( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B") ) fetches = job.setdefault("fetches", {}) # The keys are unique, like `shippable-l10n-signing-linux64-shippable-1/opt`, so we # can't ask for the tasks directly, we must filter for them. for t in config.kind_dependencies_tasks.values(): if t.kind != "shippable-l10n-signing": continue if t.attributes["build_platform"] != "linux64-shippable": continue if t.attributes["build_type"] != "opt": continue dependencies.update({t.label: t.label}) fetches.update( { t.label: [ { "artifact": f"{loc}/target.langpack.xpi", "extract": False, # Otherwise we can't disambiguate locales! "dest": f"distribution/extensions/{loc}", } for loc in t.attributes["chunk_locales"] ] } ) _fetch_subst_locale = "en-US" if locale: _fetch_subst_locale = locale worker_type = job["worker-type"] build_platform = attributes["build_platform"] use_stub = attributes.get("stub-installer") repackage_config = [] package_formats = job.get("package-formats") if use_stub and not repackage_signing_task and "msix" not in package_formats: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ["installer-stub"] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "_locale": _fetch_subst_locale, "architecture": architecture(build_platform), "version_display": config.params["version"], "mar-channel-id": attributes["mar-channel-id"], } # Allow us to replace `args` as well, but specifying things expanded in mozharness # without breaking .format and without allowing unknown through. substs.update({name: f"{{{name}}}" for name in MOZHARNESS_EXPANSIONS}) # We need to resolve `msix.*` values keyed by `package-format` for each format, not # just once, so we update a temporary copy just for extracting these values. temp_job = copy.deepcopy(job) for msix_key in ( "channel", "identity-name", "publisher", "publisher-display-name", ): resolve_keyed_by( item=temp_job, field=f"msix.{msix_key}", item_name="?", **{ "package-format": format, "release-type": config.params["release_type"], "level": config.params["level"], }, ) # Turn `msix.channel` into `msix-channel`, etc. value = temp_job.get("msix", {}).get(msix_key) if value: substs.update( {f"msix-{msix_key}": value}, ) command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } command["args"] = [arg.format(**substs) for arg in command["args"]] if "installer" in format and "aarch64" not in build_platform: command["args"].append("--use-upx") repackage_config.append(command) run = job.get("mozharness", {}) run.update( { "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, } ) worker = job.get("worker", {}) worker.update( { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, # Don't add generic artifact directory. "skip-artifacts": True, } ) if locale: # Make sure we specify the locale-specific upload dir worker.setdefault("env", {})["LOCALE"] = locale worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config=repackage_config, locale=locale, ) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "if-dependencies": [dep_job.kind], "attributes": attributes, "run-on-projects": job.get( "run-on-projects", dep_job.attributes.get("run_on_projects") ), "optimization": dep_job.optimization, "treeherder": treeherder, "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get("fetches"), ), "release-artifacts": [artifact["name"] for artifact in worker["artifacts"]], } if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend( [ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ] ) yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage-signing" treeherder = job.get("treeherder", {}) dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {}) treeherder.setdefault( "symbol", "{}(gd-s)".format(dep_treeherder["groupSymbol"]) ) treeherder.setdefault( "platform", dep_job.task.get("extra", {}).get("treeherder-platform") ) treeherder.setdefault("tier", dep_treeherder.get("tier", 1)) treeherder.setdefault("kind", "build") dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update( {k: v for k, v in signing_dependencies.items() if k != "docker-image"} ) description = "Signing Geckodriver for build '{}'".format( attributes.get("build_platform"), ) build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config ) upstream_artifacts = _craft_upstream_artifacts( dep_job, dep_job.kind, build_platform ) scopes = [signing_cert_scope] platform = build_platform.rsplit("-", 1)[0] task = { "label": job["label"], "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "treeherder": treeherder, "run-on-projects": ["mozilla-central"], "index": {"product": "geckodriver", "job-name": platform}, } if build_platform.startswith("macosx"): worker_type = task["worker-type"] worker_type_alias_map = { "linux-depsigning": "mac-depsigning", "linux-signing": "mac-signing", } assert worker_type in worker_type_alias_map, ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type) ) worker_type = worker_type_alias_map[worker_type] task["worker-type"] = worker_type_alias_map[task["worker-type"]] task["worker"]["mac-behavior"] = "mac_geckodriver" yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] locale = dep_job.attributes.get("locale") treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N") ) label = job.get("label", f"{config.kind}-{dep_job.label}") dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes["required_signoffs"] = sorted_unique_list( attributes.get("required_signoffs", []), job.pop("required_signoffs") ) attributes["shipping_phase"] = job["shipping-phase"] if locale: attributes["locale"] = locale build_platform = attributes.get("build_platform") if config.kind == "partials-signing": upstream_artifacts = generate_partials_artifacts( dep_job, config.params["release_history"], build_platform, locale ) else: upstream_artifacts = generate_complete_artifacts(dep_job, config.kind) is_shippable = job.get( "shippable", dep_job.attributes.get("shippable") # First check current job ) # Then dep job for 'shippable' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config ) scopes = [signing_cert_scope] task = { "label": label, "description": "{} {}".format( dep_job.description, job["description-suffix"] ), "worker-type": job.get("worker-type", "linux-signing"), "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "dependencies": dependencies, "attributes": attributes, "scopes": scopes, "run-on-projects": job.get( "run-on-projects", dep_job.attributes.get("run_on_projects") ), "treeherder": treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] repack_id = dep_job.task["extra"]["repack_id"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501 repack_id=repack_id, build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) if "linux" in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith("release-partner-repack"): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage-signing" signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) scopes = [signing_cert_scope] if "win" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, f"{repack_id}/target.installer.exe"), ], "formats": ["autograph_authenticode", "autograph_gpg"], }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, f"{repack_id}/target.stub-installer.exe", ), ], "formats": ["autograph_authenticode", "autograph_gpg"], }) elif "mac" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, f"{repack_id}/target.dmg"), ], "formats": ["autograph_gpg"], }] elif "linux" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, f"{repack_id}/target.tar.bz2"), ], "formats": ["autograph_gpg"], }] task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "extra": { "repack_id": repack_id, }, } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes signing_format_scopes = [] formats = set() for artifacts in job["upstream-artifacts"]: for f in artifacts["formats"]: formats.add(f) # Add each format only once is_shippable = dep_job.attributes.get("shippable", False) build_platform = dep_job.attributes.get("build_platform") treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) dep_th_platform = ( dep_job.task.get("extra", {}) .get("treeherder", {}) .get("machine", {}) .get("platform", "") ) build_type = dep_job.attributes.get("build_type") treeherder.setdefault( "platform", _generate_treeherder_platform( dep_th_platform, build_platform, build_type ), ) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1), ) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol") ), ) treeherder.setdefault("kind", "build") label = job["label"] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), ) ) attributes = ( job["attributes"] if job.get("attributes") else copy_attributes_from_dependent_job(dep_job) ) attributes["signed"] = True if dep_job.attributes.get("chunk_locales"): # Used for l10n attribute passthrough attributes["chunk_locales"] = dep_job.attributes.get("chunk_locales") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config ) worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning" mac_behavior = None task = { "label": label, "description": description, "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": job["upstream-artifacts"], "max-run-time": job.get("max-run-time", 3600), }, "scopes": [signing_cert_scope] + signing_format_scopes, "dependencies": _generate_dependencies(job), "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "routes": job.get("routes", []), "shipping-product": job.get("shipping-product"), "shipping-phase": job.get("shipping-phase"), } if dep_job.kind in task["dependencies"]: task["if-dependencies"] = [dep_job.kind] if "macosx" in build_platform: shippable = "false" if "shippable" in attributes and attributes["shippable"]: shippable = "true" mac_behavior = evaluate_keyed_by( config.graph_config["mac-notarization"]["mac-behavior"], "mac behavior", { "project": config.params["project"], "shippable": shippable, }, ) if mac_behavior == "mac_notarize": if "part-1" in config.kind: mac_behavior = "mac_notarize_part_1" elif config.kind.endswith("signing"): mac_behavior = "mac_notarize_part_3" else: raise Exception(f"Unknown kind {config.kind} for mac_behavior!") else: if "part-1" in config.kind: continue task["worker"]["mac-behavior"] = mac_behavior worker_type_alias_map = { "linux-depsigning": "mac-depsigning", "linux-signing": "mac-signing", } assert worker_type_alias in worker_type_alias_map, ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias) ) worker_type_alias = worker_type_alias_map[worker_type_alias] if job.get("entitlements-url"): task["worker"]["entitlements-url"] = job["entitlements-url"] task["worker-type"] = worker_type_alias if treeherder: task["treeherder"] = treeherder if job.get("extra"): task["extra"] = job["extra"] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ( "Signing of OpenH264 Binaries for '" "{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) # we have a genuine repackage job as our parent dependencies = {"openh264": dep_job.label} my_attributes = copy_attributes_from_dependent_job(dep_job) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope] worker_type = "linux-signing" worker = { "implementation": "scriptworker-signing", "max-run-time": 3600, } rev = attributes["openh264_rev"] upstream_artifact = { "taskId": {"task-reference": "<openh264>"}, "taskType": "build", } if "win" in build_platform: # job['primary-dependency'].task['payload']['command'] upstream_artifact["formats"] = ["autograph_authenticode"] elif "mac" in build_platform: upstream_artifact["formats"] = ["mac_single_file"] upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"] worker_type = "mac-signing" worker["mac-behavior"] = "mac_single_file" else: upstream_artifact["formats"] = ["autograph_gpg"] upstream_artifact["paths"] = [ f"private/openh264/openh264-{build_platform}-{rev}.zip", ] worker["upstream-artifacts"] = [upstream_artifact] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol") ), ) task = { "label": job["label"], "description": description, "worker-type": worker_type, "worker": worker, "scopes": scopes, "dependencies": dependencies, "attributes": my_attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) build_type = dep_job.attributes.get("build_type") treeherder.setdefault("platform", f"{dep_th_platform}/{build_type}") dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {}) treeherder.setdefault("tier", dep_treeherder.get("tier", 1)) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_treeherder.get("groupSymbol", "?"), dep_treeherder.get("symbol")), ) treeherder.setdefault("kind", "build") label = dep_job.label.replace("part-1", "poller") description = ("Mac Notarization Poller for build '" "{build_platform}/{build_type}'".format( build_platform=build_platform, build_type=attributes.get("build_type"))) attributes = (job["attributes"] if job.get("attributes") else copy_attributes_from_dependent_job(dep_job)) attributes["signed"] = True if dep_job.attributes.get("chunk_locales"): # Used for l10n attribute passthrough attributes["chunk_locales"] = dep_job.attributes.get( "chunk_locales") uuid_manifest_url = get_artifact_url("<part1>", "public/uuid_manifest.json") task = { "label": label, "description": description, "worker": { "implementation": "notarization-poller", "uuid-manifest": { "task-reference": uuid_manifest_url }, "max-run-time": 3600, }, "worker-type": "mac-notarization-poller", "dependencies": { "part1": dep_job.label }, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "routes": job.get("routes", []), "shipping-product": job.get("shipping-product"), "shipping-phase": job.get("shipping-phase"), } if treeherder: task["treeherder"] = treeherder if job.get("extra"): task["extra"] = job["extra"] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "c-Up(N)") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)) treeherder.setdefault("kind", "build") attributes = copy_attributes_from_dependent_job(dep_job) treeherder_job_symbol = dep_job.task["extra"]["treeherder"]["symbol"] treeherder["symbol"] = replace_group(treeherder_job_symbol, "c-Up") if dep_job.attributes.get("locale"): attributes["locale"] = dep_job.attributes.get("locale") label = job["label"] description = ("Balrog submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": ["public/manifest.json"], }] dependencies = {"beetmover": dep_job.label} for kind_dep in config.kind_dependencies_tasks.values(): if (kind_dep.kind == "startup-test" and kind_dep.attributes["build_platform"] == attributes.get("build_platform") and kind_dep.attributes["build_type"] == attributes.get("build_type") and kind_dep.attributes.get("shipping_product") == job.get("shipping-product")): dependencies["startup-test"] = kind_dep.label task = { "label": label, "description": description, "worker-type": "balrog", "worker": { "implementation": "balrog", "upstream-artifacts": upstream_artifacts, "balrog-action": "v2-submit-locale", "suffixes": ["", "-No-WNP"] if job.get("update-no-wnp") else [""], }, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job.get("shipping-phase", "promote"), "shipping-product": job.get("shipping-product"), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = inherit_treeherder_from_dep(job, dep_job) upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"] if "build" in job["dependent-tasks"]: upstream_symbol = job["dependent-tasks"]["build"].task["extra"][ "treeherder"]["symbol"] treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR")) label = job["label"] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) upstream_deps = job["dependent-tasks"] signing_name = "build-signing" build_name = "build" repackage_name = "repackage" repackage_signing_name = "repackage-signing" msi_signing_name = "repackage-signing-msi" msix_signing_name = "repackage-signing-shippable-l10n-msix" mar_signing_name = "mar-signing" if job.get("locale"): signing_name = "shippable-l10n-signing" build_name = "shippable-l10n" repackage_name = "repackage-l10n" repackage_signing_name = "repackage-signing-l10n" mar_signing_name = "mar-signing-l10n" dependencies = { "build": upstream_deps[build_name], "repackage": upstream_deps[repackage_name], "signing": upstream_deps[signing_name], "mar-signing": upstream_deps[mar_signing_name], } if "partials-signing" in upstream_deps: dependencies["partials-signing"] = upstream_deps[ "partials-signing"] if msi_signing_name in upstream_deps: dependencies[msi_signing_name] = upstream_deps[msi_signing_name] if msix_signing_name in upstream_deps: dependencies[msix_signing_name] = upstream_deps[msix_signing_name] if repackage_signing_name in upstream_deps: dependencies["repackage-signing"] = upstream_deps[ repackage_signing_name] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if job.get("locale"): attributes["locale"] = job["locale"] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job["shipping-phase"], "shipping-product": job.get("shipping-product"), } yield task