def set_fetches_and_locations(config, jobs): """Set defaults, including those that differ per worker implementation""" for job in jobs: dependencies = copy.deepcopy(job["dependencies"]) for platform, label in dependencies.items(): job["dependencies"] = {"build": label} aar_location = _get_aar_location(config, job, platform) prefix = get_artifact_prefix(job) if not prefix.endswith("/"): prefix = prefix + "/" if aar_location.startswith(prefix): aar_location = aar_location[len(prefix):] job.setdefault("fetches", {}).setdefault(platform, []).append({ "artifact": aar_location, "extract": False, }) aar_file_name = aar_location.split("/")[-1] env_var = MOZ_ANDROID_FAT_AAR_ENV_MAP[platform] job["worker"]["env"][env_var] = aar_file_name job["dependencies"] = dependencies yield job
def _generate_task_output_files(task, worker_implementation, repackage_config, partner): """We carefully generate an explicit list here, but there's an artifacts directory too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts(). Any errors here are likely masked by that. """ partner_output_path = f"{partner}/" artifact_prefix = get_artifact_prefix(task) if worker_implementation == ("docker-worker", "linux"): local_prefix = "/builds/worker/workspace/" elif worker_implementation == ("generic-worker", "windows"): local_prefix = "workspace/" else: raise NotImplementedError( f'Unsupported worker implementation: "{worker_implementation}"' ) output_files = [] for config in repackage_config: output_files.append( { "type": "file", "path": "{}outputs/{}{}".format( local_prefix, partner_output_path, config["output"] ), "name": "{}/{}{}".format( artifact_prefix, partner_output_path, config["output"] ), } ) return output_files
def _generate_task_output_files( task, worker_implementation, repackage_config, locale=None ): locale_output_path = f"{locale}/" if locale else "" artifact_prefix = get_artifact_prefix(task) if worker_implementation == ("docker-worker", "linux"): local_prefix = "/builds/worker/workspace/" elif worker_implementation == ("generic-worker", "windows"): local_prefix = "workspace/" else: raise NotImplementedError( f'Unsupported worker implementation: "{worker_implementation}"' ) output_files = [] for config in repackage_config: output_files.append( { "type": "file", "path": "{}outputs/{}{}".format( local_prefix, locale_output_path, config["output"] ), "name": "{}/{}{}".format( artifact_prefix, locale_output_path, config["output"] ), } ) return output_files
def generic_worker_add_artifacts(config, job, taskdesc): """ Adds an artifact directory to the task """ # The path is the location on disk; it doesn't necessarily # mean the artifacts will be public or private; that is set via the name # attribute in add_artifacts. path = get_artifact_prefix(taskdesc) taskdesc["worker"].setdefault("env", {})["UPLOAD_DIR"] = path add_artifacts(config, job, taskdesc, path=path)
def add_artifacts(config, job, taskdesc, path): taskdesc["worker"].setdefault("artifacts", []).append({ "name": get_artifact_prefix(taskdesc), "path": path, "type": "directory", })
def handle_artifact_prefix(config, jobs): """Resolve ``artifact_prefix`` in env vars""" for job in jobs: artifact_prefix = get_artifact_prefix(job) for k1, v1 in job.get("env", {}).items(): if isinstance(v1, str): job["env"][k1] = v1.format(artifact_prefix=artifact_prefix) elif isinstance(v1, dict): for k2, v2 in v1.items(): job["env"][k1][k2] = v2.format( artifact_prefix=artifact_prefix) yield job
def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None): artifact_prefix = get_artifact_prefix(job) if locale and locale != "en-US": artifact_prefix = f"{artifact_prefix}/{locale}" upstream_artifacts = [{ "taskId": { "task-reference": "<partials-signing>" }, "taskType": "signing", "paths": [f"{artifact_prefix}/{path}" for path, _ in artifacts], "locale": locale or "en-US", }] return upstream_artifacts
def _generate_task_output_files(job, filenames, locale=None): locale_output_path = f"{locale}/" if locale else "" artifact_prefix = get_artifact_prefix(job) data = list() for filename in filenames: data.append({ "type": "file", "path": f"/home/worker/artifacts/{filename}", "name": f"{artifact_prefix}/{locale_output_path}{filename}", }) data.append({ "type": "file", "path": "/home/worker/artifacts/manifest.json", "name": f"{artifact_prefix}/{locale_output_path}manifest.json", }) return data
def generate_partials_artifacts(job, release_history, platform, locale=None): artifact_prefix = get_artifact_prefix(job) if locale: artifact_prefix = f"{artifact_prefix}/{locale}" else: locale = "en-US" artifacts = get_partials_artifacts_from_params(release_history, platform, locale) upstream_artifacts = [ { "taskId": {"task-reference": "<partials>"}, "taskType": "partials", "paths": [ f"{artifact_prefix}/{path}" for path, version in artifacts # TODO Use mozilla-version to avoid comparing strings. Otherwise Firefox 100 will # be considered smaller than Firefox 56 if version is None or version >= "56" ], "formats": ["autograph_hash_only_mar384"], } ] old_mar_upstream_artifacts = { "taskId": {"task-reference": "<partials>"}, "taskType": "partials", "paths": [ f"{artifact_prefix}/{path}" for path, version in artifacts # TODO Use mozilla-version to avoid comparing strings. Otherwise Firefox 100 will be # considered smaller than Firefox 56 if version is not None and version < "56" ], "formats": ["mar"], } if old_mar_upstream_artifacts["paths"]: upstream_artifacts.append(old_mar_upstream_artifacts) return upstream_artifacts
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes["build_platform"] if job["build-platform"].startswith("win"): if dep_job.kind.endswith("signing"): continue if job["build-platform"].startswith("macosx"): if dep_job.kind.endswith("repack"): continue dependencies = {dep_job.attributes.get("kind"): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith("macosx") and dependency.endswith("signing"): signing_task = dependency elif build_platform.startswith("win") and dependency.endswith("repack"): signing_task = dependency attributes["repackage_type"] = "repackage" repack_id = job["extra"]["repack_id"] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer" ) if build_platform.startswith("win32") and repack_stub_installer: job["package-formats"].append("installer-stub") repackage_config = [] for format in job.get("package-formats"): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "executable_extension": executable_extension(build_platform), } command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } repackage_config.append(command) run = job.get("mozharness", {}) run.update( { "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, } ) worker = { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, "env": { "REPACK_ID": repack_id, }, # Don't add generic artifact directory. "skip-artifacts": True, } worker_type = "b-linux" worker["docker-image"] = {"in-tree": "debian11-amd64-build"} worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ( "Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job["extra"]["repack_id"], build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "attributes": attributes, "scopes": ["queue:get-artifact:releng/partner/*"], "run-on-projects": dep_job.attributes.get("run_on_projects"), "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer, ), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend( [ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ] ) yield task
def use_fetches(config, jobs): artifact_names = {} aliases = {} if config.kind in ("toolchain", "fetch"): jobs = list(jobs) for job in jobs: run = job.get("run", {}) label = job["label"] get_attribute(artifact_names, label, run, "toolchain-artifact") value = run.get(f"{config.kind}-alias") if not value: value = [] elif isinstance(value, str): value = [value] for alias in value: aliases[f"{config.kind}-{alias}"] = label for task in config.kind_dependencies_tasks.values(): if task.kind in ("fetch", "toolchain"): get_attribute( artifact_names, task.label, task.attributes, f"{task.kind}-artifact", ) value = task.attributes.get(f"{task.kind}-alias") if not value: value = [] elif isinstance(value, str): value = [value] for alias in value: aliases[f"{task.kind}-{alias}"] = task.label artifact_prefixes = {} for job in order_tasks(config, jobs): artifact_prefixes[job["label"]] = get_artifact_prefix(job) fetches = job.pop("fetches", None) if not fetches: yield job continue job_fetches = [] name = job.get("name", job.get("label")) dependencies = job.setdefault("dependencies", {}) worker = job.setdefault("worker", {}) prefix = get_artifact_prefix(job) has_sccache = False for kind, artifacts in fetches.items(): if kind in ("fetch", "toolchain"): for fetch_name in artifacts: label = f"{kind}-{fetch_name}" label = aliases.get(label, label) if label not in artifact_names: raise Exception( "Missing fetch job for {kind}-{name}: {fetch}". format(kind=config.kind, name=name, fetch=fetch_name)) path = artifact_names[label] dependencies[label] = label job_fetches.append({ "artifact": path, "task": f"<{label}>", "extract": True, }) if kind == "toolchain" and fetch_name.endswith("-sccache"): has_sccache = True else: if kind not in dependencies: raise Exception( "{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind)) dep_label = dependencies[kind] if dep_label in artifact_prefixes: prefix = artifact_prefixes[dep_label] else: if dep_label not in config.kind_dependencies_tasks: raise Exception( "{name} can't fetch {kind} artifacts because " "there are no tasks with label {label} in kind dependencies!" .format( name=name, kind=kind, label=dependencies[kind], )) prefix = get_artifact_prefix( config.kind_dependencies_tasks[dep_label]) for artifact in artifacts: if isinstance(artifact, str): path = artifact dest = None extract = True verify_hash = False else: path = artifact["artifact"] dest = artifact.get("dest") extract = artifact.get("extract", True) verify_hash = artifact.get("verify-hash", False) fetch = { "artifact": f"{prefix}/{path}" if not path.startswith("/") else path[1:], "task": f"<{kind}>", "extract": extract, } if dest is not None: fetch["dest"] = dest if verify_hash: fetch["verify-hash"] = verify_hash job_fetches.append(fetch) if job.get("use-sccache") and not has_sccache: raise Exception( "Must provide an sccache toolchain if using sccache.") job_artifact_prefixes = { mozpath.dirname(fetch["artifact"]) for fetch in job_fetches if not fetch["artifact"].startswith("public/") } if job_artifact_prefixes: # Use taskcluster-proxy and request appropriate scope. For example, add # 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'. worker["taskcluster-proxy"] = True for prefix in sorted(job_artifact_prefixes): scope = f"queue:get-artifact:{prefix}/*" if scope not in job.setdefault("scopes", []): job["scopes"].append(scope) artifacts = {} for f in job_fetches: _, __, artifact = f["artifact"].rpartition("/") if "dest" in f: artifact = f"{f['dest']}/{artifact}" task = f["task"][1:-1] if artifact in artifacts: raise Exception( f"Task {name} depends on {artifacts[artifact]} and {task} " f"that both provide {artifact}") artifacts[artifact] = task env = worker.setdefault("env", {}) env["MOZ_FETCHES"] = { "task-reference": json.dumps(sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True) } # The path is normalized to an absolute path in run-task env.setdefault("MOZ_FETCHES_DIR", "fetches") yield job
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get("release_history"): return for job in jobs: dep_job = job["primary-dependency"] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault("symbol", "p(N)") label = job.get("label", f"partials-{dep_job.label}") dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get("locale") if locale: attributes["locale"] = locale treeherder["symbol"] = f"p({locale})" attributes["shipping_phase"] = job["shipping-phase"] build_locale = locale or "en-US" build_platform = attributes["build_platform"] builds = get_builds(config.params["release_history"], build_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {"funsize": {"partials": list()}} update_number = 1 locale_suffix = "" if locale: locale_suffix = f"{locale}/" artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { "locale": build_locale, "from_mar": builds[build]["mar_url"], "to_mar": { "artifact-reference": artifact_path }, "branch": config.params["project"], "update_number": update_number, "dest_mar": build, } if "product" in builds[build]: partial_info["product"] = builds[build]["product"] if "previousVersion" in builds[build]: partial_info["previousVersion"] = builds[build][ "previousVersion"] if "previousBuildNumber" in builds[build]: partial_info["previousBuildNumber"] = builds[build][ "previousBuildNumber"] extra["funsize"]["partials"].append(partial_info) update_number += 1 level = config.params["level"] worker = { "artifacts": _generate_task_output_files(dep_job, builds.keys(), locale), "implementation": "docker-worker", "docker-image": { "in-tree": "funsize-update-generator" }, "os": "linux", "max-run-time": 3600 if "asan" in dep_job.label else 900, "chain-of-trust": True, "taskcluster-proxy": True, "env": { "SIGNING_CERT": identify_desired_signing_keys( config.params["project"], config.params["release_product"]), "EXTRA_PARAMS": f"--arch={architecture(build_platform)}", "MAR_CHANNEL_ID": attributes["mar-channel-id"], }, } if config.params.release_level() == "staging": worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true" task = { "label": label, "description": f"{dep_job.description} Partials", "worker-type": "b-linux", "dependencies": dependencies, "scopes": [], "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, "worker": worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 and any( [build_platform.startswith(prefix) for prefix in ["linux", "win"]]): task["scopes"].append( "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" ) yield task
def split_public_and_private(config, jobs): # we need to separate private vs public destinations because beetmover supports one # in a single task. Only use a single task for each type though. partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: upstream_artifacts = job["primary-dependency"].release_artifacts attribution_task_ref = "<{}>".format(job["primary-dependency"].label) prefix = get_artifact_prefix(job["primary-dependency"]) artifacts = defaultdict(list) for artifact in upstream_artifacts: partner, sub_partner, platform, locale, _ = artifact.replace( prefix + "/", "").split("/", 4) destination = "private" this_config = [ p for p in partner_config["configs"] if (p["campaign"] == partner and p["content"] == sub_partner) ] if this_config[0].get("upload_to_candidates"): destination = "public" artifacts[destination].append( (artifact, partner, sub_partner, platform, locale)) action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") public_bucket_scope = get_beetmover_bucket_scope(config) partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) repl_dict = { "build_number": config.params["build_number"], "release_partner_build_number": config.params["release_partner_build_number"], "version": config.params["version"], "partner": "{partner}", # we'll replace these later, per artifact "subpartner": "{subpartner}", "platform": "{platform}", "locale": "{locale}", } for destination, destination_artifacts in artifacts.items(): this_job = deepcopy(job) if destination == "public": this_job["scopes"] = [public_bucket_scope, action_scope] this_job["partner_public"] = True else: this_job["scopes"] = [partner_bucket_scope, action_scope] this_job["partner_public"] = False partner_path_key = "partner-{destination}-path".format( destination=destination) partner_path = this_job[partner_path_key].format(**repl_dict) this_job.setdefault( "worker", {})["upstream-artifacts"] = generate_upstream_artifacts( attribution_task_ref, destination_artifacts, partner_path) yield this_job