def generate_upstream_artifacts(job, signing_task_ref, build_task_ref): build_mapping = CHECKSUMS_BUILD_ARTIFACTS signing_mapping = CHECKSUMS_SIGNING_ARTIFACTS artifact_prefix = get_artifact_prefix(job) upstream_artifacts = [{ "taskId": {"task-reference": build_task_ref}, "taskType": "build", "paths": ["{}/{}".format(artifact_prefix, p) for p in build_mapping], "locale": "en-US", }, { "taskId": {"task-reference": signing_task_ref}, "taskType": "signing", "paths": ["{}/{}".format(artifact_prefix, p) for p in signing_mapping], "locale": "en-US", }] return upstream_artifacts
def docker_worker_use_artifacts(config, job, taskdesc, use_artifacts): """Set a JSON object of artifact URLs in an environment variable. This will tell the run-task script to download the artifacts. """ urls = {} prefix = get_artifact_prefix(taskdesc) for kind, artifacts in use_artifacts.items(): if kind not in taskdesc['dependencies']: raise Exception("{label} can't use '{kind}' artifacts because it has no '{kind}' " "dependency!".format(label=job['label'], kind=kind)) task_id = '<{}>'.format(kind) urls[kind] = [] for artifact in artifacts: path = '/'.join([prefix, artifact]) urls[kind].append(get_artifact_url(task_id, path)) env = taskdesc['worker'].setdefault('env', {}) env['USE_ARTIFACT_URLS'] = {'task-reference': json.dumps(urls)} env['USE_ARTIFACT_PATH'] = '/builds/worker/use-artifacts'
def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None): artifact_prefix = get_artifact_prefix(job) if locale and locale != "en-US": artifact_prefix = "{}/{}".format(artifact_prefix, locale) upstream_artifacts = [{ "taskId": { "task-reference": "<partials-signing>" }, "taskType": "signing", "paths": ["{}/{}".format(artifact_prefix, path) for path, _ in artifacts], "locale": locale or "en-US", }] return upstream_artifacts
def _generate_task_output_files(task, worker_implementation, repackage_config, locale=None): locale_output_path = '{}/'.format(locale) if locale else '' artifact_prefix = get_artifact_prefix(task) if worker_implementation == ('docker-worker', 'linux'): local_prefix = '/builds/worker/workspace/' elif worker_implementation == ('generic-worker', 'windows'): local_prefix = '' else: raise NotImplementedError( 'Unsupported worker implementation: "{}"'.format(worker_implementation)) output_files = [] for config in repackage_config: output_files.append({ 'type': 'file', 'path': '{}build/outputs/{}{}' .format(local_prefix, locale_output_path, config['output']), 'name': '{}/{}{}'.format(artifact_prefix, locale_output_path, config['output']), }) return output_files
def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None): artifact_prefix = get_artifact_prefix(job) if locale and locale != 'en-US': artifact_prefix = '{}/{}'.format(artifact_prefix, locale) upstream_artifacts = [{ 'taskId': { 'task-reference': '<partials-signing>' }, 'taskType': 'signing', 'paths': ["{}/{}".format(artifact_prefix, path) for path, _ in artifacts], 'locale': locale or 'en-US', }] return upstream_artifacts
def generate_upstream_artifacts(job, dependencies): artifact_prefix = get_artifact_prefix(job) apks = [{ 'taskId': { 'task-reference': '<{}>'.format(task_kind) }, 'taskType': 'signing', 'paths': ['{}/target.apk'.format(artifact_prefix)], } for task_kind in dependencies.keys() if 'google-play-strings' not in task_kind] google_play_strings = [{ 'taskId': { 'task-reference': '<{}>'.format(task_kind) }, 'taskType': 'build', 'paths': ['public/google_play_strings.json'], 'optional': True, } for task_kind in dependencies.keys() if 'google-play-strings' in task_kind] return apks + google_play_strings
def generate_upstream_artifacts(job, release_history, platform, locale=None): artifact_prefix = get_artifact_prefix(job) if locale: artifact_prefix = '{}/{}'.format(artifact_prefix, locale) else: locale = 'en-US' artifacts = get_partials_artifacts(release_history, platform, locale) upstream_artifacts = [{ "taskId": {"task-reference": '<partials>'}, "taskType": 'partials', "paths": [ "{}/{}".format(artifact_prefix, path) for path, version in artifacts # TODO Use mozilla-version to avoid comparing strings. Otherwise Firefox 100 will be # considered smaller than Firefox 56 if version is None or version >= '56' ], "formats": ["autograph_hash_only_mar384"], }] old_mar_upstream_artifacts = { "taskId": {"task-reference": '<partials>'}, "taskType": 'partials', "paths": [ "{}/{}".format(artifact_prefix, path) for path, version in artifacts # TODO Use mozilla-version to avoid comparing strings. Otherwise Firefox 100 will be # considered smaller than Firefox 56 if version is not None and version < '56' ], "formats": ["mar"], } if old_mar_upstream_artifacts["paths"]: upstream_artifacts.append(old_mar_upstream_artifacts) return upstream_artifacts
def _generate_task_output_files(job, filenames, locale=None): locale_output_path = "{}/".format(locale) if locale else "" artifact_prefix = get_artifact_prefix(job) data = list() for filename in filenames: data.append({ "type": "file", "path": "/home/worker/artifacts/{}".format(filename), "name": "{}/{}{}".format(artifact_prefix, locale_output_path, filename), }) data.append({ "type": "file", "path": "/home/worker/artifacts/manifest.json", "name": "{}/{}manifest.json".format(artifact_prefix, locale_output_path), }) return data
def _generate_task_output_files(job, filenames, locale=None): locale_output_path = '{}/'.format(locale) if locale else '' artifact_prefix = get_artifact_prefix(job) data = list() for filename in filenames: data.append({ 'type': 'file', 'path': '/home/worker/artifacts/{}'.format(filename), 'name': '{}/{}{}'.format(artifact_prefix, locale_output_path, filename) }) data.append({ 'type': 'file', 'path': '/home/worker/artifacts/manifest.json', 'name': '{}/{}manifest.json'.format(artifact_prefix, locale_output_path) }) return data
def _generate_task_output_files(task, build_platform, partner): """We carefully generate an explicit list here, but there's an artifacts directory too, courtesy of generic_worker_add_artifacts() (windows) or docker_worker_add_artifacts(). Any errors here are likely masked by that. """ partner_output_path = '{}/'.format(partner) artifact_prefix = get_artifact_prefix(task) if build_platform.startswith('macosx'): output_files = [{ 'type': 'file', 'path': '/builds/worker/workspace/build/artifacts/{}target.dmg'.format( partner_output_path), 'name': '{}/{}target.dmg'.format(artifact_prefix, partner_output_path), }] elif build_platform.startswith('win'): output_files = [{ 'type': 'file', 'path': '{}/{}target.installer.exe'.format(artifact_prefix, partner_output_path), 'name': '{}/{}target.installer.exe'.format(artifact_prefix, partner_output_path), }] if output_files: return output_files raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform))
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes["build_platform"] if job["build-platform"].startswith("win"): if dep_job.kind.endswith("signing"): continue if job["build-platform"].startswith("macosx"): if dep_job.kind.endswith("repack"): continue dependencies = {dep_job.attributes.get("kind"): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith("macosx") and dependency.endswith( "signing"): signing_task = dependency elif build_platform.startswith("win") and dependency.endswith( "repack"): signing_task = dependency attributes["repackage_type"] = "repackage" repack_id = job["extra"]["repack_id"] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: job["package-formats"].append("installer-stub") repackage_config = [] for format in job.get("package-formats"): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "executable_extension": executable_extension(build_platform), } command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } repackage_config.append(command) run = job.get("mozharness", {}) run.update({ "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, }) worker = { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, "env": { "REPACK_ID": repack_id, }, # Don't add generic artifact directory. "skip-artifacts": True, } worker_type = "b-linux" worker["docker-image"] = {"in-tree": "debian10-amd64-build"} worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job["extra"]["repack_id"], build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "attributes": attributes, "scopes": ["queue:get-artifact:releng/partner/*"], "run-on-projects": dep_job.attributes.get("run_on_projects"), "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer, ), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ]) yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency signing_task_ref = "<{}>".format(signing_task) attributes['repackage_type'] = 'repackage' level = config.params['level'] repack_id = job['extra']['repack_id'] run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['download_input', 'setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', }) worker = { 'env': _generate_task_env(build_platform, signing_task, signing_task_ref, partner=repack_id), 'artifacts': _generate_task_output_files(dep_job, build_platform, partner=repack_id), 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, } worker['env'].update(REPACK_ID=repack_id) if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith('macosx'): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform)) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', ] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency attributes['repackage_type'] = 'repackage' repack_id = job['extra']['repack_id'] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: job['package-formats'].append('installer-stub') repackage_config = [] for format in job.get('package-formats'): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), } command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, } worker_type = 'b-linux' worker['docker-image'] = {"in-tree": "debian8-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] if build_platform.startswith('macosx'): task.setdefault('fetches', {}).setdefault('toolchain', []).extend([ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ]) yield task
def use_fetches(config, jobs): all_fetches = {} for task in config.kind_dependencies_tasks: if task.kind != 'fetch': continue name = task.label.replace('%s-' % task.kind, '') get_attribute(all_fetches, name, task.attributes, 'fetch-artifact') for job in jobs: fetches = job.pop('fetches', None) if not fetches: yield job continue # Hack added for `mach artifact toolchain` to support reading toolchain # kinds in isolation. if 'fetch' in fetches and config.params.get('ignore_fetches'): fetches['fetch'][:] = [] job_fetches = [] name = job.get('name', job.get('label')) dependencies = job.setdefault('dependencies', {}) prefix = get_artifact_prefix(job) for kind, artifacts in fetches.items(): if kind == 'fetch': for fetch in artifacts: if fetch not in all_fetches: raise Exception( 'Missing fetch job for {kind}-{name}: {fetch}'. format(kind=config.kind, name=name, fetch=fetch)) path = all_fetches[fetch] if not path.startswith('public/'): raise Exception( 'Non-public artifacts not supported for {kind}-{name}: ' '{fetch}'.format(kind=config.kind, name=name, fetch=fetch)) dep = 'fetch-{}'.format(fetch) dependencies[dep] = dep job_fetches.append('{path}@<{dep}>'.format(path=path, dep=dep)) else: if kind not in dependencies: raise Exception( "{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind)) for path in artifacts: job_fetches.append('{prefix}/{path}@<{dep}>'.format( prefix=prefix, path=path, dep=kind)) env = job.setdefault('worker', {}).setdefault('env', {}) env['MOZ_FETCHES'] = {'task-reference': ' '.join(job_fetches)} workdir = job['run'].get('workdir', '/builds/worker') env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir)) yield job
def use_fetches(config, jobs): artifact_names = {} for task in config.kind_dependencies_tasks: if task.kind in ('fetch', 'toolchain'): get_attribute( artifact_names, task.label, task.attributes, '{kind}-artifact'.format(kind=task.kind), ) for job in jobs: fetches = job.pop('fetches', None) if not fetches: yield job continue # Hack added for `mach artifact toolchain` to support reading toolchain # kinds in isolation. if 'fetch' in fetches and config.params.get('ignore_fetches'): fetches['fetch'][:] = [] job_fetches = [] name = job.get('name', job.get('label')) dependencies = job.setdefault('dependencies', {}) prefix = get_artifact_prefix(job) for kind, artifacts in fetches.items(): if kind in ('fetch', 'toolchain'): for fetch_name in artifacts: label = '{kind}-{name}'.format(kind=kind, name=fetch_name) if label not in artifact_names: raise Exception( 'Missing fetch job for {kind}-{name}: {fetch}'. format(kind=config.kind, name=name, fetch=fetch_name)) path = artifact_names[label] if not path.startswith('public/'): raise Exception( 'Non-public artifacts not supported for {kind}-{name}: ' '{fetch}'.format(kind=config.kind, name=name, fetch=fetch_name)) dependencies[label] = label job_fetches.append({ 'artifact': path, 'task': '<{label}>'.format(label=label), 'extract': True, }) else: if kind not in dependencies: raise Exception( "{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind)) for artifact in artifacts: if isinstance(artifact, basestring): path = artifact dest = None extract = True else: path = artifact['artifact'] dest = artifact.get('dest') extract = artifact.get('extract', True) fetch = { 'artifact': '{prefix}/{path}'.format(prefix=prefix, path=path), 'task': '<{dep}>'.format(dep=kind), 'extract': extract, } if dest is not None: fetch['dest'] = dest job_fetches.append(fetch) env = job.setdefault('worker', {}).setdefault('env', {}) env['MOZ_FETCHES'] = { 'task-reference': json.dumps(job_fetches, sort_keys=True) } impl, os = worker_type_implementation(job['worker-type']) if os == 'windows': env.setdefault('MOZ_FETCHES_DIR', 'fetches') else: workdir = job['run'].get('workdir', '/builds/worker') env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir)) yield job
def generate_upstream_artifacts(job, signing_task_ref, build_task_ref, platform, locale=None): build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS artifact_prefix = get_artifact_prefix(job) if locale: artifact_prefix = '{}/{}'.format(artifact_prefix, locale) platform = "{}-l10n".format(platform) if platform.endswith("-source"): return [{ "taskId": { "task-reference": signing_task_ref }, "taskType": "signing", "paths": [ "{}/{}".format(artifact_prefix, p) for p in UPSTREAM_SOURCE_ARTIFACTS ], "locale": locale or "en-US", }] upstream_artifacts = [] # Some platforms (like android-api-16-nightly-l10n) may not depend on any unsigned artifact if build_mapping[platform]: upstream_artifacts.append({ "taskId": { "task-reference": build_task_ref }, "taskType": "build", "paths": [ "{}/{}".format(artifact_prefix, p) for p in build_mapping[platform] ], "locale": locale or "en-US", }) upstream_artifacts.append({ "taskId": { "task-reference": signing_task_ref }, "taskType": "signing", "paths": [ "{}/{}".format(artifact_prefix, p) for p in signing_mapping[platform] ], "locale": locale or "en-US", }) if not locale and "android" in platform: # edge case to support 'multi' locale paths multi_platform = "{}-multi".format(platform) upstream_artifacts.extend([{ "taskId": { "task-reference": build_task_ref }, "taskType": "build", "paths": [ "{}/{}".format(artifact_prefix, p) for p in build_mapping[multi_platform] ], "locale": "multi", }, { "taskId": { "task-reference": signing_task_ref }, "taskType": "signing", "paths": [ "{}/{}".format(artifact_prefix, p) for p in signing_mapping[multi_platform] ], "locale": "multi", }]) return upstream_artifacts
def generate_beetmover_artifact_map(config, job, **kwargs): """Generate the beetmover artifact map. Currently only applies to beetmover tasks. Args: config (): Current taskgraph configuration. job (dict): The current job being generated Common kwargs: platform (str): The current build platform locale (str): The current locale being beetmoved. Returns: list: A list of dictionaries containing source->destination maps for beetmover. """ platform = kwargs.get("platform", "") resolve_keyed_by( job, "attributes.artifact_map", job["label"], **{ "release-type": config.params["release_type"], "platform": platform, }, ) map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) base_artifact_prefix = map_config.get( "base_artifact_prefix", get_artifact_prefix(job) ) artifacts = list() dependencies = job["dependencies"].keys() if kwargs.get("locale"): if isinstance(kwargs["locale"], list): locales = kwargs["locale"] else: locales = [kwargs["locale"]] else: locales = map_config["default_locales"] resolve_keyed_by( map_config, "s3_bucket_paths", job["label"], **{"build-type": job["attributes"]["build-type"]}, ) for locale, dep in sorted(itertools.product(locales, dependencies)): paths = dict() for filename in map_config["mapping"]: # Relevancy checks if dep not in map_config["mapping"][filename]["from"]: # We don't get this file from this dependency. continue if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: # This locale either doesn't produce or shouldn't upload this file. continue if ( "only_for_platforms" in map_config["mapping"][filename] and platform not in map_config["mapping"][filename]["only_for_platforms"] ): # This platform either doesn't produce or shouldn't upload this file. continue if ( "not_for_platforms" in map_config["mapping"][filename] and platform in map_config["mapping"][filename]["not_for_platforms"] ): # This platform either doesn't produce or shouldn't upload this file. continue if "partials_only" in map_config["mapping"][filename]: continue # deepcopy because the next time we look at this file the locale will differ. file_config = deepcopy(map_config["mapping"][filename]) for field in [ "destinations", "locale_prefix", "source_path_modifier", "update_balrog_manifest", "pretty_name", "checksums_path", ]: resolve_keyed_by(file_config, field, job["label"], locale=locale) # This format string should ideally be in the configuration file, # but this would mean keeping variable names in sync between code + config. destinations = [ "{s3_bucket_path}/{dest_path}/{filename}".format( s3_bucket_path=bucket_path, dest_path=dest_path, locale_prefix=file_config["locale_prefix"], filename=file_config.get("pretty_name", filename), ) for dest_path, bucket_path in itertools.product( file_config["destinations"], map_config["s3_bucket_paths"] ) ] # Creating map entries # Key must be artifact path, to avoid trampling duplicates, such # as public/build/target.apk and public/build/multi/target.apk key = os.path.join( base_artifact_prefix, file_config["source_path_modifier"], filename, ) paths[key] = { "destinations": destinations, } if file_config.get("checksums_path"): paths[key]["checksums_path"] = file_config["checksums_path"] # optional flag: balrog manifest if file_config.get("update_balrog_manifest"): paths[key]["update_balrog_manifest"] = True if file_config.get("balrog_format"): paths[key]["balrog_format"] = file_config["balrog_format"] if not paths: # No files for this dependency/locale combination. continue # Render all variables for the artifact map platforms = deepcopy(map_config.get("platform_names", {})) if platform: for key in platforms.keys(): resolve_keyed_by(platforms, key, job["label"], platform=platform) version = read_version_file() upload_date = datetime.fromtimestamp(config.params["build_date"]) if job["attributes"]["build-type"] == "nightly": folder_prefix = upload_date.strftime("%Y/%m/%Y-%m-%d-%H-%M-%S-") # TODO: Remove this when version.txt has versioning fixed version = version.split("-")[0] else: folder_prefix = f"{version}/android/" kwargs.update( {"locale": locale, "version": version, "folder_prefix": folder_prefix} ) kwargs.update(**platforms) paths = jsone.render(paths, kwargs) artifacts.append( { "taskId": {"task-reference": "<{}>".format(dep)}, "locale": locale, "paths": paths, } ) return artifacts
def _generate_task_output_files(task, build_platform, locale=None): locale_output_path = '{}/'.format(locale) if locale else '' artifact_prefix = get_artifact_prefix(task) if build_platform.startswith('linux') or build_platform.startswith( 'macosx'): output_files = [{ 'type': 'file', 'path': '/builds/worker/workspace/build/artifacts/{}target.complete.mar'. format(locale_output_path), 'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), }] if build_platform.startswith('macosx'): output_files.append({ 'type': 'file', 'path': '/builds/worker/workspace/build/artifacts/{}target.dmg'.format( locale_output_path), 'name': '{}/{}target.dmg'.format(artifact_prefix, locale_output_path), }) elif build_platform.startswith('win'): output_files = [{ 'type': 'file', 'path': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path), 'name': '{}/{}target.installer.exe'.format(artifact_prefix, locale_output_path), }, { 'type': 'file', 'path': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), 'name': '{}/{}target.complete.mar'.format(artifact_prefix, locale_output_path), }] # Stub installer is only generated on win32 if '32' in build_platform: output_files.append({ 'type': 'file', 'path': '{}/{}target.stub-installer.exe'.format( artifact_prefix, locale_output_path), 'name': '{}/{}target.stub-installer.exe'.format( artifact_prefix, locale_output_path), }) if output_files: return output_files raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform))
def generate_upstream_artifacts(job, build_task_ref, build_signing_task_ref, repackage_task_ref, repackage_signing_task_ref, platform, locale=None, project=None): build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS artifact_prefix = get_artifact_prefix(job) if locale: artifact_prefix = '{}/{}'.format(artifact_prefix, locale) platform = "{}-l10n".format(platform) upstream_artifacts = [] task_refs = [ build_task_ref, build_signing_task_ref, repackage_task_ref, repackage_signing_task_ref ] tasktypes = ['build', 'signing', 'repackage', 'repackage'] mapping = [ build_mapping, build_signing_mapping, repackage_mapping, repackage_signing_mapping ] for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping): platform_was_previously_matched_by_regex = None for platform_regex, paths in mapping.iteritems(): if platform_regex.match(platform) is not None: _check_platform_matched_only_one_regex( tasktype, platform, platform_was_previously_matched_by_regex, platform_regex) if paths: usable_paths = paths[:] use_stub = job["attributes"].get('stub-installer') if not use_stub: if 'target.stub-installer.exe' in usable_paths: usable_paths.remove('target.stub-installer.exe') upstream_artifacts.append({ "taskId": { "task-reference": ref }, "taskType": tasktype, "paths": [ "{}/{}".format(artifact_prefix, path) for path in usable_paths ], "locale": locale or "en-US", }) platform_was_previously_matched_by_regex = platform_regex return upstream_artifacts
def generate_beetmover_upstream_artifacts( config, job, platform, locale=None, dependencies=None, **kwargs ): """Generate the upstream artifacts for beetmover, using the artifact map. Currently only applies to beetmover tasks. Args: job (dict): The current job being generated dependencies (list): A list of the job's dependency labels. platform (str): The current build platform locale (str): The current locale being beetmoved. Returns: list: A list of dictionaries conforming to the upstream_artifacts spec. """ base_artifact_prefix = get_artifact_prefix(job) resolve_keyed_by( job, "attributes.artifact_map", "artifact map", **{ "release-type": config.params["release_type"], "platform": platform, }, ) map_config = deepcopy(cached_load_yaml(job["attributes"]["artifact_map"])) upstream_artifacts = list() if not locale: locales = map_config["default_locales"] elif isinstance(locale, list): locales = locale else: locales = [locale] if not dependencies: if job.get("dependencies"): dependencies = job["dependencies"].keys() elif job.get("primary-dependency"): dependencies = [job["primary-dependency"].kind] else: raise Exception("Unsupported type of dependency. Got job: {}".format(job)) for locale, dep in itertools.product(locales, dependencies): paths = list() for filename in map_config["mapping"]: if dep not in map_config["mapping"][filename]["from"]: continue if locale != "multi" and not map_config["mapping"][filename]["all_locales"]: continue if ( "only_for_platforms" in map_config["mapping"][filename] and platform not in map_config["mapping"][filename]["only_for_platforms"] ): continue if ( "not_for_platforms" in map_config["mapping"][filename] and platform in map_config["mapping"][filename]["not_for_platforms"] ): continue if "partials_only" in map_config["mapping"][filename]: continue # The next time we look at this file it might be a different locale. file_config = deepcopy(map_config["mapping"][filename]) resolve_keyed_by( file_config, "source_path_modifier", "source path modifier", locale=locale, ) kwargs["locale"] = locale paths.append( os.path.join( base_artifact_prefix, jsone.render(file_config["source_path_modifier"], kwargs), jsone.render(filename, kwargs), ) ) if job.get("dependencies") and getattr( job["dependencies"][dep], "release_artifacts", None ): paths = [ path for path in paths if path in job["dependencies"][dep].release_artifacts ] if not paths: continue upstream_artifacts.append( { "taskId": {"task-reference": "<{}>".format(dep)}, "taskType": map_config["tasktype_map"].get(dep), "paths": sorted(paths), "locale": locale, } ) upstream_artifacts.sort(key=lambda u: u["paths"]) return upstream_artifacts
def use_fetches(config, jobs): artifact_names = {} aliases = {} if config.kind in ("toolchain", "fetch"): jobs = list(jobs) for job in jobs: run = job.get("run", {}) label = job["label"] get_attribute(artifact_names, label, run, "toolchain-artifact") value = run.get("{}-alias".format(config.kind)) if value: aliases["{}-{}".format(config.kind, value)] = label for task in config.kind_dependencies_tasks.values(): if task.kind in ("fetch", "toolchain"): get_attribute( artifact_names, task.label, task.attributes, "{kind}-artifact".format(kind=task.kind), ) value = task.attributes.get("{}-alias".format(task.kind)) if value: aliases["{}-{}".format(task.kind, value)] = task.label artifact_prefixes = {} for job in order_tasks(config, jobs): artifact_prefixes[job["label"]] = get_artifact_prefix(job) fetches = job.pop("fetches", None) if not fetches: yield job continue job_fetches = [] name = job.get("name", job.get("label")) dependencies = job.setdefault("dependencies", {}) worker = job.setdefault("worker", {}) prefix = get_artifact_prefix(job) has_sccache = False for kind, artifacts in fetches.items(): if kind in ("fetch", "toolchain"): for fetch_name in artifacts: label = "{kind}-{name}".format(kind=kind, name=fetch_name) label = aliases.get(label, label) if label not in artifact_names: raise Exception( "Missing fetch job for {kind}-{name}: {fetch}".format( kind=config.kind, name=name, fetch=fetch_name ) ) path = artifact_names[label] dependencies[label] = label job_fetches.append( { "artifact": path, "task": "<{label}>".format(label=label), "extract": True, } ) if kind == "toolchain" and fetch_name.endswith("-sccache"): has_sccache = True else: if kind not in dependencies: raise Exception( "{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind) ) dep_label = dependencies[kind] if dep_label in artifact_prefixes: prefix = artifact_prefixes[dep_label] else: if dep_label not in config.kind_dependencies_tasks: raise Exception( "{name} can't fetch {kind} artifacts because " "there are no tasks with label {label} in kind dependencies!".format( name=name, kind=kind, label=dependencies[kind], ) ) prefix = get_artifact_prefix( config.kind_dependencies_tasks[dep_label] ) for artifact in artifacts: if isinstance(artifact, text_type): path = artifact dest = None extract = True verify_hash = False else: path = artifact["artifact"] dest = artifact.get("dest") extract = artifact.get("extract", True) verify_hash = artifact.get("verify-hash", False) fetch = { "artifact": "{prefix}/{path}".format(prefix=prefix, path=path) if not path.startswith("/") else path[1:], "task": "<{dep}>".format(dep=kind), "extract": extract, } if dest is not None: fetch["dest"] = dest if verify_hash: fetch["verify-hash"] = verify_hash job_fetches.append(fetch) if job.get("use-sccache") and not has_sccache: raise Exception("Must provide an sccache toolchain if using sccache.") job_artifact_prefixes = { mozpath.dirname(fetch["artifact"]) for fetch in job_fetches if not fetch["artifact"].startswith("public/") } if job_artifact_prefixes: # Use taskcluster-proxy and request appropriate scope. For example, add # 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'. worker["taskcluster-proxy"] = True for prefix in sorted(job_artifact_prefixes): scope = "queue:get-artifact:{}/*".format(prefix) if scope not in job.setdefault("scopes", []): job["scopes"].append(scope) env = worker.setdefault("env", {}) env["MOZ_FETCHES"] = { "task-reference": six.ensure_text( json.dumps( sorted(job_fetches, key=lambda x: sorted(x.items())), sort_keys=True ) ) } # The path is normalized to an absolute path in run-task env.setdefault("MOZ_FETCHES_DIR", "fetches") yield job
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['primary-dependency'] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) attributes['shipping_phase'] = job['shipping-phase'] build_locale = locale or 'en-US' build_platform = attributes['build_platform'] builds = get_builds(config.params['release_history'], build_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {'funsize': {'partials': list()}} update_number = 1 locale_suffix = '' if locale: locale_suffix = '{}/'.format(locale) artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'artifact-reference': artifact_path }, 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 level = config.params['level'] worker = { 'artifacts': _generate_task_output_files(dep_job, builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600 if 'asan' in dep_job.label else 900, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SIGNING_CERT': identify_desired_signing_keys( config.params["project"], config.params['release_product']), 'EXTRA_PARAMS': '--arch={}'.format(architecture(build_platform)), 'MAR_CHANNEL_ID': attributes['mar-channel-id'] } } if config.params.release_level() == 'staging': worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true' task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'b-linux', 'dependencies': dependencies, 'scopes': [], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 \ and any([build_platform.startswith(prefix) for prefix in ['linux', 'win']]): task['scopes'].append( 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ) yield task
def generic_worker_add_artifacts(config, job, taskdesc): """ Adds an artifact directory to the task """ # The path is the location on disk; it doesn't necessarily # mean the artifacts will be public or private; that is set via the name # attribute in add_artifacts. add_artifacts(config, job, taskdesc, path=get_artifact_prefix(taskdesc))
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['primary-dependency'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) attributes['shipping_phase'] = job['shipping-phase'] build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {'funsize': {'partials': list()}} update_number = 1 locale_suffix = '' if locale: locale_suffix = '{}/'.format(locale) artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'artifact-reference': artifact_path }, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 mar_channel_id = None if config.params['project'] == 'mozilla-beta': if 'devedition' in label: mar_channel_id = 'firefox-mozilla-aurora' else: mar_channel_id = 'firefox-mozilla-beta' elif config.params['project'] == 'mozilla-release': mar_channel_id = 'firefox-mozilla-release' elif 'esr' in config.params['project']: mar_channel_id = 'firefox-mozilla-esr' level = config.params['level'] worker = { 'artifacts': _generate_task_output_files(dep_job, builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600 if 'asan' in dep_job.label else 600, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384', 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-{}/datadog-api-key'.format( level), 'EXTRA_PARAMS': '--arch={}'.format(architecture(attributes['build_platform'])), } } if mar_channel_id: worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id if config.params.release_level() == 'staging': worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true' task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'scopes': [ 'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level ], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 \ and any([platform in dep_th_platform for platform in ['linux', 'windows']]): task['scopes'].append( 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ) yield task
def generate_upstream_artifacts(job, build_task_ref, repackage_task_ref, repackage_signing_task_ref, platform, repack_id, partner_path): upstream_artifacts = [] artifact_prefix = get_artifact_prefix(job) if "linux" in platform: upstream_artifacts.append({ "taskId": { "task-reference": build_task_ref }, "taskType": "build", "paths": ["{}/{}/target.tar.bz2".format(artifact_prefix, repack_id)], "locale": partner_path, }) upstream_artifacts.append({ "taskId": { "task-reference": repackage_signing_task_ref }, "taskType": "repackage", "paths": ["{}/{}/target.tar.bz2.asc".format(artifact_prefix, repack_id)], "locale": partner_path, }) elif "macosx" in platform: upstream_artifacts.append({ "taskId": { "task-reference": repackage_task_ref }, "taskType": "repackage", "paths": ["{}/{}/target.dmg".format(artifact_prefix, repack_id)], "locale": partner_path, }) upstream_artifacts.append({ "taskId": { "task-reference": repackage_signing_task_ref }, "taskType": "repackage", "paths": ["{}/{}/target.dmg.asc".format(artifact_prefix, repack_id)], "locale": partner_path, }) elif "win" in platform: upstream_artifacts.append({ "taskId": { "task-reference": repackage_signing_task_ref }, "taskType": "repackage", "paths": ["{}/{}/target.installer.exe".format(artifact_prefix, repack_id)], "locale": partner_path, }) upstream_artifacts.append({ "taskId": { "task-reference": repackage_signing_task_ref }, "taskType": "repackage", "paths": [ "{}/{}/target.installer.exe.asc".format( artifact_prefix, repack_id) ], "locale": partner_path, }) if not upstream_artifacts: raise Exception("Couldn't find any upstream artifacts.") return upstream_artifacts
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency attributes['repackage_type'] = 'repackage' level = config.params['level'] repack_id = job['extra']['repack_id'] repackage_config = [] for format in job.get('package-formats'): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), } command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, } if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith('macosx'): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform)) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"]), } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task
def use_fetches(config, jobs): artifact_names = {} aliases = {} if config.kind in ('toolchain', 'fetch'): jobs = list(jobs) for job in jobs: run = job.get('run', {}) label = job['label'] get_attribute( artifact_names, label, run, 'toolchain-artifact') value = run.get('{}-alias'.format(config.kind)) if value: aliases['{}-{}'.format(config.kind, value)] = label for task in config.kind_dependencies_tasks: if task.kind in ('fetch', 'toolchain'): get_attribute( artifact_names, task.label, task.attributes, '{kind}-artifact'.format(kind=task.kind), ) value = task.attributes.get('{}-alias'.format(task.kind)) if value: aliases['{}-{}'.format(task.kind, value)] = task.label artifact_prefixes = {} for job in order_tasks(config, jobs): artifact_prefixes[job["label"]] = get_artifact_prefix(job) fetches = job.pop("fetches", None) if not fetches: yield job continue job_fetches = [] name = job.get('name', job.get('label')) dependencies = job.setdefault('dependencies', {}) worker = job.setdefault('worker', {}) prefix = get_artifact_prefix(job) for kind, artifacts in fetches.items(): if kind in ('fetch', 'toolchain'): for fetch_name in artifacts: label = '{kind}-{name}'.format(kind=kind, name=fetch_name) label = aliases.get(label, label) if label not in artifact_names: raise Exception('Missing fetch job for {kind}-{name}: {fetch}'.format( kind=config.kind, name=name, fetch=fetch_name)) path = artifact_names[label] dependencies[label] = label job_fetches.append({ 'artifact': path, 'task': '<{label}>'.format(label=label), 'extract': True, }) if kind == 'toolchain' and fetch_name.endswith('-sccache'): job['needs-sccache'] = True else: if kind not in dependencies: raise Exception("{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind)) dep_label = dependencies[kind] if dep_label in artifact_prefixes: prefix = artifact_prefixes[dep_label] else: dep_tasks = [ task for task in config.kind_dependencies_tasks if task.label == dep_label ] if len(dep_tasks) != 1: raise Exception( "{name} can't fetch {kind} artifacts because " "there are {tasks} with label {label} in kind dependencies!".format( name=name, kind=kind, label=dependencies[kind], tasks="no tasks" if len(dep_tasks) == 0 else "multiple tasks", ) ) prefix = get_artifact_prefix(dep_tasks[0]) for artifact in artifacts: if isinstance(artifact, text_type): path = artifact dest = None extract = True else: path = artifact['artifact'] dest = artifact.get('dest') extract = artifact.get('extract', True) fetch = { 'artifact': '{prefix}/{path}'.format(prefix=prefix, path=path) if not path.startswith('/') else path[1:], 'task': '<{dep}>'.format(dep=kind), 'extract': extract, } if dest is not None: fetch['dest'] = dest job_fetches.append(fetch) job_artifact_prefixes = { mozpath.dirname(fetch["artifact"]) for fetch in job_fetches if not fetch["artifact"].startswith("public/") } if job_artifact_prefixes: # Use taskcluster-proxy and request appropriate scope. For example, add # 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'. worker["taskcluster-proxy"] = True for prefix in sorted(job_artifact_prefixes): scope = "queue:get-artifact:{}/*".format(prefix) if scope not in job.setdefault("scopes", []): job["scopes"].append(scope) env = worker.setdefault('env', {}) env['MOZ_FETCHES'] = { 'task-reference': six.ensure_text(json.dumps(job_fetches, sort_keys=True)) } # The path is normalized to an absolute path in run-task env.setdefault('MOZ_FETCHES_DIR', 'fetches') yield job
def use_fetches(config, jobs): artifact_names = {} aliases = {} if config.kind == 'toolchain': jobs = list(jobs) for job in jobs: run = job.get('run', {}) label = 'toolchain-{}'.format(job['name']) get_attribute( artifact_names, label, run, 'toolchain-artifact') value = run.get('toolchain-alias') if value: aliases['toolchain-{}'.format(value)] = label for task in config.kind_dependencies_tasks: if task.kind in ('fetch', 'toolchain'): get_attribute( artifact_names, task.label, task.attributes, '{kind}-artifact'.format(kind=task.kind), ) value = task.attributes.get('{}-alias'.format(task.kind)) if value: aliases['{}-{}'.format(task.kind, value)] = task.label for job in jobs: fetches = job.pop('fetches', None) if not fetches: yield job continue job_fetches = [] name = job.get('name', job.get('label')) dependencies = job.setdefault('dependencies', {}) worker = job.setdefault('worker', {}) prefix = get_artifact_prefix(job) for kind, artifacts in fetches.items(): if kind in ('fetch', 'toolchain'): for fetch_name in artifacts: label = '{kind}-{name}'.format(kind=kind, name=fetch_name) label = aliases.get(label, label) if label not in artifact_names: raise Exception('Missing fetch job for {kind}-{name}: {fetch}'.format( kind=config.kind, name=name, fetch=fetch_name)) path = artifact_names[label] if not path.startswith('public/'): # Use taskcluster-proxy and request appropriate scope. For example, add # 'scopes: [queue:get-artifact:path/to/*]' for 'path/to/artifact.tar.xz'. worker['taskcluster-proxy'] = True dirname = mozpath.dirname(path) scope = 'queue:get-artifact:{}/*'.format(dirname) if scope not in job.setdefault('scopes', []): job['scopes'].append(scope) dependencies[label] = label job_fetches.append({ 'artifact': path, 'task': '<{label}>'.format(label=label), 'extract': True, }) if kind == 'toolchain' and fetch_name.endswith('-sccache'): job['needs-sccache'] = True else: if kind not in dependencies: raise Exception("{name} can't fetch {kind} artifacts because " "it has no {kind} dependencies!".format(name=name, kind=kind)) for artifact in artifacts: if isinstance(artifact, basestring): path = artifact dest = None extract = True else: path = artifact['artifact'] dest = artifact.get('dest') extract = artifact.get('extract', True) fetch = { 'artifact': '{prefix}/{path}'.format(prefix=prefix, path=path) if not path.startswith('/') else path[1:], 'task': '<{dep}>'.format(dep=kind), 'extract': extract, } if dest is not None: fetch['dest'] = dest job_fetches.append(fetch) env = worker.setdefault('env', {}) env['MOZ_FETCHES'] = {'task-reference': json.dumps(job_fetches, sort_keys=True)} # The path is normalized to an absolute path in run-task env.setdefault('MOZ_FETCHES_DIR', 'fetches') yield job
def generate_upstream_artifacts( config, job, dependencies, platform, locale=None, project=None ): build_mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS build_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS repackage_mapping = UPSTREAM_ARTIFACT_REPACKAGE_PATHS repackage_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS msi_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_MSI_PATHS mar_signing_mapping = UPSTREAM_ARTIFACT_SIGNED_MAR_PATHS artifact_prefix = get_artifact_prefix(job) if locale: artifact_prefix = '{}/{}'.format(artifact_prefix, locale) platform = "{}-l10n".format(platform) upstream_artifacts = [] for task_type, mapping in [ ("build", build_mapping), ("signing", build_signing_mapping), ]: platform_was_previously_matched_by_regex = None for platform_regex, paths in mapping.iteritems(): if platform_regex.match(platform) is not None: _check_platform_matched_only_one_regex( task_type, platform, platform_was_previously_matched_by_regex, platform_regex ) platform_was_previously_matched_by_regex = platform_regex if paths: usable_paths = paths[:] if 'target.langpack.xpi' in usable_paths and \ not project == "mozilla-central": # XXX This is only beetmoved for m-c nightlies. # we should determine this better usable_paths.remove('target.langpack.xpi') if not len(usable_paths): # We may have removed our only path. continue upstream_artifacts.append({ "taskId": {"task-reference": "<{}>".format(task_type)}, "taskType": task_type, "paths": ["{}/{}".format(artifact_prefix, path) for path in usable_paths], "locale": locale or "en-US", }) for task_type, cot_type, paths in [ ('repackage', 'repackage', repackage_mapping), ('repackage-signing', 'repackage', repackage_signing_mapping), ('repackage-signing-msi', 'repackage', msi_signing_mapping), ('mar-signing', 'signing', mar_signing_mapping), ]: if task_type not in dependencies: continue paths = ["{}/{}".format(artifact_prefix, path) for path in paths] paths = [ path for path in paths if path in dependencies[task_type].release_artifacts] if not paths: continue upstream_artifacts.append({ "taskId": {"task-reference": "<{}>".format(task_type)}, "taskType": cot_type, "paths": paths, "locale": locale or "en-US", }) return upstream_artifacts
def add_artifacts(config, job, taskdesc, path): taskdesc['worker'].setdefault('artifacts', []).append({ 'name': get_artifact_prefix(taskdesc), 'path': path, 'type': 'directory', })