def define_upstream_artifacts(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) if not partner_configs: return for job in jobs: dep_job = job['dependent-task'] build_platform = dep_job.attributes.get('build_platform') repack_ids = [] for partner, partner_config in partner_configs.iteritems(): for sub_partner, cfg in partner_config.iteritems(): if not cfg or build_platform not in cfg["platforms"]: continue for locale in locales_per_build_platform(build_platform, cfg.get('locales', [])): repack_ids.append("{}/{}/{}".format(partner, sub_partner, locale)) artifacts_specifications = generate_specifications_of_artifacts_to_sign( dep_job, keep_locale_template=True, kind=config.kind, ) job['upstream-artifacts'] = [{ 'taskId': {'task-reference': '<{}>'.format(job['depname'])}, 'taskType': 'build', 'paths': [ path_template.format(locale=repack_id) for repack_id in repack_ids for path_template in spec['artifacts'] ], 'formats': spec['formats'], } for spec in artifacts_specifications] yield job
def chunk_partners(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) for job in jobs: dep_job = job['primary-dependency'] build_platform = dep_job.attributes["build_platform"] # already chunked if dep_job.task.get('extra', {}).get('repack_id'): repack_id = dep_job.task['extra']['repack_id'] if _check_repack_ids_by_platform(build_platform, repack_id): continue partner_job = copy.deepcopy(job) partner_job.setdefault('extra', {}).setdefault('repack_id', repack_id) yield partner_job continue # not already chunked for partner, partner_config in partner_configs.iteritems(): for sub_partner, cfg in partner_config.iteritems(): if build_platform not in cfg.get("platforms", []): continue locales = locales_per_build_platform(build_platform, cfg.get('locales', [])) for locale in locales: repack_id = "{}/{}/{}".format(partner, sub_partner, locale) if _check_repack_ids_by_platform(build_platform, repack_id): continue partner_job = copy.deepcopy(job) # don't overwrite dict values here partner_job.setdefault('extra', {}) partner_job['extra']['repack_id'] = repack_id yield partner_job
def define_upstream_artifacts(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) if not partner_configs: return for job in jobs: dep_job = job['primary-dependency'] repack_ids = job['extra']['repack_ids'] artifacts_specifications = generate_specifications_of_artifacts_to_sign( dep_job, keep_locale_template=True, kind=config.kind, ) job['upstream-artifacts'] = [{ 'taskId': { 'task-reference': '<{}>'.format(job['depname']) }, 'taskType': 'build', 'paths': [ path_template.format(locale=repack_id) for path_template in spec['artifacts'] for repack_id in repack_ids ], 'formats': spec['formats'], } for spec in artifacts_specifications] yield job
def define_upstream_artifacts(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) if not partner_configs: return for job in jobs: dep_job = job['primary-dependency'] job['depname'] = dep_job.label job['attributes'] = copy_attributes_from_dependent_job(dep_job) repack_ids = job['extra']['repack_ids'] artifacts_specifications = generate_specifications_of_artifacts_to_sign( config, job, keep_locale_template=True, kind=config.kind, ) task_type = 'build' if 'notarization' in job['depname']: task_type = 'scriptworker' job['upstream-artifacts'] = [{ 'taskId': {'task-reference': '<{}>'.format(dep_job.kind)}, 'taskType': task_type, 'paths': [ path_template.format(locale=repack_id) for path_template in spec['artifacts'] for repack_id in repack_ids ], 'formats': spec['formats'], } for spec in artifacts_specifications] yield job
def chunk_partners(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) for job in jobs: dep_job = job['primary-dependency'] build_platform = dep_job.attributes["build_platform"] repack_id = dep_job.task.get('extra', {}).get('repack_id') repack_ids = dep_job.task.get('extra', {}).get('repack_ids') copy_repack_ids = job.pop('copy-repack-ids', False) if copy_repack_ids: assert repack_ids, "dep_job {} doesn't have repack_ids!".format( dep_job.label) job.setdefault('extra', {})['repack_ids'] = repack_ids yield job # first downstream of the repack task, no chunking or fanout has been done yet elif not any([repack_id, repack_ids]): platform_repack_ids = _get_repack_ids_by_platform( partner_configs, build_platform) # we chunk mac signing if config.kind in ("release-partner-repack-signing", "release-eme-free-repack-signing", "release-partner-repack-notarization-part-1", "release-eme-free-repack-notarization-part-1"): repacks_per_chunk = job.get('repacks-per-chunk') chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk) if remainder: chunks = int(chunks + 1) for this_chunk in range(1, chunks + 1): chunk = chunkify(platform_repack_ids, this_chunk, chunks) partner_job = copy.deepcopy(job) partner_job.setdefault('extra', {}).setdefault('repack_ids', chunk) partner_job['extra']['repack_suffix'] = str(this_chunk) yield partner_job # linux and windows we fan out immediately to one task per partner-sub_partner-locale else: for repack_id in platform_repack_ids: partner_job = copy.deepcopy( job) # don't overwrite dict values here partner_job.setdefault('extra', {}) partner_job['extra']['repack_id'] = repack_id yield partner_job # fan out chunked mac signing for repackage elif repack_ids: for repack_id in repack_ids: partner_job = copy.deepcopy(job) partner_job.setdefault('extra', {}).setdefault('repack_id', repack_id) yield partner_job # otherwise we've fully fanned out already, continue by passing repack_id on else: partner_job = copy.deepcopy(job) partner_job.setdefault('extra', {}).setdefault('repack_id', repack_id) yield partner_job
def split_public_and_private(config, jobs): public_bucket_scope = get_beetmover_bucket_scope(config) partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) partner, subpartner, _ = job["extra"]["repack_id"].split("/") if partner_config[partner][subpartner].get("upload_to_candidates"): # public yield populate_scopes_and_worker_type( config, job, public_bucket_scope, partner_public=True ) else: # private yield populate_scopes_and_worker_type( config, job, partner_bucket_scope, partner_public=False )
def add_command_arguments(config, tasks): release_config = get_release_config(config) # staging releases - pass reduced set of locales to the repacking script all_locales = set() partner_config = get_partner_config_by_kind(config, config.kind) for partner in partner_config.values(): for sub_partner in partner.values(): all_locales.update(sub_partner.get("locales", [])) for task in tasks: # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64 if not task["attributes"]["build_platform"].endswith("-shippable"): raise Exception( "Unexpected partner repack platform: {}".format( task["attributes"]["build_platform"], ), ) platform = task["attributes"]["build_platform"].partition( "-shippable")[0] task["run"]["options"] = [ "version={}".format(release_config["version"]), "build-number={}".format(release_config["build_number"]), "platform={}".format(platform), ] if task["extra"]["limit-locales"]: for locale in all_locales: task["run"]["options"].append("limit-locale={}".format(locale)) if "partner" in config.kind and config.params["release_partners"]: for partner in config.params["release_partners"]: task["run"]["options"].append("partner={}".format(partner)) # The upstream taskIds are stored a special environment variable, because we want to use # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS # blocks that. It's space-separated string of ids in the end. task["worker"]["env"]["UPSTREAM_TASKIDS"] = { "task-reference": " ".join(["<{}>".format(dep) for dep in task["dependencies"]]) } # Forward the release type for bouncer product construction task["worker"]["env"]["RELEASE_TYPE"] = config.params["release_type"] yield task
def add_command_arguments(config, tasks): release_config = get_release_config(config) # staging releases - pass reduced set of locales to the repacking script all_locales = set() partner_config = get_partner_config_by_kind(config, config.kind) for partner in partner_config.values(): for sub_partner in partner.values(): all_locales.update(sub_partner.get('locales', [])) for task in tasks: # add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64 if not task['attributes']['build_platform'].endswith('-shippable'): raise Exception( "Unexpected partner repack platform: {}".format( task['attributes']['build_platform'], ), ) platform = task['attributes']['build_platform'].partition( '-shippable')[0] task['run']['options'] = [ 'version={}'.format(release_config['version']), 'build-number={}'.format(release_config['build_number']), 'platform={}'.format(platform), ] if task['extra']['limit-locales']: for locale in all_locales: task['run']['options'].append('limit-locale={}'.format(locale)) if 'partner' in config.kind and config.params['release_partners']: for partner in config.params['release_partners']: task['run']['options'].append('partner={}'.format(partner)) # The upstream taskIds are stored a special environment variable, because we want to use # task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS # blocks that. It's space-separated string of ids in the end. task['worker']['env']['UPSTREAM_TASKIDS'] = { 'task-reference': ' '.join(['<{}>'.format(dep) for dep in task['dependencies']]) } # Forward the release type for bouncer product construction task['worker']['env']['RELEASE_TYPE'] = config.params['release_type'] yield task
def split_public_and_private(config, jobs): public_bucket_scope = get_beetmover_bucket_scope(config) partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: partner_bucket_scope = add_scope_prefix(config, job['partner-bucket-scope']) partner, subpartner, _ = job['extra']['repack_id'].split('/') # public if partner_config[partner][subpartner].get('upload_to_candidates'): yield populate_scopes_and_worker_type(config, job, public_bucket_scope, partner_public=True) # private yield populate_scopes_and_worker_type(config, job, partner_bucket_scope, partner_public=False)
def chunk_partners(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) for job in jobs: dep_job = job['dependent-task'] build_platform = dep_job.attributes["build_platform"] for partner, partner_config in partner_configs.iteritems(): for sub_partner, cfg in partner_config.iteritems(): if build_platform not in cfg.get("platforms", []): continue locales = locales_per_build_platform(build_platform, cfg.get('locales', [])) for locale in locales: repack_id = "{}/{}/{}".format(partner, sub_partner, locale) partner_job = copy.deepcopy( job) # don't overwrite dict values here partner_job.setdefault('extra', {}) partner_job['extra']['repack_id'] = repack_id yield partner_job
def define_upstream_artifacts(config, jobs): partner_configs = get_partner_config_by_kind(config, config.kind) if not partner_configs: return for job in jobs: dep_job = job["primary-dependency"] job["depname"] = dep_job.label job["attributes"] = copy_attributes_from_dependent_job(dep_job) repack_ids = job["extra"]["repack_ids"] artifacts_specifications = generate_specifications_of_artifacts_to_sign( config, job, keep_locale_template=True, kind=config.kind, ) task_type = "build" if "notarization" in job["depname"]: task_type = "scriptworker" job["upstream-artifacts"] = [{ "taskId": { "task-reference": "<{}>".format(dep_job.kind) }, "taskType": task_type, "paths": [ path_template.format(locale=repack_id) for path_template in spec["artifacts"] for repack_id in repack_ids ], "formats": spec["formats"], } for spec in artifacts_specifications] yield job
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.stub-installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"] }] task = { 'label': label, 'description': description, 'worker-type': 'linux-signing', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency attributes['repackage_type'] = 'repackage' repack_id = job['extra']['repack_id'] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: job['package-formats'].append('installer-stub') repackage_config = [] for format in job.get('package-formats'): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), } command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, } if build_platform.startswith('win'): worker_type = 'b-win2012' run['use-magic-mh-args'] = False else: if build_platform.startswith('macosx'): worker_type = 'b-linux' else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform)) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task
def make_task_worker(config, jobs): for job in jobs: platform = job["attributes"]["build_platform"] repack_id = job["extra"]["repack_id"] partner, subpartner, locale = job['extra']['repack_id'].split('/') partner_config = get_partner_config_by_kind(config, config.kind) repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') build_task = None repackage_task = None repackage_signing_task = None for dependency in job["dependencies"].keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'repackage' in dependency: repackage_task = dependency else: build_task = "build" build_task_ref = "<" + str(build_task) + ">" repackage_task_ref = "<" + str(repackage_task) + ">" repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">" # generate the partner path; we'll send this to beetmover as the "locale" ftp_platform = get_ftp_platform(platform) repl_dict = { "build_number": config.params['build_number'], "locale": locale, "partner": partner, "platform": ftp_platform, "release_partner_build_number": config.params['release_partner_build_number'], "subpartner": subpartner, "version": config.params['version'], } partner_public = job['partner_public'] if partner_public: partner_path_key = 'partner-public-path' else: partner_path_key = 'partner-private-path' # Kinds can set these to None if not job[partner_path_key]: continue partner_path = job[partner_path_key].format(**repl_dict) del (job['partner_public']) del (job['partner-private-path']) del (job['partner-public-path']) del (job['partner-bucket-scope']) worker = { 'implementation': 'beetmover', 'release-properties': craft_release_properties(config, job), 'upstream-artifacts': generate_upstream_artifacts(job, build_task_ref, repackage_task_ref, repackage_signing_task_ref, platform, repack_id, partner_path, repack_stub_installer), 'partner-public': partner_public, } job["worker"] = worker yield job
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes["build_platform"] if job["build-platform"].startswith("win"): if dep_job.kind.endswith("signing"): continue if job["build-platform"].startswith("macosx"): if dep_job.kind.endswith("repack"): continue dependencies = {dep_job.attributes.get("kind"): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith("macosx") and dependency.endswith( "signing"): signing_task = dependency elif build_platform.startswith("win") and dependency.endswith( "repack"): signing_task = dependency attributes["repackage_type"] = "repackage" repack_id = job["extra"]["repack_id"] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: job["package-formats"].append("installer-stub") repackage_config = [] for format in job.get("package-formats"): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "executable_extension": executable_extension(build_platform), } command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } repackage_config.append(command) run = job.get("mozharness", {}) run.update({ "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, }) worker = { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, "env": { "REPACK_ID": repack_id, }, # Don't add generic artifact directory. "skip-artifacts": True, } worker_type = "b-linux" worker["docker-image"] = {"in-tree": "debian10-amd64-build"} worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job["extra"]["repack_id"], build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "attributes": attributes, "scopes": ["queue:get-artifact:releng/partner/*"], "run-on-projects": dep_job.attributes.get("run_on_projects"), "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer, ), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ]) yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] repack_id = dep_job.task["extra"]["repack_id"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501 repack_id=repack_id, build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) if "linux" in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith("release-partner-repack"): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage-signing" signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) scopes = [signing_cert_scope] if "win" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"], }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.stub-installer.exe".format(repack_id), ), ], "formats": ["autograph_authenticode", "autograph_gpg"], }) elif "mac" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"], }] elif "linux" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"], }] task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "extra": { "repack_id": repack_id, }, } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def add_command_arguments(config, tasks): enabled_partners = config.params.get("release_partners") dependencies = {} fetches = defaultdict(set) attributions = [] release_artifacts = [] attribution_config = get_partner_config_by_kind(config, config.kind) for partner_config in attribution_config.get("configs", []): # we might only be interested in a subset of all partners, eg for a respin if enabled_partners and partner_config[ "campaign"] not in enabled_partners: continue attribution_code = generate_attribution_code( attribution_config["defaults"], partner_config) for platform in partner_config["platforms"]: stage_platform = platform.replace("-shippable", "") for locale in partner_config["locales"]: # find the upstream, throw away locales we don't have, somehow. Skip ? if locale == "en-US": upstream_label = "repackage-signing-{platform}/opt".format( platform=platform) upstream_artifact = "target.installer.exe" else: upstream_label = "repackage-signing-l10n-{locale}-{platform}/opt".format( locale=locale, platform=platform) upstream_artifact = "{locale}/target.installer.exe".format( locale=locale) if upstream_label not in config.kind_dependencies_tasks: raise Exception( "Can't find upstream task for {} {}".format( platform, locale)) upstream = config.kind_dependencies_tasks[upstream_label] # set the dependencies to just what we need rather than all of l10n dependencies.update({upstream.label: upstream.label}) fetches[upstream_label].add( (upstream_artifact, stage_platform, locale)) artifact_part = "{platform}/{locale}/target.installer.exe".format( platform=stage_platform, locale=locale) artifact = "releng/partner/{partner}/{sub_partner}/{artifact_part}".format( partner=partner_config["campaign"], sub_partner=partner_config["content"], artifact_part=artifact_part, ) # config for script # TODO - generalise input & output ?? # add releng/partner prefix via get_artifact_prefix..() attributions.append({ "input": "/builds/worker/fetches/{}".format(artifact_part), "output": "/builds/worker/artifacts/{}".format(artifact), "attribution": attribution_code, }) release_artifacts.append(artifact) # bail-out early if we don't have any attributions to do if not attributions: return for task in tasks: worker = task.get("worker", {}) worker["chain-of-trust"] = True task.setdefault("dependencies", {}).update(dependencies) task.setdefault("fetches", {}) for upstream_label, upstream_artifacts in fetches.items(): task["fetches"][upstream_label] = [{ "artifact": upstream_artifact, "dest": "{platform}/{locale}".format(platform=platform, locale=locale), "extract": False, "verify-hash": True, } for upstream_artifact, platform, locale in upstream_artifacts] worker.setdefault("env", {})["ATTRIBUTION_CONFIG"] = six.ensure_text( json.dumps(attributions, sort_keys=True)) worker["artifacts"] = [{ "name": "releng/partner", "path": "/builds/worker/artifacts/releng/partner", "type": "directory", }] task["release-artifacts"] = release_artifacts task["label"] = config.kind yield task
def split_public_and_private(config, jobs): # we need to separate private vs public destinations because beetmover supports one # in a single task. Only use a single task for each type though. partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: upstream_artifacts = job["primary-dependency"].release_artifacts attribution_task_ref = "<{}>".format(job["primary-dependency"].label) prefix = get_artifact_prefix(job["primary-dependency"]) artifacts = defaultdict(list) for artifact in upstream_artifacts: partner, sub_partner, platform, locale, _ = artifact.replace( prefix + "/", "").split("/", 4) destination = "private" this_config = [ p for p in partner_config["configs"] if (p["campaign"] == partner and p["content"] == sub_partner) ] if this_config[0].get("upload_to_candidates"): destination = "public" artifacts[destination].append( (artifact, partner, sub_partner, platform, locale)) action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") public_bucket_scope = get_beetmover_bucket_scope(config) partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) repl_dict = { "build_number": config.params["build_number"], "release_partner_build_number": config.params["release_partner_build_number"], "version": config.params["version"], "partner": "{partner}", # we'll replace these later, per artifact "subpartner": "{subpartner}", "platform": "{platform}", "locale": "{locale}", } for destination, destination_artifacts in artifacts.items(): this_job = deepcopy(job) if destination == "public": this_job["scopes"] = [public_bucket_scope, action_scope] this_job["partner_public"] = True else: this_job["scopes"] = [partner_bucket_scope, action_scope] this_job["partner_public"] = False partner_path_key = "partner-{destination}-path".format( destination=destination) partner_path = this_job[partner_path_key].format(**repl_dict) this_job.setdefault( "worker", {})["upstream-artifacts"] = generate_upstream_artifacts( attribution_task_ref, destination_artifacts, partner_path) yield this_job