def make_task_description(config, jobs): for job in jobs: if len(job["dependencies"]) != 1: raise Exception("Exactly 1 dependency is required") job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( job["dependencies"]) resolve_keyed_by(job, "worker.channel", item_name=job["name"], **{"release-type": config.params["release_type"]}) resolve_keyed_by(job, "worker-type", item_name=job["name"], **{"release-level": config.params.release_level()}) if config.params.release_level() == "production": job.setdefault("scopes", []).append( add_scope_prefix( config, "snapcraft:firefox:{}".format( job["worker"]["channel"].split("/")[0]), )) yield job
def make_task_description(config, jobs): for job in jobs: if len(job['dependencies']) != 1: raise Exception('Exactly 1 dependency is required') job['worker']['upstream-artifacts'] = generate_upstream_artifacts( job['dependencies']) resolve_keyed_by(job, 'worker.channel', item_name=job['name'], **{'release-type': config.params['release_type']}) resolve_keyed_by(job, 'worker-type', item_name=job['name'], **{'release-level': config.params.release_level()}) if config.params.release_level() == 'production': job.setdefault('scopes', []).append( add_scope_prefix( config, "snapcraft:firefox:{}".format( job['worker']['channel'].split('/')[0]), )) yield job
def make_beetmover_push_to_release_description(config, jobs): for job in jobs: treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "Rel(BM-C)") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") treeherder.setdefault("platform", job["treeherder-platform"]) label = job["name"] description = "Beetmover push to release for '{product}'".format( product=job["product"]) bucket_scope = get_beetmover_bucket_scope(config) action_scope = add_scope_prefix(config, "beetmover:action:push-to-releases") task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "product": job["product"], "dependencies": job["dependencies"], "attributes": job.get("attributes", {}), "run-on-projects": job.get("run-on-projects"), "treeherder": treeherder, "shipping-phase": job.get("shipping-phase", "push"), "shipping-product": job.get("shipping-product"), "routes": job.get("routes", []), "extra": job.get("extra", {}), } yield task
def make_beetmover_push_to_release_description(config, jobs): for job in jobs: treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Rel(BM-C)') treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') treeherder.setdefault('platform', job['treeherder-platform']) label = job['name'] description = ( "Beetmover push to release for '{product}'".format( product=job['product'] ) ) bucket_scope = get_beetmover_bucket_scope(config) action_scope = add_scope_prefix(config, 'beetmover:action:push-to-releases') task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'product': job['product'], 'dependencies': job['dependencies'], 'attributes': job.get('attributes', {}), 'run-on-projects': job.get('run-on-projects'), 'treeherder': treeherder, 'shipping-phase': job.get('shipping-phase', 'push'), 'shipping-product': job.get('shipping-product'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), } yield task
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'css(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = "Signing of release-source checksums file" dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target-source.checksums", ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg'), ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False): action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner') task = deepcopy(job) task['scopes'] = [bucket_scope, action_scope] task['worker-type'] = get_worker_type_for_scope(config, bucket_scope) task['partner_public'] = partner_public if partner_public: task['label'] = "{}-public".format(task['label']) return task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False): action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") task = deepcopy(job) task["scopes"] = [bucket_scope, action_scope] task["worker-type"] = "beetmover" task["partner_public"] = partner_public if partner_public: task["label"] = "{}-public".format(task["label"]) return task
def make_release_generate_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'SGenChcks') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') job_template = "{}-{}".format(dep_job.label, "signing") label = job.get("label", job_template) description = "Signing of the overall release-related checksums" dependencies = { "build": dep_job.label } upstream_artifacts = [{ "taskId": {"task-reference": "<build>"}, "taskType": "build", "paths": [ get_artifact_path(dep_job, "SHA256SUMS"), get_artifact_path(dep_job, "SHA512SUMS"), ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg'), ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def split_public_and_private(config, jobs): public_bucket_scope = get_beetmover_bucket_scope(config) partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) partner, subpartner, _ = job["extra"]["repack_id"].split("/") if partner_config[partner][subpartner].get("upload_to_candidates"): # public yield populate_scopes_and_worker_type( config, job, public_bucket_scope, partner_public=True ) else: # private yield populate_scopes_and_worker_type( config, job, partner_bucket_scope, partner_public=False )
def split_public_and_private(config, jobs): public_bucket_scope = get_beetmover_bucket_scope(config) partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: partner_bucket_scope = add_scope_prefix(config, job['partner-bucket-scope']) partner, subpartner, _ = job['extra']['repack_id'].split('/') # public if partner_config[partner][subpartner].get('upload_to_candidates'): yield populate_scopes_and_worker_type(config, job, public_bucket_scope, partner_public=True) # private yield populate_scopes_and_worker_type(config, job, partner_bucket_scope, partner_public=False)
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg') ] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode", "gpg"] }] scopes.append( add_scope_prefix(config, "signing:format:sha2signcode")) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["gpg"] }] task = { 'label': label, 'description': description, # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] locale = dep_job.attributes.get('locale') treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', join_symbol(job.get('treeherder-group', 'ms'), locale or 'N')) label = job.get('label', "{}-{}".format(config.kind, dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['required_signoffs'] = sorted_unique_list( attributes.get('required_signoffs', []), job.pop('required_signoffs')) attributes['shipping_phase'] = job['shipping-phase'] if locale: attributes['locale'] = locale balrog_platform = get_balrog_platform_name(dep_th_platform) if config.kind == 'partials-signing': upstream_artifacts = generate_partials_artifacts( dep_job, config.params['release_history'], balrog_platform, locale) else: upstream_artifacts = generate_complete_artifacts( dep_job, config.kind) build_platform = dep_job.attributes.get('build_platform') is_nightly = job.get( 'nightly', # First check current job dep_job.attributes.get( 'nightly', # Then dep job for 'nightly' dep_job.attributes.get( 'shippable'))) # lastly dep job for 'shippable' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] + list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': label, 'description': "{} {}".format(dep_job.task["metadata"]["description"], job['description-suffix']), 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': job.get('run-on-projects', dep_job.attributes.get('run_on_projects')), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) treeherder = job.get('treeherder', {}) is_nightly = dep_job.attributes.get('nightly', False) treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') build_platform = dep_job.attributes.get('build_platform') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( dep_job.attributes.get('build_platform'), is_nightly, config ) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': 3600}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'treeherder': treeherder, 'routes': job.get('routes', []), } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly', dep_job.attributes.get('shippable')) # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope] if 'win' in build_platform: upstream_artifacts = [{ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode", "autograph_gpg"] }] scopes.append(add_scope_prefix(config, "signing:format:sha2signcode")) partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: upstream_artifacts.append({ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.stub-installer.exe".format( repack_id)), ], "formats": ["sha2signcode", "autograph_gpg"] }) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": {"task-reference": "<repack>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"] }] task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def split_public_and_private(config, jobs): # we need to separate private vs public destinations because beetmover supports one # in a single task. Only use a single task for each type though. partner_config = get_partner_config_by_kind(config, config.kind) for job in jobs: upstream_artifacts = job["primary-dependency"].release_artifacts attribution_task_ref = "<{}>".format(job["primary-dependency"].label) prefix = get_artifact_prefix(job["primary-dependency"]) artifacts = defaultdict(list) for artifact in upstream_artifacts: partner, sub_partner, platform, locale, _ = artifact.replace( prefix + "/", "").split("/", 4) destination = "private" this_config = [ p for p in partner_config["configs"] if (p["campaign"] == partner and p["content"] == sub_partner) ] if this_config[0].get("upload_to_candidates"): destination = "public" artifacts[destination].append( (artifact, partner, sub_partner, platform, locale)) action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") public_bucket_scope = get_beetmover_bucket_scope(config) partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"]) repl_dict = { "build_number": config.params["build_number"], "release_partner_build_number": config.params["release_partner_build_number"], "version": config.params["version"], "partner": "{partner}", # we'll replace these later, per artifact "subpartner": "{subpartner}", "platform": "{platform}", "locale": "{locale}", } for destination, destination_artifacts in artifacts.items(): this_job = deepcopy(job) if destination == "public": this_job["scopes"] = [public_bucket_scope, action_scope] this_job["partner_public"] = True else: this_job["scopes"] = [partner_bucket_scope, action_scope] this_job["partner_public"] = False partner_path_key = "partner-{destination}-path".format( destination=destination) partner_path = this_job[partner_path_key].format(**repl_dict) this_job.setdefault( "worker", {})["upstream-artifacts"] = generate_upstream_artifacts( attribution_task_ref, destination_artifacts, partner_path) yield this_job
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of Checksums file for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'cs({})'.format( dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target.checksums", ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg'), ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get('locale', dep_job.attributes.get('locale')) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'rs(N)') else: treeherder.setdefault('symbol', 'rs(B)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') if locale: treeherder['symbol'] = 'rs({})'.format(locale) if config.kind == 'repackage-signing-msi': treeherder['symbol'] = 'MSIs({})'.format(locale or 'N') label = job['label'] dep_kind = dep_job.kind if 'l10n' in dep_kind: dep_kind = 'repackage' dependencies = {dep_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] upstream_artifacts = [] for artifact in dep_job.release_artifacts: basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: upstream_artifacts.append({ "taskId": { "task-reference": "<{}>".format(dep_kind) }, "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) scopes += list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Gd(s)') treeherder.setdefault( 'platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}/{}'".format( attributes.get('build_platform'), attributes.get('build_type'), ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] scopes += list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ("Signing of OpenH264 Binaries for '" "{build_platform}/{build_type}'".format( build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) # we have a genuine repackage job as our parent dependencies = {"openh264": dep_job.label} my_attributes = copy_attributes_from_dependent_job(dep_job) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: # job['primary-dependency'].task['payload']['command'] scopes.append( add_scope_prefix(config, "signing:format:sha2signcode")) formats = ['sha2signcode'] else: formats = ['autograph_gpg'] rev = attributes['openh264_rev'] upstream_artifacts = [{ "taskId": { "task-reference": "<openh264>" }, "taskType": "build", "paths": [ "private/openh264/openh264-{}-{}.zip".format( build_platform, rev), ], "formats": formats }] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': my_attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'rs(N)') else: treeherder.setdefault('symbol', 'rs(B)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('kind', 'build') label = job['label'] dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({k: v for k, v in signing_dependencies.items() if k != 'docker-image'}) locale_str = "" if dep_job.attributes.get('locale'): treeherder['symbol'] = 'rs({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') locale_str = "{}/".format(dep_job.attributes.get('locale')) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope, add_scope_prefix(config, 'signing:format:mar_sha384')] upstream_artifacts = [{ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}target.complete.mar".format(locale_str)), ], "formats": ["mar_sha384"] }] if 'win' in build_platform: upstream_artifacts.append({ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}target.installer.exe".format(locale_str)), ], "formats": ["sha2signcode"] }) scopes.append(add_scope_prefix(config, "signing:format:sha2signcode")) use_stub = attributes.get('stub-installer') if use_stub: upstream_artifacts.append({ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}target.stub-installer.exe".format(locale_str) ), ], "formats": ["sha2signcodestub"] }) scopes.append(add_scope_prefix(config, "signing:format:sha2signcodestub")) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format))) is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type'))) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config, job_release_type=dep_job.attributes.get('release-type')) worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope) mac_behavior = None task = { 'label': label, 'description': description, 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600) }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': { job['depname']: dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: assert worker_type_alias.startswith("linux-"), \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" ) worker_type_alias = worker_type_alias.replace("linux-", "mac-") mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'release-type': config.params['release_type'], 'platform': build_platform, }, ) task['worker']['mac-behavior'] = mac_behavior if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') )) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) worker_type = get_worker_type_for_scope(config, signing_cert_scope) task = { 'label': label, 'description': description, 'worker-type': worker_type, 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600)}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes label = dep_job.label.replace("repackage-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' not in build_platform: raise Exception( "Repackage signing is not supported for non-Windows partner repacks." ) upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode"] }] scopes.append(add_scope_prefix(config, "signing:format:sha2signcode")) task = { 'label': label, 'description': description, # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } yield task