def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'ps(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') label = job.get('label', "partials-signing-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = 'ps({})'.format(locale) balrog_platform = get_balrog_platform_name(dep_th_platform) upstream_artifacts = generate_upstream_artifacts( dep_job, config.params['release_history'], balrog_platform, locale) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope, 'project:releng:signing:format:autograph_hash_only_mar384'] if any("mar" in upstream_details["formats"] for upstream_details in upstream_artifacts): scopes.append('project:releng:signing:format:mar') task = { 'label': label, 'description': "{} Partials".format( dep_job.task["metadata"]["description"]), 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'ps(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') label = job.get('label', "partials-signing-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = 'ps({})'.format(locale) balrog_platform = get_balrog_platform_name(dep_th_platform) upstream_artifacts = generate_upstream_artifacts( config.params['release_history'], balrog_platform, locale) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384'] task = { 'label': label, 'description': "{} Partials".format( dep_job.task["metadata"]["description"]), 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Gd(s)') treeherder.setdefault( 'platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}/{}'".format( attributes.get('build_platform'), attributes.get('build_type'), ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ("Signing of OpenH264 Binaries for '" "{build_platform}/{build_type}'".format( build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) # we have a genuine repackage job as our parent dependencies = {"openh264": dep_job.label} my_attributes = copy_attributes_from_dependent_job(dep_job) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: # job['primary-dependency'].task['payload']['command'] formats = ['autograph_authenticode'] else: formats = ['autograph_gpg'] rev = attributes['openh264_rev'] upstream_artifacts = [{ "taskId": { "task-reference": "<openh264>" }, "taskType": "build", "paths": [ "private/openh264/openh264-{}-{}.zip".format( build_platform, rev), ], "formats": formats }] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) task = { 'label': job['label'], 'description': description, 'worker-type': 'linux-signing', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': my_attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type'))) attributes = job['attributes'] if job.get('attributes') else \ copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope) mac_behavior = None task = { 'label': label, 'description': description, 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600) }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': { job['depname']: dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: assert worker_type_alias.startswith("linux-"), \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" ) worker_type_alias = worker_type_alias.replace("linux-", "mac-") mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'release-type': config.params['release_type'], 'platform': build_platform, }, ) task['worker']['mac-behavior'] = mac_behavior if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes label = dep_job.label.replace("repackage-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' not in build_platform: raise Exception( "Repackage signing is not supported for non-Windows partner repacks." ) upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode"] }] scopes.append(add_scope_prefix(config, "signing:format:sha2signcode")) task = { 'label': label, 'description': description, # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'rs(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' locale_str = "" if dep_job.attributes.get('locale'): treeherder['symbol'] = 'rs({})'.format( dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') locale_str = "{}/".format(dep_job.attributes.get('locale')) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [ signing_cert_scope, 'project:releng:signing:format:mar_sha384' ] upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.complete.mar".format(locale_str), ], "formats": ["mar_sha384"] }] if 'win' in build_platform: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.installer.exe".format(locale_str), ], "formats": ["sha2signcode"] }) scopes.append("project:releng:signing:format:sha2signcode") # Stub installer is only generated on win32 if '32' in build_platform: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.stub-installer.exe".format( locale_str), ], "formats": ["sha2signcodestub"] }) scopes.append("project:releng:signing:format:sha2signcodestub") task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) dep_treeherder = dep_job.task.get('extra', {}).get('treeherder', {}) treeherder.setdefault('symbol', '{}(gd-s)'.format(dep_treeherder['groupSymbol'])) treeherder.setdefault('platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault('tier', dep_treeherder.get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}'".format( attributes.get('build_platform'), ) build_platform = dep_job.attributes.get('build_platform') is_shippable = dep_job.attributes.get('shippable') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config ) upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind, build_platform) scopes = [signing_cert_scope] platform = build_platform.split("-")[0] task = { 'label': job['label'], 'description': description, 'worker-type': 'linux-signing', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'treeherder': treeherder, 'run-on-projects': ['mozilla-central'], 'index': {"product": "geckodriver", "job-name": platform}, } if build_platform.startswith('macosx'): worker_type = task['worker-type'] worker_type_alias_map = { 'linux-depsigning': 'mac-depsigning', 'linux-signing': 'mac-signing', } assert worker_type in worker_type_alias_map, \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type) ) worker_type = worker_type_alias_map[worker_type] task['worker-type'] = worker_type_alias_map[task['worker-type']] task['worker']['mac-behavior'] = 'mac_geckodriver' yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] locale = dep_job.attributes.get('locale') treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( 'symbol', join_symbol(job.get('treeherder-group', 'ms'), locale or 'N')) label = job.get('label', "{}-{}".format(config.kind, dep_job.label)) dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['required_signoffs'] = sorted_unique_list( attributes.get('required_signoffs', []), job.pop('required_signoffs')) attributes['shipping_phase'] = job['shipping-phase'] if locale: attributes['locale'] = locale build_platform = attributes.get('build_platform') if config.kind == 'partials-signing': upstream_artifacts = generate_partials_artifacts( dep_job, config.params['release_history'], build_platform, locale) else: upstream_artifacts = generate_complete_artifacts( dep_job, config.kind) is_nightly = job.get( 'nightly', # First check current job dep_job.attributes.get( 'nightly', # Then dep job for 'nightly' dep_job.attributes.get( 'shippable'))) # lastly dep job for 'shippable' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] task = { 'label': label, 'description': "{} {}".format(dep_job.task["metadata"]["description"], job['description-suffix']), 'worker-type': 'linux-signing', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': job.get('run-on-projects', dep_job.attributes.get('run_on_projects')), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job["upstream-artifacts"]: for f in artifacts["formats"]: formats.add(f) # Add each format only once is_shippable = dep_job.attributes.get("shippable", False) build_platform = dep_job.attributes.get("build_platform") treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) build_type = dep_job.attributes.get("build_type") treeherder.setdefault( "platform", _generate_treeherder_platform(dep_th_platform, build_platform, build_type), ) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1), ) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")), ) treeherder.setdefault("kind", "build") label = job["label"] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), )) attributes = (job["attributes"] if job.get("attributes") else copy_attributes_from_dependent_job(dep_job)) attributes["signed"] = True if dep_job.attributes.get("chunk_locales"): # Used for l10n attribute passthrough attributes["chunk_locales"] = dep_job.attributes.get( "chunk_locales") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning" mac_behavior = None task = { "label": label, "description": description, "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": job["upstream-artifacts"], "max-run-time": job.get("max-run-time", 3600), }, "scopes": [signing_cert_scope] + signing_format_scopes, "dependencies": _generate_dependencies(job), "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "routes": job.get("routes", []), "shipping-product": job.get("shipping-product"), "shipping-phase": job.get("shipping-phase"), } if dep_job.kind in task["dependencies"]: task["if-dependencies"] = [dep_job.kind] if "macosx" in build_platform: shippable = "false" if "shippable" in attributes and attributes["shippable"]: shippable = "true" mac_behavior = evaluate_keyed_by( config.graph_config["mac-notarization"]["mac-behavior"], "mac behavior", { "project": config.params["project"], "shippable": shippable, }, ) if mac_behavior == "mac_notarize": if "part-1" in config.kind: mac_behavior = "mac_notarize_part_1" elif config.kind.endswith("signing"): mac_behavior = "mac_notarize_part_3" else: raise Exception("Unknown kind {} for mac_behavior!".format( config.kind)) else: if "part-1" in config.kind: continue task["worker"]["mac-behavior"] = mac_behavior worker_type_alias_map = { "linux-depsigning": "mac-depsigning", "linux-signing": "mac-signing", } assert worker_type_alias in worker_type_alias_map, ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias)) worker_type_alias = worker_type_alias_map[worker_type_alias] if job.get("entitlements-url"): task["worker"]["entitlements-url"] = job["entitlements-url"] task["worker-type"] = worker_type_alias if treeherder: task["treeherder"] = treeherder if job.get("extra"): task["extra"] = job["extra"] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage-signing" treeherder = job.get("treeherder", {}) dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {}) treeherder.setdefault("symbol", "{}(gd-s)".format(dep_treeherder["groupSymbol"])) treeherder.setdefault( "platform", dep_job.task.get("extra", {}).get("treeherder-platform")) treeherder.setdefault("tier", dep_treeherder.get("tier", 1)) treeherder.setdefault("kind", "build") dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != "docker-image" }) description = "Signing Geckodriver for build '{}'".format( attributes.get("build_platform"), ) build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind, build_platform) scopes = [signing_cert_scope] platform = build_platform.split("-")[0] task = { "label": job["label"], "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "treeherder": treeherder, "run-on-projects": ["mozilla-central"], "index": { "product": "geckodriver", "job-name": platform }, } if build_platform.startswith("macosx"): worker_type = task["worker-type"] worker_type_alias_map = { "linux-depsigning": "mac-depsigning", "linux-signing": "mac-signing", } assert worker_type in worker_type_alias_map, ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type)) worker_type = worker_type_alias_map[worker_type] task["worker-type"] = worker_type_alias_map[task["worker-type"]] task["worker"]["mac-behavior"] = "mac_geckodriver" yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] repack_id = dep_job.task["extra"]["repack_id"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501 repack_id=repack_id, build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) if "linux" in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith("release-partner-repack"): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage-signing" signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) scopes = [signing_cert_scope] if "win" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"], }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.stub-installer.exe".format(repack_id), ), ], "formats": ["autograph_authenticode", "autograph_gpg"], }) elif "mac" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"], }] elif "linux" in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"], }] task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "extra": { "repack_id": repack_id, }, } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes build_platform = dep_job.attributes.get("build_platform") is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ("Signing of OpenH264 Binaries for '" "{build_platform}/{build_type}'".format( build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) # we have a genuine repackage job as our parent dependencies = {"openh264": dep_job.label} my_attributes = copy_attributes_from_dependent_job(dep_job) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if "win" in build_platform: # job['primary-dependency'].task['payload']['command'] formats = ["autograph_authenticode"] else: formats = ["autograph_gpg"] rev = attributes["openh264_rev"] upstream_artifacts = [{ "taskId": { "task-reference": "<openh264>" }, "taskType": "build", "paths": [ "private/openh264/openh264-{}-{}.zip".format( build_platform, rev), ], "formats": formats, }] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")), ) task = { "label": job["label"], "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": my_attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get("locale", dep_job.attributes.get("locale")) attributes["repackage_type"] = "repackage-signing" treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "rs(B)") dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") treeherder.setdefault("platform", dep_th_platform) treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)) treeherder.setdefault("kind", "build") if locale: treeherder["symbol"] = "rs({})".format(locale) if config.kind == "repackage-signing-msi": treeherder["symbol"] = "MSIs({})".format(locale or "N") label = job["label"] dep_kind = dep_job.kind if "l10n" in dep_kind: dep_kind = "repackage" dependencies = {dep_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != "docker-image" }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) build_platform = dep_job.attributes.get("build_platform") is_shippable = dep_job.attributes.get("shippable") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) scopes = [signing_cert_scope] upstream_artifacts = [] for artifact in sorted(dep_job.release_artifacts): basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: upstream_artifacts.append({ "taskId": { "task-reference": "<{}>".format(dep_kind) }, "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) task = { "label": label, "description": description, "worker-type": "linux-signing", "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "scopes": scopes, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "treeherder": treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Gd(s)') treeherder.setdefault( 'platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}/{}'".format( attributes.get('build_platform'), attributes.get('build_type'), ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } if build_platform.startswith('macosx'): assert task['worker-type'].startswith("linux-"), \ ( "Make sure to adjust the below worker-type logic for " "mac if you change the signing workerType aliases!" ) task['worker-type'] = task['worker-type'].replace("linux-", "mac-") task['worker']['mac-behavior'] = 'mac_geckodriver' yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( "project:releng:signing:format:{}".format(format)) treeherder = job.get('treeherder', {}) is_nightly = dep_job.attributes.get('nightly', False) treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') build_platform = dep_job.attributes.get('build_platform') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job.get('label', "{}-signing".format(dep_job.label)) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( dep_job.attributes.get('build_platform'), is_nightly, config) task = { 'label': label, 'description': "{} Signing".format(dep_job.task["metadata"]["description"]), 'worker-type': _generate_worker_type(signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': 3600 }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': { job['depname']: dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] locale = dep_job.attributes.get('locale') treeherder = job.get('treeherder', {}) treeherder['symbol'] = join_symbol(job.get('treeherder-group', 'ms'), locale or 'N') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') label = job.get('label', "{}-{}".format(config.kind, dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['required_signoffs'] = sorted_unique_list( attributes.get('required_signoffs', []), job.pop('required_signoffs')) attributes['shipping_phase'] = job['shipping-phase'] if locale: attributes['locale'] = locale balrog_platform = get_balrog_platform_name(dep_th_platform) if config.kind == 'partials-signing': upstream_artifacts = generate_partials_artifacts( dep_job, config.params['release_history'], balrog_platform, locale) else: upstream_artifacts = generate_complete_artifacts(dep_job) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] + list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': label, 'description': "{} {}".format(dep_job.task["metadata"]["description"], job['description-suffix']), 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get('locale', dep_job.attributes.get('locale')) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'rs(N)') else: treeherder.setdefault('symbol', 'rs(B)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') if locale: treeherder['symbol'] = 'rs({})'.format(locale) if config.kind == 'repackage-signing-msi': treeherder['symbol'] = 'MSIs({})'.format(locale or 'N') label = job['label'] dep_kind = dep_job.kind if 'l10n' in dep_kind: dep_kind = 'repackage' dependencies = {dep_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] upstream_artifacts = [] for artifact in dep_job.release_artifacts: basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: upstream_artifacts.append({ "taskId": { "task-reference": "<{}>".format(dep_kind) }, "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) scopes += list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) treeherder = job.get('treeherder', {}) is_nightly = dep_job.attributes.get('nightly', False) treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') build_platform = dep_job.attributes.get('build_platform') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( dep_job.attributes.get('build_platform'), is_nightly, config ) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': 3600}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'treeherder': treeherder, 'routes': job.get('routes', []), } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.stub-installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"] }] task = { 'label': label, 'description': description, 'worker-type': 'linux-signing', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] locale = dep_job.attributes.get("locale") treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( "symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N") ) label = job.get("label", "{}-{}".format(config.kind, dep_job.label)) dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes["required_signoffs"] = sorted_unique_list( attributes.get("required_signoffs", []), job.pop("required_signoffs") ) attributes["shipping_phase"] = job["shipping-phase"] if locale: attributes["locale"] = locale build_platform = attributes.get("build_platform") if config.kind == "partials-signing": upstream_artifacts = generate_partials_artifacts( dep_job, config.params["release_history"], build_platform, locale ) else: upstream_artifacts = generate_complete_artifacts(dep_job, config.kind) is_shippable = job.get( "shippable", dep_job.attributes.get("shippable") # First check current job ) # Then dep job for 'shippable' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config ) scopes = [signing_cert_scope] task = { "label": label, "description": "{} {}".format( dep_job.description, job["description-suffix"] ), "worker-type": job.get("worker-type", "linux-signing"), "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": upstream_artifacts, "max-run-time": 3600, }, "dependencies": dependencies, "attributes": attributes, "scopes": scopes, "run-on-projects": job.get( "run-on-projects", dep_job.attributes.get("run_on_projects") ), "treeherder": treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once is_shippable = dep_job.attributes.get('shippable', False) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type'))) attributes = job['attributes'] if job.get('attributes') else \ copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) worker_type_alias = 'linux-signing' if is_shippable else 'linux-depsigning' mac_behavior = None task = { 'label': label, 'description': description, 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600) }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': _generate_dependencies(job), 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: shippable = "false" if "shippable" in attributes and attributes["shippable"]: shippable = "true" mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'project': config.params['project'], 'shippable': shippable, }, ) if mac_behavior == 'mac_notarize': if 'part-1' in config.kind: mac_behavior = 'mac_notarize_part_1' elif config.kind.endswith('signing'): mac_behavior = 'mac_notarize_part_3' else: raise Exception("Unknown kind {} for mac_behavior!".format( config.kind)) else: if 'part-1' in config.kind: continue task['worker']['mac-behavior'] = mac_behavior worker_type_alias_map = { 'linux-depsigning': 'mac-depsigning', 'linux-signing': 'mac-signing', } assert worker_type_alias in worker_type_alias_map, \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias) ) worker_type_alias = worker_type_alias_map[worker_type_alias] if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg') ] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode", "gpg"] }] scopes.append( add_scope_prefix(config, "signing:format:sha2signcode")) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["gpg"] }] task = { 'label': label, 'description': description, # 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker-type': 'scriptworker-prov-v1/signing-linux-v1', 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') )) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) worker_type = get_worker_type_for_scope(config, signing_cert_scope) task = { 'label': label, 'description': description, 'worker-type': worker_type, 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600)}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'tc-rs(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job.get('label', "repackage-signing-{}".format(dep_job.label)) dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' locale_str = "" if dep_job.attributes.get('locale'): treeherder['symbol'] = 'tc-rs({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') locale_str = "{}/".format(dep_job.attributes.get('locale')) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384'] upstream_artifacts = [{ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ "public/build/{}target.complete.mar".format(locale_str), ], "formats": ["mar_sha384"] }] if 'win' in build_platform: upstream_artifacts.append({ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ "public/build/{}target.installer.exe".format(locale_str), ], "formats": ["sha2signcode"] }) scopes.append("project:releng:signing:format:sha2signcode") # Stub installer is only generated on win32 if '32' in build_platform: upstream_artifacts.append({ "taskId": {"task-reference": "<repackage>"}, "taskType": "repackage", "paths": [ "public/build/{}target.stub-installer.exe".format(locale_str), ], "formats": ["sha2signcodestub"] }) scopes.append("project:releng:signing:format:sha2signcodestub") task = { 'label': label, 'description': "Repackage signing {} ".format( dep_job.task["metadata"]["description"]), 'worker-type': "scriptworker-prov-v1/signing-linux-v1", 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } funsize_platforms = [ 'linux-nightly', 'linux64-nightly', 'macosx64-nightly', 'win32-nightly', 'win64-nightly' ] if build_platform in funsize_platforms and is_nightly: route_template = "project.releng.funsize.level-{level}.{project}" task['routes'] = [ route_template.format(project=config.params['project'], level=config.params['level']) ] yield task