def add_notifications(config, jobs): xpi_name = config.params.get("xpi_name") xpi_revision = config.params.get("xpi_revision") shipping_phase = config.params.get("shipping_phase") additional_shipit_emails = config.params.get("additional_shipit_emails", []) if not all([xpi_name, xpi_revision, shipping_phase]): return manifest = get_manifest() for job in jobs: if "primary-dependency" in job: dep = job.pop("primary-dependency") if dep.task.get("extra", {}).get("xpi-name") != xpi_name: continue attributes = dep.attributes.copy() if job.get("attributes"): attributes.update(job["attributes"]) job["attributes"] = attributes job.setdefault("dependencies", {}).update({"signing": dep.label}) if job.get("attributes", {}).get("shipping-phase") != shipping_phase: continue job["label"] = f"{config.kind}-{shipping_phase}" xpi_config = manifest[xpi_name] xpi_type = xpi_config["addon-type"] emails = evaluate_keyed_by( config.graph_config["release-promotion"]["notifications"] [xpi_type], "email", dict(phase=shipping_phase, level=config.params["level"]), ) if not emails: continue emails = (emails + additional_shipit_emails + xpi_config.get("additional-emails", [])) notifications = evaluate_keyed_by(job.pop("notifications"), "notification config", dict(phase=shipping_phase)) format_kwargs = dict(config=config.__dict__) subject = notifications["subject"].format(**format_kwargs) message = notifications["message"].format(**format_kwargs) # We only send mail on success to avoid messages like 'blah is in the # candidates dir' when cancelling graphs, dummy job failure, etc job.setdefault("routes", []).extend( [f"notify.email.{email}.on-completed" for email in emails]) job.setdefault("extra", {}).update({"notify": { "email": { "subject": subject } }}) if message: job["extra"]["notify"]["email"]["content"] = message yield job
def add_notifications(config, jobs): xpi_name = config.params.get("xpi_name") xpi_revision = config.params.get("xpi_revision") shipping_phase = config.params.get("shipping_phase") if not all([xpi_name, xpi_revision, shipping_phase]): return manifest = get_manifest() for job in jobs: if "primary-dependency" in job: dep = job.pop("primary-dependency") if dep.task.get("extra", {}).get("xpi-name") != xpi_name: continue attributes = dep.attributes.copy() if job.get("attributes"): attributes.update(job["attributes"]) job["attributes"] = attributes job.setdefault("dependencies", {}).update({"signing": dep.label}) if job.get("attributes", {}).get("shipping-phase") != shipping_phase: continue job['label'] = '{}-{}'.format(config.kind, shipping_phase) xpi_config = manifest[xpi_name] xpi_type = xpi_config['addon-type'] emails = evaluate_keyed_by( config.graph_config['release-promotion']['notifications'] [xpi_type], 'email', dict( phase=shipping_phase, )) + xpi_config.get( "additional-emails", []) notifications = evaluate_keyed_by(job.pop('notifications'), 'notification config', dict(phase=shipping_phase, )) format_kwargs = dict(config=config.__dict__, ) subject = notifications['subject'].format(**format_kwargs) message = notifications['message'].format(**format_kwargs) # We only send mail on success to avoid messages like 'blah is in the # candidates dir' when cancelling graphs, dummy job failure, etc job.setdefault('routes', []).extend( ['notify.email.{}.on-completed'.format(email) for email in emails]) job.setdefault('extra', {}).update( {'notify': { 'email': { 'subject': subject, } }}) if message: job['extra']['notify']['email']['content'] = message yield job
def build_signing_task(config, tasks): for task in tasks: dep = task["primary-dependency"] task["dependencies"] = {"build": dep.label} if not dep.task["payload"]["env"]["ARTIFACT_PREFIX"].startswith( "public"): scopes = task.setdefault('scopes', []) scopes.append("queue:get-artifact:{}/*".format( dep.task["payload"]["env"]["ARTIFACT_PREFIX"].rstrip('/'))) paths = dep.attributes["xpis"].values() format = evaluate_keyed_by( config.graph_config["scriptworker"]["signing-format"], "signing-format", { "xpi-type": task["attributes"]["addon-type"], "kind": config.kind, "level": config.params["level"], }) assert format in KNOWN_FORMATS task["worker"]["upstream-artifacts"] = [{ "taskId": { "task-reference": "<build>" }, "taskType": "build", "paths": paths, "formats": [format], }] task.setdefault("extra", {})["xpi-name"] = dep.task["extra"]["xpi-name"] del task["primary-dependency"] yield task
def add_entitlements_link(config, jobs): for job in jobs: entitlements_path = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-entitlements'], "mac entitlements", { 'platform': job['primary-dependency'].attributes.get('build_platform'), 'release-level': config.params.release_level(), }, ) if entitlements_path: job['entitlements-url'] = config.params.file_url( entitlements_path, endpoint="raw-file" ) yield job
def add_entitlements_link(config, jobs): for job in jobs: entitlements_path = evaluate_keyed_by( config.graph_config["mac-notarization"]["mac-entitlements"], "mac entitlements", { "platform": job["primary-dependency"].attributes.get("build_platform"), "release-level": config.params.release_level(), }, ) if entitlements_path: job["entitlements-url"] = config.params.file_url( entitlements_path, ) yield job
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once is_shippable = dep_job.attributes.get('shippable', False) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type'))) attributes = job['attributes'] if job.get('attributes') else \ copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) worker_type_alias = 'linux-signing' if is_shippable else 'linux-depsigning' mac_behavior = None task = { 'label': label, 'description': description, 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600) }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': _generate_dependencies(job), 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: shippable = "false" if "shippable" in attributes and attributes["shippable"]: shippable = "true" mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'project': config.params['project'], 'shippable': shippable, }, ) if mac_behavior == 'mac_notarize': if 'part-1' in config.kind: mac_behavior = 'mac_notarize_part_1' elif config.kind.endswith('signing'): mac_behavior = 'mac_notarize_part_3' else: raise Exception("Unknown kind {} for mac_behavior!".format( config.kind)) else: if 'part-1' in config.kind: continue task['worker']['mac-behavior'] = mac_behavior worker_type_alias_map = { 'linux-depsigning': 'mac-depsigning', 'linux-signing': 'mac-signing', } assert worker_type_alias in worker_type_alias_map, \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias) ) worker_type_alias = worker_type_alias_map[worker_type_alias] if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault( 'platform', _generate_treeherder_platform(dep_th_platform, build_platform, build_type)) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type'))) attributes = job['attributes'] if job.get('attributes') else \ copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get( 'chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope) mac_behavior = None task = { 'label': label, 'description': description, 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600) }, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': { job['depname']: dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: assert worker_type_alias.startswith("linux-"), \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" ) worker_type_alias = worker_type_alias.replace("linux-", "mac-") mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'release-type': config.params['release_type'], 'platform': build_platform, }, ) task['worker']['mac-behavior'] = mac_behavior if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job["upstream-artifacts"]: for f in artifacts["formats"]: formats.add(f) # Add each format only once is_shippable = dep_job.attributes.get("shippable", False) build_platform = dep_job.attributes.get("build_platform") treeherder = None if "partner" not in config.kind and "eme-free" not in config.kind: treeherder = job.get("treeherder", {}) dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) build_type = dep_job.attributes.get("build_type") treeherder.setdefault( "platform", _generate_treeherder_platform(dep_th_platform, build_platform, build_type), ) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1), ) treeherder.setdefault( "symbol", _generate_treeherder_symbol( dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")), ) treeherder.setdefault("kind", "build") label = job["label"] description = ("Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), )) attributes = (job["attributes"] if job.get("attributes") else copy_attributes_from_dependent_job(dep_job)) attributes["signed"] = True if dep_job.attributes.get("chunk_locales"): # Used for l10n attribute passthrough attributes["chunk_locales"] = dep_job.attributes.get( "chunk_locales") signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_shippable, config) worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning" mac_behavior = None task = { "label": label, "description": description, "worker": { "implementation": "scriptworker-signing", "upstream-artifacts": job["upstream-artifacts"], "max-run-time": job.get("max-run-time", 3600), }, "scopes": [signing_cert_scope] + signing_format_scopes, "dependencies": _generate_dependencies(job), "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "routes": job.get("routes", []), "shipping-product": job.get("shipping-product"), "shipping-phase": job.get("shipping-phase"), } if dep_job.kind in task["dependencies"]: task["if-dependencies"] = [dep_job.kind] if "macosx" in build_platform: shippable = "false" if "shippable" in attributes and attributes["shippable"]: shippable = "true" mac_behavior = evaluate_keyed_by( config.graph_config["mac-notarization"]["mac-behavior"], "mac behavior", { "project": config.params["project"], "shippable": shippable, }, ) if mac_behavior == "mac_notarize": if "part-1" in config.kind: mac_behavior = "mac_notarize_part_1" elif config.kind.endswith("signing"): mac_behavior = "mac_notarize_part_3" else: raise Exception("Unknown kind {} for mac_behavior!".format( config.kind)) else: if "part-1" in config.kind: continue task["worker"]["mac-behavior"] = mac_behavior worker_type_alias_map = { "linux-depsigning": "mac-depsigning", "linux-signing": "mac-signing", } assert worker_type_alias in worker_type_alias_map, ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias)) worker_type_alias = worker_type_alias_map[worker_type_alias] if job.get("entitlements-url"): task["worker"]["entitlements-url"] = job["entitlements-url"] task["worker-type"] = worker_type_alias if treeherder: task["treeherder"] = treeherder if job.get("extra"): task["extra"] = job["extra"] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] yield task
def get_default_priority(graph_config, project): return evaluate_keyed_by( graph_config["task-priority"], "Graph Config", {"project": project} )