def add_notifications(config, jobs):
xpi_name = config.params.get("xpi_name")
xpi_revision = config.params.get("xpi_revision")
shipping_phase = config.params.get("shipping_phase")
additional_shipit_emails = config.params.get("additional_shipit_emails",
[])
if not all([xpi_name, xpi_revision, shipping_phase]):
return
manifest = get_manifest()
for job in jobs:
if "primary-dependency" in job:
dep = job.pop("primary-dependency")
if dep.task.get("extra", {}).get("xpi-name") != xpi_name:
continue
attributes = dep.attributes.copy()
if job.get("attributes"):
attributes.update(job["attributes"])
job["attributes"] = attributes
job.setdefault("dependencies", {}).update({"signing": dep.label})
if job.get("attributes", {}).get("shipping-phase") != shipping_phase:
continue
job["label"] = f"{config.kind}-{shipping_phase}"
xpi_config = manifest[xpi_name]
xpi_type = xpi_config["addon-type"]
emails = evaluate_keyed_by(
config.graph_config["release-promotion"]["notifications"]
[xpi_type],
"email",
dict(phase=shipping_phase, level=config.params["level"]),
)
if not emails:
continue
emails = (emails + additional_shipit_emails +
xpi_config.get("additional-emails", []))
notifications = evaluate_keyed_by(job.pop("notifications"),
"notification config",
dict(phase=shipping_phase))
format_kwargs = dict(config=config.__dict__)
subject = notifications["subject"].format(**format_kwargs)
message = notifications["message"].format(**format_kwargs)
# We only send mail on success to avoid messages like 'blah is in the
# candidates dir' when cancelling graphs, dummy job failure, etc
job.setdefault("routes", []).extend(
[f"notify.email.{email}.on-completed" for email in emails])
job.setdefault("extra",
{}).update({"notify": {
"email": {
"subject": subject
}
}})
if message:
job["extra"]["notify"]["email"]["content"] = message
yield job
def add_notifications(config, jobs):
xpi_name = config.params.get("xpi_name")
xpi_revision = config.params.get("xpi_revision")
shipping_phase = config.params.get("shipping_phase")
if not all([xpi_name, xpi_revision, shipping_phase]):
return
manifest = get_manifest()
for job in jobs:
if "primary-dependency" in job:
dep = job.pop("primary-dependency")
if dep.task.get("extra", {}).get("xpi-name") != xpi_name:
continue
attributes = dep.attributes.copy()
if job.get("attributes"):
attributes.update(job["attributes"])
job["attributes"] = attributes
job.setdefault("dependencies", {}).update({"signing": dep.label})
if job.get("attributes", {}).get("shipping-phase") != shipping_phase:
continue
job['label'] = '{}-{}'.format(config.kind, shipping_phase)
xpi_config = manifest[xpi_name]
xpi_type = xpi_config['addon-type']
emails = evaluate_keyed_by(
config.graph_config['release-promotion']['notifications']
[xpi_type], 'email', dict(
phase=shipping_phase, )) + xpi_config.get(
"additional-emails", [])
notifications = evaluate_keyed_by(job.pop('notifications'),
'notification config',
dict(phase=shipping_phase, ))
format_kwargs = dict(config=config.__dict__, )
subject = notifications['subject'].format(**format_kwargs)
message = notifications['message'].format(**format_kwargs)
# We only send mail on success to avoid messages like 'blah is in the
# candidates dir' when cancelling graphs, dummy job failure, etc
job.setdefault('routes', []).extend(
['notify.email.{}.on-completed'.format(email) for email in emails])
job.setdefault('extra', {}).update(
{'notify': {
'email': {
'subject': subject,
}
}})
if message:
job['extra']['notify']['email']['content'] = message
yield job
def build_signing_task(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["dependencies"] = {"build": dep.label}
if not dep.task["payload"]["env"]["ARTIFACT_PREFIX"].startswith(
"public"):
scopes = task.setdefault('scopes', [])
scopes.append("queue:get-artifact:{}/*".format(
dep.task["payload"]["env"]["ARTIFACT_PREFIX"].rstrip('/')))
paths = dep.attributes["xpis"].values()
format = evaluate_keyed_by(
config.graph_config["scriptworker"]["signing-format"],
"signing-format", {
"xpi-type": task["attributes"]["addon-type"],
"kind": config.kind,
"level": config.params["level"],
})
assert format in KNOWN_FORMATS
task["worker"]["upstream-artifacts"] = [{
"taskId": {
"task-reference": "<build>"
},
"taskType": "build",
"paths": paths,
"formats": [format],
}]
task.setdefault("extra",
{})["xpi-name"] = dep.task["extra"]["xpi-name"]
del task["primary-dependency"]
yield task
def add_entitlements_link(config, jobs):
for job in jobs:
entitlements_path = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-entitlements'],
"mac entitlements",
{
'platform': job['primary-dependency'].attributes.get('build_platform'),
'release-level': config.params.release_level(),
},
)
if entitlements_path:
job['entitlements-url'] = config.params.file_url(
entitlements_path, endpoint="raw-file"
)
yield job
def add_entitlements_link(config, jobs):
for job in jobs:
entitlements_path = evaluate_keyed_by(
config.graph_config["mac-notarization"]["mac-entitlements"],
"mac entitlements",
{
"platform":
job["primary-dependency"].attributes.get("build_platform"),
"release-level":
config.params.release_level(),
},
)
if entitlements_path:
job["entitlements-url"] = config.params.file_url(
entitlements_path, )
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_shippable = dep_job.attributes.get('shippable', False)
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder',
{}).get('tier', 1))
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
worker_type_alias = 'linux-signing' if is_shippable else 'linux-depsigning'
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': _generate_dependencies(job),
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'project': config.params['project'],
'shippable': shippable,
},
)
if mac_behavior == 'mac_notarize':
if 'part-1' in config.kind:
mac_behavior = 'mac_notarize_part_1'
elif config.kind.endswith('signing'):
mac_behavior = 'mac_notarize_part_3'
else:
raise Exception("Unknown kind {} for mac_behavior!".format(
config.kind))
else:
if 'part-1' in config.kind:
continue
task['worker']['mac-behavior'] = mac_behavior
worker_type_alias_map = {
'linux-depsigning': 'mac-depsigning',
'linux-signing': 'mac-signing',
}
assert worker_type_alias in worker_type_alias_map, \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias)
)
worker_type_alias = worker_type_alias_map[worker_type_alias]
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder',
{}).get('tier', 1))
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
worker_type_alias = get_worker_type_for_scope(config,
signing_cert_scope)
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {
job['depname']: dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
assert worker_type_alias.startswith("linux-"), \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
)
worker_type_alias = worker_type_alias.replace("linux-", "mac-")
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'release-type': config.params['release_type'],
'platform': build_platform,
},
)
task['worker']['mac-behavior'] = mac_behavior
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job["upstream-artifacts"]:
for f in artifacts["formats"]:
formats.add(f) # Add each format only once
is_shippable = dep_job.attributes.get("shippable", False)
build_platform = dep_job.attributes.get("build_platform")
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
build_type = dep_job.attributes.get("build_type")
treeherder.setdefault(
"platform",
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type),
)
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder",
{}).get("tier", 1),
)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder",
{}).get("symbol")),
)
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
))
attributes = (job["attributes"] if job.get("attributes") else
copy_attributes_from_dependent_job(dep_job))
attributes["signed"] = True
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
attributes["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning"
mac_behavior = None
task = {
"label": label,
"description": description,
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": job["upstream-artifacts"],
"max-run-time": job.get("max-run-time", 3600),
},
"scopes": [signing_cert_scope] + signing_format_scopes,
"dependencies": _generate_dependencies(job),
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"routes": job.get("routes", []),
"shipping-product": job.get("shipping-product"),
"shipping-phase": job.get("shipping-phase"),
}
if dep_job.kind in task["dependencies"]:
task["if-dependencies"] = [dep_job.kind]
if "macosx" in build_platform:
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config["mac-notarization"]["mac-behavior"],
"mac behavior",
{
"project": config.params["project"],
"shippable": shippable,
},
)
if mac_behavior == "mac_notarize":
if "part-1" in config.kind:
mac_behavior = "mac_notarize_part_1"
elif config.kind.endswith("signing"):
mac_behavior = "mac_notarize_part_3"
else:
raise Exception("Unknown kind {} for mac_behavior!".format(
config.kind))
else:
if "part-1" in config.kind:
continue
task["worker"]["mac-behavior"] = mac_behavior
worker_type_alias_map = {
"linux-depsigning": "mac-depsigning",
"linux-signing": "mac-signing",
}
assert worker_type_alias in worker_type_alias_map, (
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias))
worker_type_alias = worker_type_alias_map[worker_type_alias]
if job.get("entitlements-url"):
task["worker"]["entitlements-url"] = job["entitlements-url"]
task["worker-type"] = worker_type_alias
if treeherder:
task["treeherder"] = treeherder
if job.get("extra"):
task["extra"] = job["extra"]
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def get_default_priority(graph_config, project):
return evaluate_keyed_by(
graph_config["task-priority"], "Graph Config", {"project": project}
)
