def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = 'ps({})'.format(locale)
balrog_platform = get_balrog_platform_name(dep_th_platform)
upstream_artifacts = generate_upstream_artifacts(
dep_job, config.params['release_history'], balrog_platform, locale)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:autograph_hash_only_mar384']
if any("mar" in upstream_details["formats"] for upstream_details in upstream_artifacts):
scopes.append('project:releng:signing:format:mar')
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'dependencies': dependencies,
'attributes': attributes,
'scopes': scopes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = 'ps({})'.format(locale)
balrog_platform = get_balrog_platform_name(dep_th_platform)
upstream_artifacts = generate_upstream_artifacts(
config.params['release_history'], balrog_platform, locale)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'dependencies': dependencies,
'attributes': attributes,
'scopes': scopes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
# job['primary-dependency'].task['payload']['command']
formats = ['autograph_authenticode']
else:
formats = ['autograph_gpg']
rev = attributes['openh264_rev']
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
task = {
'label': job['label'],
'description': description,
'worker-type': 'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': my_attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder',
{}).get('tier', 1))
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
worker_type_alias = get_worker_type_for_scope(config,
signing_cert_scope)
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {
job['depname']: dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
assert worker_type_alias.startswith("linux-"), \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
)
worker_type_alias = worker_type_alias.replace("linux-", "mac-")
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'release-type': config.params['release_type'],
'platform': build_platform,
},
)
task['worker']['mac-behavior'] = mac_behavior
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
label = dep_job.label.replace("repackage-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' not in build_platform:
raise Exception(
"Repackage signing is not supported for non-Windows partner repacks."
)
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode"]
}]
scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
task = {
'label': label,
'description': description,
# 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'rs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'rs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope, 'project:releng:signing:format:mar_sha384'
]
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.complete.mar".format(locale_str),
],
"formats": ["mar_sha384"]
}]
if 'win' in build_platform:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.installer.exe".format(locale_str),
],
"formats": ["sha2signcode"]
})
scopes.append("project:releng:signing:format:sha2signcode")
# Stub installer is only generated on win32
if '32' in build_platform:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.stub-installer.exe".format(
locale_str),
],
"formats": ["sha2signcodestub"]
})
scopes.append("project:releng:signing:format:sha2signcodestub")
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
dep_treeherder = dep_job.task.get('extra', {}).get('treeherder', {})
treeherder.setdefault('symbol', '{}(gd-s)'.format(dep_treeherder['groupSymbol']))
treeherder.setdefault('platform', dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault('tier', dep_treeherder.get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v for k, v in signing_dependencies.items()
if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}'".format(
attributes.get('build_platform'),
)
build_platform = dep_job.attributes.get('build_platform')
is_shippable = dep_job.attributes.get('shippable')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind, build_platform)
scopes = [signing_cert_scope]
platform = build_platform.split("-")[0]
task = {
'label': job['label'],
'description': description,
'worker-type': 'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'treeherder': treeherder,
'run-on-projects': ['mozilla-central'],
'index': {"product": "geckodriver", "job-name": platform},
}
if build_platform.startswith('macosx'):
worker_type = task['worker-type']
worker_type_alias_map = {
'linux-depsigning': 'mac-depsigning',
'linux-signing': 'mac-signing',
}
assert worker_type in worker_type_alias_map, \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type)
)
worker_type = worker_type_alias_map[worker_type]
task['worker-type'] = worker_type_alias_map[task['worker-type']]
task['worker']['mac-behavior'] = 'mac_geckodriver'
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
join_symbol(job.get('treeherder-group', 'ms'), locale or 'N'))
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
build_platform = attributes.get('build_platform')
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], build_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(
dep_job, config.kind)
is_nightly = job.get(
'nightly', # First check current job
dep_job.attributes.get(
'nightly', # Then dep job for 'nightly'
dep_job.attributes.get(
'shippable'))) # lastly dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
job.get('run-on-projects',
dep_job.attributes.get('run_on_projects')),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job["upstream-artifacts"]:
for f in artifacts["formats"]:
formats.add(f) # Add each format only once
is_shippable = dep_job.attributes.get("shippable", False)
build_platform = dep_job.attributes.get("build_platform")
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
build_type = dep_job.attributes.get("build_type")
treeherder.setdefault(
"platform",
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type),
)
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder",
{}).get("tier", 1),
)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder",
{}).get("symbol")),
)
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
))
attributes = (job["attributes"] if job.get("attributes") else
copy_attributes_from_dependent_job(dep_job))
attributes["signed"] = True
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
attributes["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning"
mac_behavior = None
task = {
"label": label,
"description": description,
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": job["upstream-artifacts"],
"max-run-time": job.get("max-run-time", 3600),
},
"scopes": [signing_cert_scope] + signing_format_scopes,
"dependencies": _generate_dependencies(job),
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"routes": job.get("routes", []),
"shipping-product": job.get("shipping-product"),
"shipping-phase": job.get("shipping-phase"),
}
if dep_job.kind in task["dependencies"]:
task["if-dependencies"] = [dep_job.kind]
if "macosx" in build_platform:
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config["mac-notarization"]["mac-behavior"],
"mac behavior",
{
"project": config.params["project"],
"shippable": shippable,
},
)
if mac_behavior == "mac_notarize":
if "part-1" in config.kind:
mac_behavior = "mac_notarize_part_1"
elif config.kind.endswith("signing"):
mac_behavior = "mac_notarize_part_3"
else:
raise Exception("Unknown kind {} for mac_behavior!".format(
config.kind))
else:
if "part-1" in config.kind:
continue
task["worker"]["mac-behavior"] = mac_behavior
worker_type_alias_map = {
"linux-depsigning": "mac-depsigning",
"linux-signing": "mac-signing",
}
assert worker_type_alias in worker_type_alias_map, (
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias))
worker_type_alias = worker_type_alias_map[worker_type_alias]
if job.get("entitlements-url"):
task["worker"]["entitlements-url"] = job["entitlements-url"]
task["worker-type"] = worker_type_alias
if treeherder:
task["treeherder"] = treeherder
if job.get("extra"):
task["extra"] = job["extra"]
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage-signing"
treeherder = job.get("treeherder", {})
dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {})
treeherder.setdefault("symbol",
"{}(gd-s)".format(dep_treeherder["groupSymbol"]))
treeherder.setdefault(
"platform",
dep_job.task.get("extra", {}).get("treeherder-platform"))
treeherder.setdefault("tier", dep_treeherder.get("tier", 1))
treeherder.setdefault("kind", "build")
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != "docker-image"
})
description = "Signing Geckodriver for build '{}'".format(
attributes.get("build_platform"), )
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
platform = build_platform.split("-")[0]
task = {
"label": job["label"],
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"treeherder": treeherder,
"run-on-projects": ["mozilla-central"],
"index": {
"product": "geckodriver",
"job-name": platform
},
}
if build_platform.startswith("macosx"):
worker_type = task["worker-type"]
worker_type_alias_map = {
"linux-depsigning": "mac-depsigning",
"linux-signing": "mac-signing",
}
assert worker_type in worker_type_alias_map, (
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type))
worker_type = worker_type_alias_map[worker_type]
task["worker-type"] = worker_type_alias_map[task["worker-type"]]
task["worker"]["mac-behavior"] = "mac_geckodriver"
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
repack_id = dep_job.task["extra"]["repack_id"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501
repack_id=repack_id,
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
if "linux" in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith("release-partner-repack"):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage-signing"
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
scopes = [signing_cert_scope]
if "win" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"],
}]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split("/")
repack_stub_installer = partner_config[partner][subpartner].get(
"repack_stub_installer")
if build_platform.startswith("win32") and repack_stub_installer:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job,
"{}/target.stub-installer.exe".format(repack_id),
),
],
"formats": ["autograph_authenticode", "autograph_gpg"],
})
elif "mac" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["autograph_gpg"],
}]
elif "linux" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["autograph_gpg"],
}]
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"extra": {
"repack_id": repack_id,
},
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if "win" in build_platform:
# job['primary-dependency'].task['payload']['command']
formats = ["autograph_authenticode"]
else:
formats = ["autograph_gpg"]
rev = attributes["openh264_rev"]
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats,
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder",
{}).get("symbol")),
)
task = {
"label": job["label"],
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": my_attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get("locale", dep_job.attributes.get("locale"))
attributes["repackage_type"] = "repackage-signing"
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "rs(B)")
dep_th_platform = dep_job.task.get("extra",
{}).get("treeherder-platform")
treeherder.setdefault("platform", dep_th_platform)
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
if locale:
treeherder["symbol"] = "rs({})".format(locale)
if config.kind == "repackage-signing-msi":
treeherder["symbol"] = "MSIs({})".format(locale or "N")
label = job["label"]
dep_kind = dep_job.kind
if "l10n" in dep_kind:
dep_kind = "repackage"
dependencies = {dep_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != "docker-image"
})
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
scopes = [signing_cert_scope]
upstream_artifacts = []
for artifact in sorted(dep_job.release_artifacts):
basename = os.path.basename(artifact)
if basename in SIGNING_FORMATS:
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep_kind)
},
"taskType":
"repackage",
"paths": [artifact],
"formats":
SIGNING_FORMATS[os.path.basename(artifact)],
})
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"treeherder": treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
if build_platform.startswith('macosx'):
assert task['worker-type'].startswith("linux-"), \
(
"Make sure to adjust the below worker-type logic for "
"mac if you change the signing workerType aliases!"
)
task['worker-type'] = task['worker-type'].replace("linux-", "mac-")
task['worker']['mac-behavior'] = 'mac_geckodriver'
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
"project:releng:signing:format:{}".format(format))
treeherder = job.get('treeherder', {})
is_nightly = dep_job.attributes.get('nightly', False)
treeherder.setdefault('symbol',
_generate_treeherder_symbol(is_nightly))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
build_platform = dep_job.attributes.get('build_platform')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "{}-signing".format(dep_job.label))
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
dep_job.attributes.get('build_platform'), is_nightly, config)
task = {
'label':
label,
'description':
"{} Signing".format(dep_job.task["metadata"]["description"]),
'worker-type':
_generate_worker_type(signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {
job['depname']: dep_job.label
},
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'routes':
job.get('routes', []),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = job.get('treeherder', {})
treeherder['symbol'] = join_symbol(job.get('treeherder-group', 'ms'),
locale or 'N')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
balrog_platform = get_balrog_platform_name(dep_th_platform)
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], balrog_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(dep_job)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope] + list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get('locale', dep_job.attributes.get('locale'))
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
if locale:
treeherder['symbol'] = 'rs({})'.format(locale)
if config.kind == 'repackage-signing-msi':
treeherder['symbol'] = 'MSIs({})'.format(locale or 'N')
label = job['label']
dep_kind = dep_job.kind
if 'l10n' in dep_kind:
dep_kind = 'repackage'
dependencies = {dep_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
upstream_artifacts = []
for artifact in dep_job.release_artifacts:
basename = os.path.basename(artifact)
if basename in SIGNING_FORMATS:
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep_kind)
},
"taskType":
"repackage",
"paths": [artifact],
"formats":
SIGNING_FORMATS[os.path.basename(artifact)],
})
scopes += list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
treeherder = job.get('treeherder', {})
is_nightly = dep_job.attributes.get('nightly', False)
treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
build_platform = dep_job.attributes.get('build_platform')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
dep_job.attributes.get('build_platform'), is_nightly, config
)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'treeherder': treeherder,
'routes': job.get('routes', []),
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
}]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split('/')
repack_stub_installer = partner_config[partner][subpartner].get(
'repack_stub_installer')
if build_platform.startswith('win32') and repack_stub_installer:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job,
"{}/target.stub-installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
})
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
task = {
'label': label,
'description': description,
'worker-type': 'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_shippable = dep_job.attributes.get('shippable', False)
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder',
{}).get('tier', 1))
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
worker_type_alias = 'linux-signing' if is_shippable else 'linux-depsigning'
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': _generate_dependencies(job),
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'project': config.params['project'],
'shippable': shippable,
},
)
if mac_behavior == 'mac_notarize':
if 'part-1' in config.kind:
mac_behavior = 'mac_notarize_part_1'
elif config.kind.endswith('signing'):
mac_behavior = 'mac_notarize_part_3'
else:
raise Exception("Unknown kind {} for mac_behavior!".format(
config.kind))
else:
if 'part-1' in config.kind:
continue
task['worker']['mac-behavior'] = mac_behavior
worker_type_alias_map = {
'linux-depsigning': 'mac-depsigning',
'linux-signing': 'mac-signing',
}
assert worker_type_alias in worker_type_alias_map, \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias)
)
worker_type_alias = worker_type_alias_map[worker_type_alias]
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg')
]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode", "gpg"]
}]
scopes.append(
add_scope_prefix(config, "signing:format:sha2signcode"))
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["gpg"]
}]
task = {
'label': label,
'description': description,
# 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('symbol', _generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol')
))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
worker_type = get_worker_type_for_scope(config, signing_cert_scope)
task = {
'label': label,
'description': description,
'worker-type': worker_type,
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-rs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "repackage-signing-{}".format(dep_job.label))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-rs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
upstream_artifacts = [{
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
"public/build/{}target.complete.mar".format(locale_str),
],
"formats": ["mar_sha384"]
}]
if 'win' in build_platform:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
"public/build/{}target.installer.exe".format(locale_str),
],
"formats": ["sha2signcode"]
})
scopes.append("project:releng:signing:format:sha2signcode")
# Stub installer is only generated on win32
if '32' in build_platform:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
"public/build/{}target.stub-installer.exe".format(locale_str),
],
"formats": ["sha2signcodestub"]
})
scopes.append("project:releng:signing:format:sha2signcodestub")
task = {
'label': label,
'description': "Repackage signing {} ".format(
dep_job.task["metadata"]["description"]),
'worker-type': "scriptworker-prov-v1/signing-linux-v1",
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
funsize_platforms = [
'linux-nightly',
'linux64-nightly',
'macosx64-nightly',
'win32-nightly',
'win64-nightly'
]
if build_platform in funsize_platforms and is_nightly:
route_template = "project.releng.funsize.level-{level}.{project}"
task['routes'] = [
route_template.format(project=config.params['project'],
level=config.params['level'])
]
yield task
