def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'cs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of Checksums file for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'cs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {"task-reference": "<beetmover>"},
"taskType": "beetmover",
"paths": [
"public/target.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': [
signing_cert_scope,
"project:releng:signing:format:gpg"
],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol', 'BMcslang(N{})'.format(attributes.get('l10n_chunk', '')))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for langpack files"
extra = {}
if 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if 'chunk_locales' in dep_job.attributes:
attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = 'ps({})'.format(locale)
balrog_platform = get_balrog_platform_name(dep_th_platform)
upstream_artifacts = generate_upstream_artifacts(
dep_job, config.params['release_history'], balrog_platform, locale)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'dependencies': dependencies,
'attributes': attributes,
'scopes': scopes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != 'release-snap-repackage' and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
}
yield task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False):
action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner')
task = deepcopy(job)
task['scopes'] = [bucket_scope, action_scope]
task['worker-type'] = get_worker_type_for_scope(config, bucket_scope)
task['partner_public'] = partner_public
if partner_public:
task['label'] = "{}-public".format(task['label'])
return task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-gv')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
resolve_keyed_by(job,
'bucket-scope',
item_name=job['label'],
**{'release-level': config.params.release_level()})
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, job['bucket-scope']),
'scopes': [
job['bucket-scope'],
'project:releng:beetmover:action:push-to-maven'
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects': ['mozilla-central'],
'treeherder':
treeherder,
'shipping-phase':
job['shipping-phase'],
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-gv')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
attributes['run_on_hg_branches'] = job['run-on-hg-branches']
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, job['bucket-scope']),
'scopes': [
job['bucket-scope'],
'project:releng:beetmover:action:push-to-maven'
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
job['run-on-projects'],
'treeherder':
treeherder,
'shipping-phase':
job['shipping-phase'],
'shipping-product':
job.get('shipping-product'),
}
yield task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {"build": dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": "<build>"
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes':
[signing_cert_scope, "project:releng:signing:format:gpg"],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def add_balrog_scopes(config, jobs):
for job in jobs:
worker = job['worker']
server_scope = get_balrog_server_scope(config)
action_scope = get_balrog_action_scope(config,
action=worker['balrog-action'])
job['scopes'] = [server_scope, action_scope]
job['worker-type'] = get_worker_type_for_scope(config, server_scope)
yield job
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'css(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = "Signing of release-source checksums file"
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target-source.checksums",
],
"formats": ["autograph_gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {
str(dep_job.kind): dep_job.label
}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': 'promote',
}
yield task
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = ("Beetmover push to release for '{product}'".format(
product=job['product']))
bucket_scope = get_beetmover_bucket_scope(config,
job_release_type=job.get(
'attributes',
{}).get('release-type'))
action_scope = add_scope_prefix(config,
'beetmover:action:push-to-releases')
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'push'),
'shipping-product': job.get('shipping-product'),
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-R')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = dep_job.kind
if dependent_kind == 'repackage-signing-l10n':
dependent_kind = "repackage-signing"
dependencies = {dependent_kind: dep_job}
signing_name = "build-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
dependencies['signing'] = job['grandparent-tasks'][signing_name]
build_name = "build"
if job.get('locale'):
build_name = "unsigned-repack"
dependencies["build"] = job['grandparent-tasks'][build_name]
# repackage-l10n actually uses the repackage depname here
dependencies["repackage"] = job['grandparent-tasks']["repackage"]
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
if job.get('locale'):
repackage_signing_name = "repackage-signing-l10n"
dependencies["repackage-signing"] = job['grandparent-tasks'][
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', phase),
'shipping-product': job.get('shipping-product'),
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get('locale', dep_job.attributes.get('locale'))
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
if locale:
treeherder['symbol'] = 'rs({})'.format(locale)
if config.kind == 'repackage-signing-msi':
treeherder['symbol'] = 'MSIs({})'.format(locale or 'N')
label = job['label']
dep_kind = dep_job.kind
if 'l10n' in dep_kind:
dep_kind = 'repackage'
dependencies = {dep_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
upstream_artifacts = []
for artifact in dep_job.release_artifacts:
basename = os.path.basename(artifact)
if basename in SIGNING_FORMATS:
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep_kind)
},
"taskType":
"repackage",
"paths": [artifact],
"formats":
SIGNING_FORMATS[os.path.basename(artifact)],
})
scopes += list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg')
]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode", "gpg"]
}]
scopes.append(
add_scope_prefix(config, "signing:format:sha2signcode"))
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["gpg"]
}]
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
join_symbol(job.get('treeherder-group', 'ms'), locale or 'N'))
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
balrog_platform = get_balrog_platform_name(dep_th_platform)
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], balrog_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(
dep_job, config.kind)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = job.get(
'nightly', # First check current job
dep_job.attributes.get(
'nightly', # Then dep job for 'nightly'
dep_job.attributes.get(
'shippable'))) # lastly dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
job.get('run-on-projects',
dep_job.attributes.get('run_on_projects')),
'treeherder':
treeherder,
}
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of Checksums file for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'cs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg'),
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('symbol', _generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol')
))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config,
job_release_type=dep_job.attributes.get('release-type')
)
worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope)
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
worker_type_alias_map = {
'linux-depsigning': 'mac-depsigning',
'linux-signing': 'mac-signing',
}
assert worker_type_alias in worker_type_alias_map, \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias)
)
worker_type_alias = worker_type_alias_map[worker_type_alias]
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'release-type': config.params['release_type'],
'platform': build_platform,
},
)
task['worker']['mac-behavior'] = mac_behavior
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
treeherder = job.get('treeherder', {})
is_nightly = dep_job.attributes.get('nightly', False)
treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
build_platform = dep_job.attributes.get('build_platform')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
dep_job.attributes.get('build_platform'), is_nightly, config
)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'treeherder': treeherder,
'routes': job.get('routes', []),
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'rs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'rs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope, 'project:releng:signing:format:mar_sha384'
]
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.complete.mar".format(locale_str),
],
"formats": ["mar_sha384"]
}]
if 'win' in build_platform:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.installer.exe".format(locale_str),
],
"formats": ["sha2signcode"]
})
scopes.append("project:releng:signing:format:sha2signcode")
# Stub installer is only generated on win32
if '32' in build_platform:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
"public/build/{}target.stub-installer.exe".format(
locale_str),
],
"formats": ["sha2signcodestub"]
})
scopes.append("project:releng:signing:format:sha2signcodestub")
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
}]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split('/')
repack_stub_installer = partner_config[partner][subpartner].get(
'repack_stub_installer')
if build_platform.startswith('win32') and repack_stub_installer:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job,
"{}/target.stub-installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
})
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
# job['primary-dependency'].task['payload']['command']
scopes.append(
add_scope_prefix(config, "signing:format:sha2signcode"))
formats = ['sha2signcode']
else:
formats = ['autograph_gpg']
rev = attributes['openh264_rev']
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': my_attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('symbol', _generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol')
))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
worker_type = get_worker_type_for_scope(config, signing_cert_scope)
task = {
'label': label,
'description': description,
'worker-type': worker_type,
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
if build_platform.startswith('macosx'):
worker_type = task['worker-type']
worker_type_alias_map = {
'linux-depsigning': 'mac-depsigning',
'linux-signing': 'mac-signing',
}
assert worker_type in worker_type_alias_map, \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type)
)
worker_type = worker_type_alias_map[worker_type]
task['worker-type'] = worker_type_alias_map[task['worker-type']]
task['worker']['mac-behavior'] = 'mac_geckodriver'
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
upstream_symbol = dep_job.task['extra']['treeherder']['symbol']
if 'build' in job['dependent-tasks']:
upstream_symbol = job['dependent-tasks']['build'].task['extra'][
'treeherder']['symbol']
treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_deps = job['dependent-tasks']
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
build_name = "nightly-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if 'partials-signing' in upstream_deps:
dependencies['partials-signing'] = upstream_deps[
'partials-signing']
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.task['extra']['treeherder']['symbol']
treeherder['symbol'] = replace_group(treeherder_job_symbol, 'c-Up')
if dep_job.attributes.get('locale'):
attributes['locale'] = dep_job.attributes.get('locale')
label = job['label']
description = ("Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": ["public/manifest.json"],
}]
server_scope = get_balrog_server_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, server_scope),
'worker': {
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
'balrog-action': 'submit-locale',
'suffixes':
['', '-No-WNP'] if job.get('update-no-wnp') else [''],
},
'dependencies': {
'beetmover': dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'promote'),
'shipping-product': job.get('shipping-product'),
}
yield task
