def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'cs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of Checksums file for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'cs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {"task-reference": "<beetmover>"},
"taskType": "beetmover",
"paths": [
"public/target.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': [
signing_cert_scope,
"project:releng:signing:format:gpg"
],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
attributes['locale'] = dep_job.attributes.get('locale')
label = job['label']
description = ("Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": ["public/manifest.json"],
}]
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
'worker': {
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
'balrog-action': 'submit-locale',
},
'dependencies': {
'beetmover': dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'promote'),
'shipping-product': job.get('shipping-product'),
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != 'release-snap-repackage' and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcss(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for source file"
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = 'ps({})'.format(locale)
balrog_platform = get_balrog_platform_name(dep_th_platform)
upstream_artifacts = generate_upstream_artifacts(
config.params['release_history'], balrog_platform, locale)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'dependencies': dependencies,
'attributes': attributes,
'scopes': scopes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
repack_id = dep_job.task.get("extra", {}).get("repack_id")
if not repack_id:
raise Exception("Cannot find repack id!")
label = dep_job.label.replace("beetmover-", "beetmover-checksums-")
description = (
"Beetmove checksums for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
extra["partner_path"] = dep_job.task["payload"]["upstreamArtifacts"][0][
"locale"
]
extra["repack_id"] = repack_id
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
"label": label,
"description": description,
"worker-type": "{}/{}".format(
dep_job.task["provisionerId"],
dep_job.task["workerType"],
),
"scopes": dep_job.task["scopes"],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
repack_id = dep_job.task.get('extra', {}).get('repack_id')
if not repack_id:
raise Exception("Cannot find repack id!")
label = dep_job.label.replace("beetmover-", "beetmover-checksums-")
description = (
"Beetmove checksums for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
extra = {}
extra['partner_path'] = dep_job.task['payload']['upstreamArtifacts'][0]['locale']
extra['repack_id'] = repack_id
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
'label': label,
'description': description,
'worker-type': '{}/{}'.format(
dep_job.task['provisionerId'],
dep_job.task['workerType'],
),
'scopes': dep_job.task['scopes'],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-gv')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
attributes['run_on_hg_branches'] = job['run-on-hg-branches']
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, job['bucket-scope']),
'scopes': [
job['bucket-scope'],
'project:releng:beetmover:action:push-to-maven'
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
job['run-on-projects'],
'treeherder':
treeherder,
'shipping-phase':
job['shipping-phase'],
'shipping-product':
job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-gv')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform))
treeherder.setdefault('tier', 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
resolve_keyed_by(job,
'bucket-scope',
item_name=job['label'],
**{'release-level': config.params.release_level()})
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, job['bucket-scope']),
'scopes': [
job['bucket-scope'],
'project:releng:beetmover:action:push-to-maven'
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects': ['mozilla-central'],
'treeherder':
treeherder,
'shipping-phase':
job['shipping-phase'],
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
repack_id = dep_job.task.get('extra', {}).get('repack_id')
if not repack_id:
raise Exception("Cannot find repack id!")
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
label = dep_job.label.replace("repackage-signing-l10n", "beetmover-")
label = dep_job.label.replace("repackage-signing-", "beetmover-")
label = label.replace("repackage-", "beetmover-")
label = label.replace("chunking-dummy-", "beetmover-")
description = (
"Beetmover submission for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=build_platform,
build_type=attributes.get('build_type')))
dependencies = {}
base_label = "release-partner-repack"
if "eme" in config.kind:
base_label = "release-eme-free-repack"
dependencies["build"] = "{}-{}".format(base_label, build_platform)
if "macosx" in build_platform or "win" in build_platform:
dependencies["repackage"] = "{}-repackage-{}-{}".format(
base_label, build_platform, repack_id.replace('/', '-'))
dependencies[
"repackage-signing"] = "{}-repackage-signing-{}-{}".format(
base_label, build_platform, repack_id.replace('/', '-'))
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
'label': label,
'description': description,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
'partner-private-path': job['partner-private-path'],
'partner-public-path': job['partner-public-path'],
'partner-bucket-scope': job['partner-bucket-scope'],
'extra': {
'repack_id': repack_id,
},
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))
)
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for langpack files"
extra = {}
if 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if 'chunk_locales' in dep_job.attributes:
attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'beetmover',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BMcss(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for source file"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {dep_job.kind: dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": "<{}>".format(str(dep_job.kind))
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["autograph_gpg"],
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": [
signing_cert_scope,
],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {"build": dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": "<build>"
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes':
[signing_cert_scope, "project:releng:signing:format:gpg"],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", "")))
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'css(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = "Signing of release-source checksums file"
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target-source.checksums",
],
"formats": ["autograph_gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-gv")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 2)
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
dependencies = deepcopy(dep_job.dependencies)
dependencies[dep_job.kind] = dep_job.label
if job.get("locale"):
attributes["locale"] = job["locale"]
attributes["run_on_hg_branches"] = job["run-on-hg-branches"]
task = {
"label":
label,
"description":
description,
"worker-type":
"beetmover",
"scopes": [
job["bucket-scope"],
"project:releng:beetmover:action:push-to-maven",
],
"dependencies":
dependencies,
"attributes":
attributes,
"run-on-projects":
job["run-on-projects"],
"treeherder":
treeherder,
"shipping-phase":
job["shipping-phase"],
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {
str(dep_job.kind): dep_job.label
}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
resolve_keyed_by(
job, 'worker-type', item_name=label, project=config.params['project']
)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "css(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
description = "Signing of release-source checksums file"
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target-source.checksums",
],
"formats": ["autograph_gpg"],
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": [
signing_cert_scope,
],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task['primary-dependency']
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get('enable-full-crashsymbols'):
logger.debug("Skipping upload symbols task for %s", task['label'])
continue
task['dependencies'] = {'build': dep.label}
task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[
'head_repository']
task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][
'SYMBOL_SECRET'].format(level=config.params['level'])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get('attributes', {}))
task['attributes'] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get('extra')['treeherder']
th_symbol = th.get('symbol')
th_groupsymbol = th.get('groupSymbol', '?')
# Disambiguate the treeherder symbol.
sym = 'Sym' + (th_symbol[1:]
if th_symbol.startswith('B') else th_symbol)
treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym))
task['treeherder'] = treeherder
if attributes.get('nightly'):
# For nightly builds, we want to run these tasks if the build is run.
task['run-on-projects'] = dep.attributes.get('run_on_projects')
elif attributes.get('shippable'):
# For shippable builds, we want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task['run-on-projects'] = dep.attributes.get('run_on_projects')
else:
# For other builds, these can be requested to upload to the try symbol sever.
task['run-on-projects'] = ['try']
# clear out the stuff that's not part of a task description
del task['primary-dependency']
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-BMcs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
upstream_artifact_task = job.pop("upstream-artifact-task", dep_job)
job["attributes"] = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
locale_specifications = generate_specifications_of_artifacts_to_sign(
config,
job,
keep_locale_template=True,
dep_kind=upstream_artifact_task.kind,
)
upstream_artifacts = []
for spec in locale_specifications:
task_type = "l10n"
if "notarization" in upstream_artifact_task.kind:
task_type = "scriptworker"
upstream_artifacts.append({
"taskId": {
"task-reference":
"<{}>".format(upstream_artifact_task.kind)
},
"taskType":
task_type,
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
# Pass paths through set and sorted() so we get a list back
# and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack)
"paths":
sorted(
set([
path_template.format(locale=locale)
for locale in upstream_artifact_task.attributes.get(
"chunk_locales", [])
for path_template in spec["artifacts"]
])),
"formats":
spec["formats"],
})
job["upstream-artifacts"] = upstream_artifacts
yield job
def one_task_per_product_and_platform(config, jobs):
unique_products_and_platforms = set()
for job in jobs:
dep_task = job["primary-dependency"]
if 'primary-dependency' in job:
del job['primary-dependency']
product = dep_task.attributes.get("shipping_product")
platform = dep_task.attributes.get("build_platform")
if (product, platform) not in unique_products_and_platforms:
attributes = copy_attributes_from_dependent_job(dep_task)
attributes.update(job.get('attributes', {}))
job['attributes'] = attributes
job["name"] = "{}-{}".format(product, platform)
yield job
unique_products_and_platforms.add((product, platform))
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-S)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label':
label,
'description':
"{} Beetmover".format(dep_job.task["metadata"]["description"]),
'worker-type':
'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def one_task_per_product_and_platform(config, jobs):
unique_products_and_platforms = set()
for job in jobs:
dep_task = job["primary-dependency"]
if "primary-dependency" in job:
del job["primary-dependency"]
product = dep_task.attributes.get("shipping_product")
platform = dep_task.attributes.get("build_platform")
if (product, platform) not in unique_products_and_platforms:
attr_denylist = ("l10n_chunk", "locale", "artifact_map",
"artifact_prefix")
attributes = copy_attributes_from_dependent_job(
dep_task, denylist=attr_denylist)
attributes.update(job.get("attributes", {}))
job["attributes"] = attributes
job["name"] = "{}-{}".format(product, platform)
yield job
unique_products_and_platforms.add((product, platform))
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {dep_job.kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": "promote",
}
yield task
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
job['attributes']['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
locale_specifications = generate_specifications_of_artifacts_to_sign(
config,
job,
keep_locale_template=True,
)
upstream_artifacts = []
for spec in locale_specifications:
upstream_artifacts.append({
'taskId': {
'task-reference': '<unsigned-repack>'
},
'taskType':
'l10n',
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
# Pass paths through set and sorted() so we get a list back
# and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack)
'paths':
sorted(
set([
path_template.format(locale=locale)
for locale in dep_job.attributes.get(
'chunk_locales', [])
for path_template in spec['artifacts']
])),
'formats':
spec['formats']
})
job['upstream-artifacts'] = upstream_artifacts
yield job
def one_task_per_product_and_platform(config, jobs):
unique_products_and_platforms = set()
for job in jobs:
dep_task = job["primary-dependency"]
if 'primary-dependency' in job:
del job['primary-dependency']
product = dep_task.attributes.get("shipping_product")
platform = dep_task.attributes.get("build_platform")
release_type = dep_task.attributes.get("release-type")
if (product, platform,
release_type) not in unique_products_and_platforms:
attributes = copy_attributes_from_dependent_job(dep_task)
attributes.update(job.get('attributes', {}))
job['attributes'] = attributes
job["name"] = "-".join(
filter(None, (product, platform, release_type)))
yield job
unique_products_and_platforms.add(
(product, platform, release_type))
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
label = config.kind
description = "Beetmover for partner attribution"
if job["partner_public"]:
label = "{}-public".format(label)
description = "{} public".format(description)
else:
label = "{}-private".format(label)
description = "{} private".format(description)
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
"label": label,
"description": description,
"dependencies": {
dep_job.kind: dep_job.label
},
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
"partner_public": job["partner_public"],
"worker": job["worker"],
"scopes": job["scopes"],
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
signing_task = None
for dependency in sorted(dependencies.keys()):
if 'repackage-signing' in dependency:
signing_task = dependency
break
signing_task_ref = '<{}>'.format(signing_task)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar'
)
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {'task-reference': artifact_path},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build]['previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
mar_channel_id = None
if config.params['project'] == 'mozilla-beta':
if 'devedition' in label:
mar_channel_id = 'firefox-mozilla-aurora'
else:
mar_channel_id = 'firefox-mozilla-beta'
elif config.params['project'] == 'mozilla-release':
mar_channel_id = 'firefox-mozilla-release'
elif 'esr' in config.params['project']:
mar_channel_id = 'firefox-mozilla-esr'
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {'in-tree': 'funsize-update-generator'},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'taskcluster-proxy': True,
'env': {
'SHA1_SIGNING_CERT': 'nightly_sha1',
'SHA384_SIGNING_CERT': 'nightly_sha384',
'DATADOG_API_SECRET': 'project/releng/gecko/build/level-3/datadog-api-key'
}
}
if mar_channel_id:
worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
level = config.params['level']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies': dependencies,
'scopes': [
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level,
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
],
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
'worker': worker,
}
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
# This is so we get the build task in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Nr')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
build_task = None
signing_task = None
for dependency in dependencies.keys():
if 'signing' in dependency:
signing_task = dependency
else:
build_task = dependency
if job.get('locale'):
# XXXCallek: todo: rewrite dependency finding
# Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
# We need this additional dependency to support finding the mar binary
# Which is needed in order to generate a new complete.mar
dependencies['build'] = "build-{}/opt".format(
dependencies[build_task][13:dependencies[build_task].rfind('-')])
build_task = 'build'
signing_task_ref = "<{}>".format(signing_task)
build_task_ref = "<{}>".format(build_task)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
locale = None
if job.get('locale'):
locale = job['locale']
attributes['locale'] = locale
level = config.params['level']
build_platform = attributes['build_platform']
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['download_input', 'setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
})
worker = {
'env': _generate_task_env(build_platform, build_task_ref,
signing_task_ref, locale=locale),
'artifacts': _generate_task_output_files(build_platform, locale=locale),
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
}
if locale:
# Make sure we specify the locale-specific upload dir
worker['env'].update(LOCALE=locale)
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
run['use-magic-mh-args'] = False
else:
if build_platform.startswith('macosx'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
elif build_platform.startswith('linux'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
else:
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform)
)
run['tooltool-downloads'] = 'internal'
worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
task = {
'label': job['label'],
'description': description,
'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
}
if build_platform.startswith('macosx'):
task['toolchains'] = [
'linux64-libdmg',
'linux64-hfsplus',
]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-R')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_name = "build-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
signing_dependencies = {signing_name:
dep_job.dependencies[signing_name]
}
dependencies.update(signing_dependencies)
build_name = "build"
if job.get('locale'):
build_name = "unsigned-repack"
build_dependencies = {"build":
dep_job.dependencies[build_name]
}
dependencies.update(build_dependencies)
repackage_name = "repackage"
# repackage-l10n actually uses the repackage depname here
repackage_dependencies = {"repackage":
dep_job.dependencies[repackage_name]
}
dependencies.update(repackage_dependencies)
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
repackage_signing_deps = {"repackage-signing":
dep_job.dependencies[repackage_signing_name]
}
dependencies.update(repackage_signing_deps)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', phase),
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
treeherder = job.get('treeherder', {})
is_nightly = dep_job.attributes.get('nightly', False)
treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
build_platform = dep_job.attributes.get('build_platform')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
dep_job.attributes.get('build_platform'), is_nightly, config
)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'treeherder': treeherder,
'routes': job.get('routes', []),
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
if 'notifications' in job:
task['notifications'] = job['notifications']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
attributes['locale'] = dep_job.attributes.get('locale')
label = job['label']
description = (
"Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
upstream_artifacts = [{
"taskId": {"task-reference": "<beetmover>"},
"taskType": "beetmover",
"paths": [
"public/manifest.json"
],
}]
server_scope = get_balrog_server_scope(config)
channel_scopes = get_balrog_channel_scopes(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
'worker': {
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
},
'scopes': [server_scope] + channel_scopes,
'dependencies': {'beetmover': dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', phase),
'shipping-product': job.get('shipping-product'),
'notifications': job.get('notifications'),
}
yield task
