def make_task_description(config, jobs):
for job in jobs:
if len(job["dependencies"]) != 1:
raise Exception("Exactly 1 dependency is required")
job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(
job["dependencies"])
resolve_keyed_by(job,
"worker.channel",
item_name=job["name"],
**{"release-type": config.params["release_type"]})
resolve_keyed_by(job,
"worker-type",
item_name=job["name"],
**{"release-level": config.params.release_level()})
if config.params.release_level() == "production":
job.setdefault("scopes", []).append(
add_scope_prefix(
config,
"snapcraft:firefox:{}".format(
job["worker"]["channel"].split("/")[0]),
))
yield job
def make_task_description(config, jobs):
for job in jobs:
if len(job['dependencies']) != 1:
raise Exception('Exactly 1 dependency is required')
job['worker']['upstream-artifacts'] = generate_upstream_artifacts(
job['dependencies'])
resolve_keyed_by(job,
'worker.channel',
item_name=job['name'],
**{'release-type': config.params['release_type']})
resolve_keyed_by(job,
'worker-type',
item_name=job['name'],
**{'release-level': config.params.release_level()})
if config.params.release_level() == 'production':
job.setdefault('scopes', []).append(
add_scope_prefix(
config,
"snapcraft:firefox:{}".format(
job['worker']['channel'].split('/')[0]),
))
yield job
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rel(BM-C)")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
treeherder.setdefault("platform", job["treeherder-platform"])
label = job["name"]
description = "Beetmover push to release for '{product}'".format(
product=job["product"])
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-releases")
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"product": job["product"],
"dependencies": job["dependencies"],
"attributes": job.get("attributes", {}),
"run-on-projects": job.get("run-on-projects"),
"treeherder": treeherder,
"shipping-phase": job.get("shipping-phase", "push"),
"shipping-product": job.get("shipping-product"),
"routes": job.get("routes", []),
"extra": job.get("extra", {}),
}
yield task
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = (
"Beetmover push to release for '{product}'".format(
product=job['product']
)
)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = add_scope_prefix(config, 'beetmover:action:push-to-releases')
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'push'),
'shipping-product': job.get('shipping-product'),
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
}
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'css(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = "Signing of release-source checksums file"
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target-source.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg'),
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False):
action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner')
task = deepcopy(job)
task['scopes'] = [bucket_scope, action_scope]
task['worker-type'] = get_worker_type_for_scope(config, bucket_scope)
task['partner_public'] = partner_public
if partner_public:
task['label'] = "{}-public".format(task['label'])
return task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False):
action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner")
task = deepcopy(job)
task["scopes"] = [bucket_scope, action_scope]
task["worker-type"] = "beetmover"
task["partner_public"] = partner_public
if partner_public:
task["label"] = "{}-public".format(task["label"])
return task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {
"build": dep_job.label
}
upstream_artifacts = [{
"taskId": {"task-reference": "<build>"},
"taskType": "build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg'),
],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def split_public_and_private(config, jobs):
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"])
partner, subpartner, _ = job["extra"]["repack_id"].split("/")
if partner_config[partner][subpartner].get("upload_to_candidates"):
# public
yield populate_scopes_and_worker_type(
config, job, public_bucket_scope, partner_public=True
)
else:
# private
yield populate_scopes_and_worker_type(
config, job, partner_bucket_scope, partner_public=False
)
def split_public_and_private(config, jobs):
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
partner_bucket_scope = add_scope_prefix(config,
job['partner-bucket-scope'])
partner, subpartner, _ = job['extra']['repack_id'].split('/')
# public
if partner_config[partner][subpartner].get('upload_to_candidates'):
yield populate_scopes_and_worker_type(config,
job,
public_bucket_scope,
partner_public=True)
# private
yield populate_scopes_and_worker_type(config,
job,
partner_bucket_scope,
partner_public=False)
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg')
]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode", "gpg"]
}]
scopes.append(
add_scope_prefix(config, "signing:format:sha2signcode"))
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["gpg"]
}]
task = {
'label': label,
'description': description,
# 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
join_symbol(job.get('treeherder-group', 'ms'), locale or 'N'))
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
balrog_platform = get_balrog_platform_name(dep_th_platform)
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], balrog_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(
dep_job, config.kind)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = job.get(
'nightly', # First check current job
dep_job.attributes.get(
'nightly', # Then dep job for 'nightly'
dep_job.attributes.get(
'shippable'))) # lastly dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope] + list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
job.get('run-on-projects',
dep_job.attributes.get('run_on_projects')),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
treeherder = job.get('treeherder', {})
is_nightly = dep_job.attributes.get('nightly', False)
treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
build_platform = dep_job.attributes.get('build_platform')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
dep_job.attributes.get('build_platform'), is_nightly, config
)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'treeherder': treeherder,
'routes': job.get('routes', []),
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly', dep_job.attributes.get('shippable'))
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode", "autograph_gpg"]
}]
scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split('/')
repack_stub_installer = partner_config[partner][subpartner].get(
'repack_stub_installer')
if build_platform.startswith('win32') and repack_stub_installer:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}/target.stub-installer.exe".format(
repack_id)),
],
"formats": ["sha2signcode", "autograph_gpg"]
})
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {"task-reference": "<repack>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def split_public_and_private(config, jobs):
# we need to separate private vs public destinations because beetmover supports one
# in a single task. Only use a single task for each type though.
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
upstream_artifacts = job["primary-dependency"].release_artifacts
attribution_task_ref = "<{}>".format(job["primary-dependency"].label)
prefix = get_artifact_prefix(job["primary-dependency"])
artifacts = defaultdict(list)
for artifact in upstream_artifacts:
partner, sub_partner, platform, locale, _ = artifact.replace(
prefix + "/", "").split("/", 4)
destination = "private"
this_config = [
p for p in partner_config["configs"]
if (p["campaign"] == partner and p["content"] == sub_partner)
]
if this_config[0].get("upload_to_candidates"):
destination = "public"
artifacts[destination].append(
(artifact, partner, sub_partner, platform, locale))
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-partner")
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_bucket_scope = add_scope_prefix(config,
job["partner-bucket-scope"])
repl_dict = {
"build_number":
config.params["build_number"],
"release_partner_build_number":
config.params["release_partner_build_number"],
"version":
config.params["version"],
"partner":
"{partner}", # we'll replace these later, per artifact
"subpartner":
"{subpartner}",
"platform":
"{platform}",
"locale":
"{locale}",
}
for destination, destination_artifacts in artifacts.items():
this_job = deepcopy(job)
if destination == "public":
this_job["scopes"] = [public_bucket_scope, action_scope]
this_job["partner_public"] = True
else:
this_job["scopes"] = [partner_bucket_scope, action_scope]
this_job["partner_public"] = False
partner_path_key = "partner-{destination}-path".format(
destination=destination)
partner_path = this_job[partner_path_key].format(**repl_dict)
this_job.setdefault(
"worker",
{})["upstream-artifacts"] = generate_upstream_artifacts(
attribution_task_ref, destination_artifacts, partner_path)
yield this_job
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of Checksums file for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'cs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg'),
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get('locale', dep_job.attributes.get('locale'))
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
if locale:
treeherder['symbol'] = 'rs({})'.format(locale)
if config.kind == 'repackage-signing-msi':
treeherder['symbol'] = 'MSIs({})'.format(locale or 'N')
label = job['label']
dep_kind = dep_job.kind
if 'l10n' in dep_kind:
dep_kind = 'repackage'
dependencies = {dep_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
upstream_artifacts = []
for artifact in dep_job.release_artifacts:
basename = os.path.basename(artifact)
if basename in SIGNING_FORMATS:
upstream_artifacts.append({
"taskId": {
"task-reference": "<{}>".format(dep_kind)
},
"taskType":
"repackage",
"paths": [artifact],
"formats":
SIGNING_FORMATS[os.path.basename(artifact)],
})
scopes += list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Gd(s)')
treeherder.setdefault(
'platform',
dep_job.task.get('extra', {}).get('treeherder-platform'))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
description = "Signing Geckodriver for build '{}/{}'".format(
attributes.get('build_platform'),
attributes.get('build_type'),
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
upstream_artifacts = _craft_upstream_artifacts(dep_job.kind,
build_platform)
scopes = [signing_cert_scope]
scopes += list({
add_scope_prefix(config, 'signing:format:{}'.format(format))
for artifact in upstream_artifacts
for format in artifact['formats']
})
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
# job['primary-dependency'].task['payload']['command']
scopes.append(
add_scope_prefix(config, "signing:format:sha2signcode"))
formats = ['sha2signcode']
else:
formats = ['autograph_gpg']
rev = attributes['openh264_rev']
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
task = {
'label': job['label'],
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': my_attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('kind', 'build')
label = job['label']
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({k: v for k, v in signing_dependencies.items()
if k != 'docker-image'})
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'rs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, add_scope_prefix(config, 'signing:format:mar_sha384')]
upstream_artifacts = [{
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}target.complete.mar".format(locale_str)),
],
"formats": ["mar_sha384"]
}]
if 'win' in build_platform:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}target.installer.exe".format(locale_str)),
],
"formats": ["sha2signcode"]
})
scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
use_stub = attributes.get('stub-installer')
if use_stub:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(
dep_job, "{}target.stub-installer.exe".format(locale_str)
),
],
"formats": ["sha2signcodestub"]
})
scopes.append(add_scope_prefix(config, "signing:format:sha2signcodestub"))
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format)))
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
_generate_treeherder_platform(dep_th_platform, build_platform,
build_type))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder',
{}).get('tier', 1))
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get(
'chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform,
is_nightly,
config,
job_release_type=dep_job.attributes.get('release-type'))
worker_type_alias = get_worker_type_for_scope(config,
signing_cert_scope)
mac_behavior = None
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)
},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {
job['depname']: dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if 'macosx' in build_platform:
assert worker_type_alias.startswith("linux-"), \
(
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
)
worker_type_alias = worker_type_alias.replace("linux-", "mac-")
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',
{
'release-type': config.params['release_type'],
'platform': build_platform,
},
)
task['worker']['mac-behavior'] = mac_behavior
if job.get('entitlements-url'):
task['worker']['entitlements-url'] = job['entitlements-url']
task['worker-type'] = worker_type_alias
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
add_scope_prefix(config, 'signing:format:{}'.format(format))
)
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault('platform', _generate_treeherder_platform(
dep_th_platform, build_platform, build_type
))
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('symbol', _generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol')
))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
worker_type = get_worker_type_for_scope(config, signing_cert_scope)
task = {
'label': label,
'description': description,
'worker-type': worker_type,
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': job.get('max-run-time', 3600)},
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
label = dep_job.label.replace("repackage-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != 'docker-image'
})
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' not in build_platform:
raise Exception(
"Repackage signing is not supported for non-Windows partner repacks."
)
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.installer.exe".format(repack_id)),
],
"formats": ["sha2signcode"]
}]
scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
task = {
'label': label,
'description': description,
# 'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
yield task
