def make_beetmover_cdns_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = ("Beetmover push to cdns for '{product}'".format(
product=job['product']))
resolve_keyed_by(job,
'worker-type',
item_name=job['name'],
project=config.params['project'])
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk', '')))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
job['description'] = job['description'].format(
locales='/'.join(job['attributes']['chunk_locales']),
platform=job['attributes']['build_platform']
)
job['scopes'] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job['dependencies'] = {dep_job.kind: dep_job.label}
job['run-on-projects'] = dep_job.attributes['run_on_projects']
job['treeherder'] = treeherder
job['shipping-phase'] = dep_job.attributes['shipping_phase']
job['shipping-product'] = dep_job.attributes['shipping_product']
yield job
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rel(BM-C)")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
treeherder.setdefault("platform", job["treeherder-platform"])
label = job["name"]
description = "Beetmover push to release for '{product}'".format(
product=job["product"])
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-releases")
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"product": job["product"],
"dependencies": job["dependencies"],
"attributes": job.get("attributes", {}),
"run-on-projects": job.get("run-on-projects"),
"treeherder": treeherder,
"shipping-phase": job.get("shipping-phase", "push"),
"shipping-product": job.get("shipping-product"),
"routes": job.get("routes", []),
"extra": job.get("extra", {}),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(BM{})".format(attributes.get("l10n_chunk",
"")))
job["attributes"].update(copy_attributes_from_dependent_job(dep_job))
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"])
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
platform=job["attributes"]["build_platform"],
)
job["scopes"] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job["dependencies"] = {"langpack-copy": dep_job.label}
job["run-on-projects"] = job.get("run_on_projects",
dep_job.attributes["run_on_projects"])
job["treeherder"] = treeherder
job["shipping-phase"] = job.get("shipping-phase",
dep_job.attributes["shipping_phase"])
job["shipping-product"] = dep_job.attributes["shipping_product"]
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-R)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
# macosx nightly builds depend on repackage which use in tree docker
# images and thus have two dependencies
# change the signing_dependencies to be use the ones in
docker_dependencies = {"docker-image":
dep_job.dependencies["docker-image"]
}
dependencies.update(docker_dependencies)
signing_dependencies = {"build-signing":
dep_job.dependencies["build-signing"]
}
dependencies.update(signing_dependencies)
repackage_dependencies = {"repackage":
dep_job.label
}
dependencies.update(repackage_dependencies)
build_dependencies = {"build":
dep_job.dependencies["build-signing"].replace("signing-", "build-")
}
dependencies.update(build_dependencies)
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_cdns_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = ("Beetmover push to cdns for '{product}'".format(
product=job['product']))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'push'),
'shipping-product': job.get('shipping-product'),
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk',
'')))
job['attributes'].update(copy_attributes_from_dependent_job(dep_job))
job['attributes']['chunk_locales'] = dep_job.attributes[
'chunk_locales']
job['description'] = job['description'].format(
locales='/'.join(job['attributes']['chunk_locales']),
platform=job['attributes']['build_platform'])
job['scopes'] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job['dependencies'] = {dep_job.kind: dep_job.label}
job['run-on-projects'] = dep_job.attributes['run_on_projects']
job['treeherder'] = treeherder
job['shipping-phase'] = dep_job.attributes['shipping_phase']
job['shipping-product'] = dep_job.attributes['shipping_product']
yield job
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))
)
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for langpack files"
extra = {}
if 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if 'chunk_locales' in dep_job.attributes:
attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != 'release-snap-repackage' and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(
config, job_release_type=attributes.get('release-type'))
action_scope = get_beetmover_action_scope(
config, job_release_type=attributes.get('release-type'))
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcss(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for source file"
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'beetmover',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-S')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BMcss(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for source file"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", "")))
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'signed': dep_job.attributes.get('signed', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-BMcs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label':
label,
'description':
"Beetmover {} ".format(dep_job.task["metadata"]["description"]),
'worker-type':
'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-S)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'signed': dep_job.attributes.get('signed', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label':
label,
'description':
"{} Beetmover".format(dep_job.task["metadata"]["description"]),
'worker-type':
'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {
str(dep_job.kind): dep_job.label
}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
resolve_keyed_by(
job, 'worker-type', item_name=label, project=config.params['project']
)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-BMcs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-S)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'signed': dep_job.attributes.get('signed', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'signed': dep_job.attributes.get('signed', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': "Beetmover {} ".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def split_public_and_private(config, jobs):
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"])
partner, subpartner, _ = job["extra"]["repack_id"].split("/")
if partner_config[partner][subpartner].get("upload_to_candidates"):
# public
yield populate_scopes_and_worker_type(
config, job, public_bucket_scope, partner_public=True
)
else:
# private
yield populate_scopes_and_worker_type(
config, job, partner_bucket_scope, partner_public=False
)
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {dep_job.kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": "promote",
}
yield task
def split_public_and_private(config, jobs):
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
partner_bucket_scope = add_scope_prefix(config,
job['partner-bucket-scope'])
partner, subpartner, _ = job['extra']['repack_id'].split('/')
# public
if partner_config[partner][subpartner].get('upload_to_candidates'):
yield populate_scopes_and_worker_type(config,
job,
public_bucket_scope,
partner_public=True)
# private
yield populate_scopes_and_worker_type(config,
job,
partner_bucket_scope,
partner_public=False)
def make_beetmover_cdns_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = (
"Beetmover push to cdns for '{product}'".format(
product=job['product']
)
)
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'push'),
'shipping-product': job.get('shipping-product'),
'notifications': job.get('notifications'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
upstream_symbol = dep_job.task['extra']['treeherder']['symbol']
if 'build' in job['dependent-tasks']:
upstream_symbol = job['dependent-tasks']['build'].task['extra'][
'treeherder']['symbol']
treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_deps = job['dependent-tasks']
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
build_name = "nightly-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if 'partials-signing' in upstream_deps:
dependencies['partials-signing'] = upstream_deps[
'partials-signing']
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-R')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = dep_job.kind
if dependent_kind == 'repackage-signing-l10n':
dependent_kind = "repackage-signing"
dependencies = {dependent_kind: dep_job}
signing_name = "build-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
dependencies['signing'] = job['grandparent-tasks'][signing_name]
build_name = "build"
if job.get('locale'):
build_name = "unsigned-repack"
dependencies["build"] = job['grandparent-tasks'][build_name]
# repackage-l10n actually uses the repackage depname here
dependencies["repackage"] = job['grandparent-tasks']["repackage"]
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
if job.get('locale'):
repackage_signing_name = "repackage-signing-l10n"
dependencies["repackage-signing"] = job['grandparent-tasks'][
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', phase),
'shipping-product': job.get('shipping-product'),
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
if 'notifications' in job:
task['notifications'] = job['notifications']
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BMcs(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
if 'notifications' in job:
task['notifications'] = job['notifications']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-R)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
if 'docker-image' in dep_job.dependencies:
# macosx nightly builds depend on repackage which use in tree
# docker images and thus have two dependencies
# change the signing_dependencies to be use the ones in
docker_dependencies = {
"docker-image": dep_job.dependencies['docker-image']
}
dependencies.update(docker_dependencies)
signing_name = "build-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
signing_dependencies = {
signing_name: dep_job.dependencies[signing_name]
}
dependencies.update(signing_dependencies)
build_name = "build"
if job.get('locale'):
build_name = "unsigned-repack"
build_dependencies = {"build": dep_job.dependencies[build_name]}
dependencies.update(build_dependencies)
repackage_name = "repackage"
# repackage-l10n actually uses the repackage depname here
repackage_dependencies = {
"repackage": dep_job.dependencies[repackage_name]
}
dependencies.update(repackage_dependencies)
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
repackage_signing_deps = {
"repackage-signing":
dep_job.dependencies[repackage_signing_name]
}
dependencies.update(repackage_signing_deps)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'],
'BMcs'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')))
extra = {}
if 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'beetmover',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def split_public_and_private(config, jobs):
# we need to separate private vs public destinations because beetmover supports one
# in a single task. Only use a single task for each type though.
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
upstream_artifacts = job["primary-dependency"].release_artifacts
attribution_task_ref = "<{}>".format(job["primary-dependency"].label)
prefix = get_artifact_prefix(job["primary-dependency"])
artifacts = defaultdict(list)
for artifact in upstream_artifacts:
partner, sub_partner, platform, locale, _ = artifact.replace(
prefix + "/", "").split("/", 4)
destination = "private"
this_config = [
p for p in partner_config["configs"]
if (p["campaign"] == partner and p["content"] == sub_partner)
]
if this_config[0].get("upload_to_candidates"):
destination = "public"
artifacts[destination].append(
(artifact, partner, sub_partner, platform, locale))
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-partner")
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_bucket_scope = add_scope_prefix(config,
job["partner-bucket-scope"])
repl_dict = {
"build_number":
config.params["build_number"],
"release_partner_build_number":
config.params["release_partner_build_number"],
"version":
config.params["version"],
"partner":
"{partner}", # we'll replace these later, per artifact
"subpartner":
"{subpartner}",
"platform":
"{platform}",
"locale":
"{locale}",
}
for destination, destination_artifacts in artifacts.items():
this_job = deepcopy(job)
if destination == "public":
this_job["scopes"] = [public_bucket_scope, action_scope]
this_job["partner_public"] = True
else:
this_job["scopes"] = [partner_bucket_scope, action_scope]
this_job["partner_public"] = False
partner_path_key = "partner-{destination}-path".format(
destination=destination)
partner_path = this_job[partner_path_key].format(**repl_dict)
this_job.setdefault(
"worker",
{})["upstream-artifacts"] = generate_upstream_artifacts(
attribution_task_ref, destination_artifacts, partner_path)
yield this_job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc(BM-R)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job.get('label', "beetmover-{}".format(dep_job.label))
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
# macosx nightly builds depend on repackage which use in tree docker
# images and thus have two dependencies
# change the signing_dependencies to be use the ones in
docker_dependencies = {"docker-image":
dep_job.dependencies["docker-image"]
}
dependencies.update(docker_dependencies)
signing_dependencies = {"build-signing":
dep_job.dependencies["build-signing"]
}
dependencies.update(signing_dependencies)
repackage_dependencies = {"repackage":
dep_job.label
}
dependencies.update(repackage_dependencies)
build_dependencies = {"build":
dep_job.dependencies["build-signing"].replace("signing-", "build-")
}
dependencies.update(build_dependencies)
attributes = {
'nightly': dep_job.attributes.get('nightly', False),
'signed': dep_job.attributes.get('signed', False),
'build_platform': dep_job.attributes.get('build_platform'),
'build_type': dep_job.attributes.get('build_type'),
}
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM")
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != "release-snap-repackage" and len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies"
)
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"),
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if dep_job.attributes.get("locale"):
treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale"))
attributes["locale"] = dep_job.attributes.get("locale")
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-R')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_name = "build-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
signing_dependencies = {signing_name:
dep_job.dependencies[signing_name]
}
dependencies.update(signing_dependencies)
build_name = "build"
if job.get('locale'):
build_name = "unsigned-repack"
build_dependencies = {"build":
dep_job.dependencies[build_name]
}
dependencies.update(build_dependencies)
repackage_name = "repackage"
# repackage-l10n actually uses the repackage depname here
repackage_dependencies = {"repackage":
dep_job.dependencies[repackage_name]
}
dependencies.update(repackage_dependencies)
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
repackage_signing_deps = {"repackage-signing":
dep_job.dependencies[repackage_signing_name]
}
dependencies.update(repackage_signing_deps)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
task = {
'label': label,
'description': description,
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', phase),
'shipping-product': job.get('shipping-product'),
}
yield task