def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(BM{})".format(attributes.get("l10n_chunk",
"")))
job["attributes"].update(copy_attributes_from_dependent_job(dep_job))
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"])
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
platform=job["attributes"]["build_platform"],
)
job["scopes"] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job["dependencies"] = {"langpack-copy": dep_job.label}
job["run-on-projects"] = job.get("run_on_projects",
dep_job.attributes["run_on_projects"])
job["treeherder"] = treeherder
job["shipping-phase"] = job.get("shipping-phase",
dep_job.attributes["shipping_phase"])
job["shipping-product"] = dep_job.attributes["shipping_product"]
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol', 'langpack(BM{})'.format(attributes.get('l10n_chunk',
'')))
job['attributes'].update(copy_attributes_from_dependent_job(dep_job))
job['attributes']['chunk_locales'] = dep_job.attributes[
'chunk_locales']
job['description'] = job['description'].format(
locales='/'.join(job['attributes']['chunk_locales']),
platform=job['attributes']['build_platform'])
job['scopes'] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job['dependencies'] = {dep_job.kind: dep_job.label}
job['run-on-projects'] = dep_job.attributes['run_on_projects']
job['treeherder'] = treeherder
job['shipping-phase'] = dep_job.attributes['shipping_phase']
job['shipping-product'] = dep_job.attributes['shipping_product']
yield job
def build_browsertime_task(config, tasks):
for task in tasks:
signing = task.pop("primary-dependency")
task.setdefault("dependencies", {})["signing"] = signing.label
build_type = task["attributes"]["build-type"]
abi = task["attributes"]["abi"]
apk = task["attributes"]["apk"]
test_name = task.pop("test-name")
task["name"] = "{}-{}-{}".format(task["name"], build_type, abi)
task["description"] = "{}-{}".format(build_type, abi)
for key in ("args", "treeherder.platform", "worker-type"):
resolve_keyed_by(task, key, item_name=task["name"], **{"abi": abi})
task["treeherder"] = inherit_treeherder_from_dep(task, signing)
extra_config = {
"installer_url": "<signing/{}>".format(apk),
"test_packages_url": "<geckoview-nightly/public/build/en-US/target.test_packages.json>",
}
env = task["worker"]["env"]
env["EXTRA_MOZHARNESS_CONFIG"] = {
"artifact-reference": json.dumps(extra_config, sort_keys=True)
}
env["GECKO_HEAD_REV"] = "default"
env["MOZILLA_BUILD_URL"] = {"artifact-reference": "<signing/{}>".format(apk)}
env["MOZHARNESS_URL"] = {
"artifact-reference": "<geckoview-nightly/public/build/en-US/mozharness.zip>"
}
env["TASKCLUSTER_WORKER_TYPE"] = task["worker-type"]
worker = task["worker"]
worker.setdefault("mounts", []).append(
{
"content": {
"url": "https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/scripts/tester/test-linux.sh"
},
"file": "./test-linux.sh",
}
)
task["run"]["command"].append("--test={}".format(test_name))
task["run"]["command"].extend(task.pop("args", []))
# Setup visual metrics
run_visual_metrics = task.pop("run-visual-metrics", False)
if run_visual_metrics:
task["run"]["command"].append("--browsertime-video")
task["run"]["command"].append("--browsertime-no-ffwindowrecorder")
task["attributes"]["run-visual-metrics"] = True
# taskcluster is merging task attributes with the default ones
# resulting the --cold extra option in the ytp warm tasks
if 'youtube-playback' in task["name"]:
task["run"]["command"].remove("--cold")
yield task
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
treeherder_symbol = join_symbol(treeherder_group, 'gp')
task["treeherder"]["symbol"] = treeherder_symbol
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))
)
label = job['label']
build_platform = attributes.get('build_platform')
description = "Beetmover submission of checksums for langpack files"
extra = {}
if 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith('beetmover'):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if 'chunk_locales' in dep_job.attributes:
attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
attributes.update(job.get('attributes', {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'beetmover',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep))
job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?")
job_symbol = task["treeherder"].pop("job-symbol")
full_symbol = join_symbol(job_group, job_symbol)
task["treeherder"]["symbol"] = full_symbol
yield task
def build_raptor_task(config, tasks):
for task in tasks:
signing = task.pop("primary-dependency")
build_type = task["attributes"]["build-type"]
abi = task["attributes"]["abi"]
apk = task["attributes"]["apk"]
test_name = task.pop("test-name")
task["name"] = "{}-{}-{}".format(task["name"], build_type, abi)
task["description"] = "{}: {}-{}".format(task["description"],
build_type, abi)
resolve_keyed_by(task,
"worker-type",
item_name=task["name"],
**{"abi": abi})
task["treeherder"] = inherit_treeherder_from_dep(task, signing)
task["treeherder"]["platform"] += "-{}".format(abi)
task["dependencies"]["signing"] = signing.label
extra_config = {
"installer_url":
"<signing/{}>".format(apk),
"test_packages_url":
"<geckoview-nightly/public/build/en-US/target.test_packages.json>",
}
env = task["worker"]["env"]
env["EXTRA_MOZHARNESS_CONFIG"] = {
"artifact-reference": json.dumps(extra_config, sort_keys=True)
}
env["GECKO_HEAD_REV"] = "default"
env["MOZILLA_BUILD_URL"] = {
"artifact-reference": "<signing/{}>".format(apk)
}
env["MOZHARNESS_URL"] = {
"artifact-reference":
"<geckoview-nightly/public/build/en-US/mozharness.zip>"
}
worker = task["worker"]
worker.setdefault("mounts", []).append({
"content": {
"url":
"https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/scripts/tester/test-linux.sh"
},
"file": "./test-linux.sh",
})
task["run"]["command"].append("--test={}".format(test_name))
task["run"]["command"].extend(task.pop("args", []))
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", "")))
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task['primary-dependency']
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get('enable-full-crashsymbols'):
logger.debug("Skipping upload symbols task for %s", task['label'])
continue
task['dependencies'] = {'build': dep.label}
task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[
'head_repository']
task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][
'SYMBOL_SECRET'].format(level=config.params['level'])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get('attributes', {}))
task['attributes'] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get('extra')['treeherder']
th_symbol = th.get('symbol')
th_groupsymbol = th.get('groupSymbol', '?')
# Disambiguate the treeherder symbol.
sym = 'Sym' + (th_symbol[1:]
if th_symbol.startswith('B') else th_symbol)
treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym))
task['treeherder'] = treeherder
if attributes.get('nightly'):
# For nightly builds, we want to run these tasks if the build is run.
task['run-on-projects'] = dep.attributes.get('run_on_projects')
elif attributes.get('shippable'):
# For shippable builds, we want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task['run-on-projects'] = dep.attributes.get('run_on_projects')
else:
# For other builds, these can be requested to upload to the try symbol sever.
task['run-on-projects'] = ['try']
# clear out the stuff that's not part of a task description
del task['primary-dependency']
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
'langpack(SnP{})'.format(job['attributes'].get('l10n_chunk', '')))
job['description'] = job['description'].format(locales='/'.join(
job['attributes']['chunk_locales']), )
job['dependencies'] = {dep_job.kind: dep_job.label}
job['treeherder'] = treeherder
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(SnP{})".format(job["attributes"].get("l10n_chunk", ""))
)
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
)
job["dependencies"] = {dep_job.kind: dep_job.label}
job["treeherder"] = treeherder
yield job
def build_pushapk_task(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task["dependencies"] = {"signing": dep.label}
task["name"] = dep.label[len(dep.kind) + 1 :]
task["attributes"] = dep.attributes.copy()
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
task["worker"]["upstream-artifacts"] = [
{
"taskId": {"task-reference": "<signing>"},
"taskType": "signing",
"paths": dep.attributes["apks"].values(),
}
]
task["worker"]["dep"] = config.params["level"] != "3"
yield task
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks.values():
if (
"update-verify-config" in dep.kind
or "update-verify-next-config" in dep.kind
):
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task["name"]]
total_chunks = task["extra"]["chunks"]
task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][
"channel"
]
task.setdefault("fetches", {})[config_task.label] = [
"update-verify.cfg",
]
task["treeherder"] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk
)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
"using": "run-task",
"cwd": "{checkout}",
"command": "tools/update-verify/scripts/chunked-verify.sh "
"{} {}".format(
total_chunks,
this_chunk,
),
"sparse-profile": "update-verify",
}
yield chunked
def define_signing_flags(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
# Current kind will be prepended later in the transform chain.
task["name"] = _get_dependent_job_name_without_its_kind(dep)
task["attributes"] = dep.attributes.copy()
task["attributes"]["signed"] = True
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
for key in ("worker-type", "worker.signing-type"):
resolve_keyed_by(
task,
key,
item_name=task["name"],
variant=task["attributes"]["build-type"],
level=config.params["level"],
)
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
yield task
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks:
if 'update-verify-config' in dep.kind or 'update-verify-next-config' in dep.kind:
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task['name']]
total_chunks = task["extra"]["chunks"]
task['worker'].setdefault(
'env', {})['CHANNEL'] = (config_task.task['extra']['channel'])
task.setdefault('fetches', {})[config_task.label] = [
"update-verify.cfg",
]
task['treeherder'] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
'using':
'run-task',
'cwd':
'{checkout}',
'command':
'tools/update-verify/scripts/chunked-verify.sh '
'{} {}'.format(
total_chunks,
this_chunk,
),
'sparse-profile':
'update-verify',
}
yield chunked
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task['extra']['treeherder']['symbol']
if 'build' in job['dependent-tasks']:
upstream_symbol = job['dependent-tasks']['build'].task['extra'][
'treeherder']['symbol']
treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR'))
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_deps = job['dependent-tasks']
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get('locale'):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if 'partials-signing' in upstream_deps:
dependencies['partials-signing'] = upstream_deps[
'partials-signing']
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': 'beetmover',
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
join_symbol(job.get('treeherder-group', 'ms'), locale or 'N'))
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
build_platform = attributes.get('build_platform')
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], build_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(
dep_job, config.kind)
is_nightly = job.get(
'nightly', # First check current job
dep_job.attributes.get(
'nightly', # Then dep job for 'nightly'
dep_job.attributes.get(
'shippable'))) # lastly dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
job.get('run-on-projects',
dep_job.attributes.get('run_on_projects')),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get("release_history"):
return
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault("symbol", "p(N)")
label = job.get("label", "partials-{}".format(dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get("locale")
if locale:
attributes["locale"] = locale
treeherder["symbol"] = "p({})".format(locale)
attributes["shipping_phase"] = job["shipping-phase"]
build_locale = locale or "en-US"
build_platform = attributes["build_platform"]
builds = get_builds(config.params["release_history"], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {"funsize": {"partials": list()}}
update_number = 1
locale_suffix = ""
if locale:
locale_suffix = "{}/".format(locale)
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
"locale": build_locale,
"from_mar": builds[build]["mar_url"],
"to_mar": {
"artifact-reference": artifact_path
},
"branch": config.params["project"],
"update_number": update_number,
"dest_mar": build,
}
if "product" in builds[build]:
partial_info["product"] = builds[build]["product"]
if "previousVersion" in builds[build]:
partial_info["previousVersion"] = builds[build][
"previousVersion"]
if "previousBuildNumber" in builds[build]:
partial_info["previousBuildNumber"] = builds[build][
"previousBuildNumber"]
extra["funsize"]["partials"].append(partial_info)
update_number += 1
level = config.params["level"]
worker = {
"artifacts":
_generate_task_output_files(dep_job, builds.keys(), locale),
"implementation":
"docker-worker",
"docker-image": {
"in-tree": "funsize-update-generator"
},
"os":
"linux",
"max-run-time":
3600 if "asan" in dep_job.label else 900,
"chain-of-trust":
True,
"taskcluster-proxy":
True,
"env": {
"SIGNING_CERT":
identify_desired_signing_keys(
config.params["project"],
config.params["release_product"]),
"EXTRA_PARAMS":
"--arch={}".format(architecture(build_platform)),
"MAR_CHANNEL_ID":
attributes["mar-channel-id"],
},
}
if config.params.release_level() == "staging":
worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true"
task = {
"label": label,
"description": "{} Partials".format(dep_job.description),
"worker-type": "b-linux",
"dependencies": dependencies,
"scopes": [],
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
"worker": worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 and any(
[build_platform.startswith(prefix)
for prefix in ["linux", "win"]]):
task["scopes"].append(
"auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
)
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task
def build_browsertime_task(config, tasks):
for task in tasks:
signing = task.pop("primary-dependency")
task.setdefault("dependencies", {})["signing"] = signing.label
build_type = task["attributes"]["build-type"]
abi = task["attributes"]["abi"]
apk = task["attributes"]["apk"]
for key in ("args", "treeherder.platform", "worker-type"):
resolve_keyed_by(task, key, item_name=task["name"], **{"abi": abi})
task["treeherder"] = inherit_treeherder_from_dep(task, signing)
test_name = task.pop("test-name")
platform = task["treeherder"]["platform"]
task_name = "{}-{}-{}-{}".format(platform, test_name, build_type, abi)
task["name"] = task_name
task["description"] = task_name
extra_config = {
"installer_url":
"<signing/{}>".format(apk),
"test_packages_url":
"<geckoview-nightly/public/build/en-US/target.test_packages.json>",
}
env = task["worker"]["env"]
env["EXTRA_MOZHARNESS_CONFIG"] = {
"artifact-reference": json.dumps(extra_config, sort_keys=True)
}
env["GECKO_HEAD_REV"] = "default"
env["MOZILLA_BUILD_URL"] = {
"artifact-reference": "<signing/{}>".format(apk)
}
env["MOZHARNESS_URL"] = {
"artifact-reference":
"<geckoview-nightly/public/build/en-US/mozharness.zip>"
}
env["TASKCLUSTER_WORKER_TYPE"] = task["worker-type"]
worker = task["worker"]
worker.setdefault("mounts", []).append({
"content": {
"url":
"https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/scripts/tester/test-linux.sh"
},
"file": "./test-linux.sh",
})
task["run"]["command"].append("--test={}".format(test_name))
task["run"]["command"].extend(task.pop("args", []))
# Setup treherder symbol
symbol = task.pop("subtest-symbol", None)
# taskcluster is merging task attributes with the default ones
# resulting the --cold extra option in the ytp warm tasks
if "youtube-playback" in task["name"]:
symbol = test_name.replace("youtube-playback-", "ytp-")
# Setup chimera for combined warm+cold testing
if task.pop("chimera", False):
task["run"]["command"].append("--chimera")
# Add '-c' to taskcluster symbol when running cold tests
elif "--cold" in task["run"]["command"]:
symbol += "-c"
# Setup visual metrics
run_visual_metrics = task.pop("run-visual-metrics", False)
if run_visual_metrics:
task["run"]["command"].append("--browsertime-video")
task["run"]["command"].append("--browsertime-visualmetrics")
task["run"]["command"].append("--browsertime-no-ffwindowrecorder")
# Build taskcluster group and symol
task["treeherder"]["symbol"] = "Btime(%s)" % symbol
task["name"] = (task["name"].replace(
"tp6m-", "tp6m-{}-".format(symbol)).replace("-hv", ""))
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['primary-dependency']
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
attributes['shipping_phase'] = job['shipping-phase']
build_locale = locale or 'en-US'
build_platform = attributes['build_platform']
builds = get_builds(config.params['release_history'], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {'funsize': {'partials': list()}}
update_number = 1
locale_suffix = ''
if locale:
locale_suffix = '{}/'.format(locale)
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {
'artifact-reference': artifact_path
},
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build][
'previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build][
'previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
level = config.params['level']
worker = {
'artifacts':
_generate_task_output_files(dep_job, builds.keys(), locale),
'implementation':
'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os':
'linux',
'max-run-time':
3600 if 'asan' in dep_job.label else 900,
'chain-of-trust':
True,
'taskcluster-proxy':
True,
'env': {
'SIGNING_CERT':
identify_desired_signing_keys(
config.params["project"],
config.params['release_product']),
'EXTRA_PARAMS':
'--arch={}'.format(architecture(build_platform)),
'MAR_CHANNEL_ID':
attributes['mar-channel-id']
}
}
if config.params.release_level() == 'staging':
worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true'
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'b-linux',
'dependencies':
dependencies,
'scopes': [],
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 \
and any([build_platform.startswith(prefix) for prefix in ['linux', 'win']]):
task['scopes'].append(
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
)
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
# job['primary-dependency'].task['payload']['command']
formats = ['autograph_authenticode']
else:
formats = ['autograph_gpg']
rev = attributes['openh264_rev']
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
_generate_treeherder_symbol(
dep_job.task.get('extra', {}).get('treeherder',
{}).get('symbol')))
task = {
'label': job['label'],
'description': description,
'worker-type': 'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': my_attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder
}
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = ("Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if "win" in build_platform:
# job['primary-dependency'].task['payload']['command']
formats = ["autograph_authenticode"]
else:
formats = ["autograph_gpg"]
rev = attributes["openh264_rev"]
upstream_artifacts = [{
"taskId": {
"task-reference": "<openh264>"
},
"taskType":
"build",
"paths": [
"private/openh264/openh264-{}-{}.zip".format(
build_platform, rev),
],
"formats":
formats,
}]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder",
{}).get("symbol")),
)
task = {
"label": job["label"],
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": my_attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task