def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds, designed for use as
multiple-dependent needs.
Required ``group-by-fn`` is used to define how we coalesce the
multiple deps together to pass to transforms, e.g. all kinds specified get
collapsed by platform with `platform`
Optional ``primary-dependency`` (ordered list or string) is used to determine
which upstream kind to inherit attrs from. See ``get_primary_dep``.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a job will be created. Alternatively there is
'not-for-build-platforms' kind configuration which will be consulted only after
'only-for-build-platforms' is checked (if present), and omit any jobs where the
build platform matches.
Optional ``job-template`` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
job_template = config.get('job-template')
for dep_tasks in group_tasks(config, loaded_tasks):
job = {'dependent-tasks': dep_tasks}
job['primary-dependency'] = get_primary_dep(config, dep_tasks)
if job_template:
job.update(copy.deepcopy(job_template))
# copy shipping_product from upstream
product = job['primary-dependency'].attributes.get(
'shipping_product',
job['primary-dependency'].task.get('shipping-product')
)
if product:
job.setdefault('shipping-product', product)
job.setdefault('attributes', {})['required_signoffs'] = sorted_unique_list(
*[task.attributes.get('required_signoffs', [])
for task in dep_tasks.values()
]
)
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
'symbol',
join_symbol(job.get('treeherder-group', 'ms'), locale or 'N'))
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
build_platform = attributes.get('build_platform')
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], build_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(
dep_job, config.kind)
is_nightly = job.get(
'nightly', # First check current job
dep_job.attributes.get(
'nightly', # Then dep job for 'nightly'
dep_job.attributes.get(
'shippable'))) # lastly dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
job.get('run-on-projects',
dep_job.attributes.get('run_on_projects')),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
treeherder = job.get('treeherder', {})
treeherder['symbol'] = join_symbol(job.get('treeherder-group', 'ms'),
locale or 'N')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "{}-{}".format(config.kind, dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['required_signoffs'] = sorted_unique_list(
attributes.get('required_signoffs', []),
job.pop('required_signoffs'))
attributes['shipping_phase'] = job['shipping-phase']
if locale:
attributes['locale'] = locale
balrog_platform = get_balrog_platform_name(dep_th_platform)
if config.kind == 'partials-signing':
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params['release_history'], balrog_platform,
locale)
else:
upstream_artifacts = generate_complete_artifacts(dep_job)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [
signing_cert_scope,
'project:releng:signing:format:autograph_hash_only_mar384'
]
if any("mar" in upstream_details["formats"]
for upstream_details in upstream_artifacts):
scopes.append('project:releng:signing:format:mar')
task = {
'label':
label,
'description':
"{} {}".format(dep_job.task["metadata"]["description"],
job['description-suffix']),
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'dependencies':
dependencies,
'attributes':
attributes,
'scopes':
scopes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", "{}-{}".format(config.kind, dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task
