def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_")
)
for this_chunk in range(1, total_chunks+1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] += str(this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools && " +
"hg up -r {} && cd .. && ".format(
release_tag,
) +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(chunked, thing, thing, **config.params)
yield chunked
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
branch = config.params['project']
product = run['product']
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
})
if 'channels' in run:
release_props['channels'] = run['channels']
resolve_keyed_by(release_props, 'channels', 'channels', **config.params)
if product in ('devedition', 'firefox'):
release_props['balrog_api_root'] = _get_balrog_api_root(branch)
if run.get('release-eta'):
# TODO Use same property name when we move away from BuildBot
release_props['schedule_at'] = run['release-eta']
worker['properties'].update(release_props)
# Setting script_repo_revision to the gecko revision doesn't work for
# jobs that clone build/tools or other repos instead of gecko.
if 'script_repo_revision' not in worker['properties']:
worker['properties']['script_repo_revision'] = revision
def set_tier(config, tests):
"""Set the tier based on policy for all test descriptions that do not
specify a tier otherwise."""
for test in tests:
if 'tier' in test:
resolve_keyed_by(test, 'tier', item_name=test['test-name'])
# only override if not set for the test
if 'tier' not in test or test['tier'] == 'default':
if test['test-platform'] in ['linux32/opt',
'linux32/debug',
'linux32-nightly/opt',
'linux64/opt',
'linux64-nightly/opt',
'linux64/debug',
'linux64-pgo/opt',
'linux64-asan/opt',
'android-4.3-arm7-api-15/opt',
'android-4.3-arm7-api-15/debug',
'android-4.2-x86/opt']:
test['tier'] = 1
elif test['worker-implementation'] == 'native-engine':
test['tier'] = 3
else:
test['tier'] = 2
yield test
def test_nested(self):
x = {
'by-foo': {
'F1': {
'by-bar': {
'B1': 11,
'B2': 12,
},
},
'F2': 20,
'default': 0,
},
}
self.assertEqual(
resolve_keyed_by({'x': x}, 'x', 'x', foo='F1', bar='B1'),
{'x': 11})
self.assertEqual(
resolve_keyed_by({'x': x}, 'x', 'x', foo='F1', bar='B2'),
{'x': 12})
self.assertEqual(
resolve_keyed_by({'x': x}, 'x', 'x', foo='F2'),
{'x': 20})
self.assertEqual(
resolve_keyed_by({'x': x}, 'x', 'x', foo='F99', bar='B1'),
{'x': 0})
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
real_task = deepcopy(task)
real_task.setdefault("worker", {}).setdefault("properties", {})
real_task["worker"]["properties"]["version"] = release_config["version"]
real_task["worker"]["properties"]["appVersion"] = release_config["appVersion"]
real_task["worker"]["properties"]["build_number"] = release_config["build_number"]
real_task["worker"]["properties"]["partial_versions"] = release_config.get(
"partial_versions", ""
)
for thing in ("generate_bz2_blob", "balrog_api_root", "channels", "repo_path"):
thing = "worker.properties.{}".format(thing)
resolve_keyed_by(real_task, thing, thing, **config.params)
# Non-RC builds from mozilla-release shouldn't use the beta channel.
if config.params.get('project') == 'mozilla-release':
if config.params.get('desktop_release_type') != "rc":
real_task["worker"]["properties"]["channels"] = \
real_task["worker"]["properties"]["channels"].replace("beta,", "")
yield real_task
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
"locales-file",
"chunks",
"worker-type",
"description",
"run-time",
"tooltool",
"env",
"ignore-locales",
"mozharness.config",
"mozharness.options",
"mozharness.actions",
"mozharness.script",
"treeherder.tier",
"treeherder.platform",
"index.product",
"index.job-name",
"when.files-changed",
]
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
for field in fields:
resolve_keyed_by(item=job, field=field, item_name=job['name'])
yield job
def handle_platform(config, jobs):
"""
Handle the 'platform' property, setting up treeherder context as well as
try-related attributes.
"""
fields = [
'worker-type',
'worker',
]
for job in jobs:
platform = job['platform']
for field in fields:
resolve_keyed_by(job, field, item_name=job['name'])
if 'treeherder' in job:
job['treeherder']['platform'] = platform
build_platform, build_type = platform.split('/')
job['attributes'].update({
'build_platform': build_platform,
'build_type': build_type,
})
del job['platform']
yield job
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
"mozharness.config",
]
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
for field in fields:
resolve_keyed_by(item=job, field=field, item_name="?")
yield job
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
"toolchains"
]
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
for field in fields:
resolve_keyed_by(item=job, field=field, item_name=job['name'])
yield job
def set_worker_data(config, jobs):
for job in jobs:
worker = job['worker']
env = worker.setdefault('env', {})
resolve_keyed_by(
env, 'PACKAGE_NAME', item_name=job['name'],
project=config.params['project']
)
yield job
def build_raptor_task(config, tasks):
for task in tasks:
signing = task.pop("primary-dependency")
build_type = task["attributes"]["build-type"]
abi = task["attributes"]["abi"]
apk = task["attributes"]["apk"]
test_name = task.pop("test-name")
task["name"] = "{}-{}-{}".format(task["name"], build_type, abi)
task["description"] = "{}: {}-{}".format(task["description"],
build_type, abi)
resolve_keyed_by(task,
"worker-type",
item_name=task["name"],
**{"abi": abi})
task["treeherder"] = inherit_treeherder_from_dep(task, signing)
task["treeherder"]["platform"] += "-{}".format(abi)
task["dependencies"]["signing"] = signing.label
extra_config = {
"installer_url":
"<signing/{}>".format(apk),
"test_packages_url":
"<geckoview-nightly/public/build/en-US/target.test_packages.json>",
}
env = task["worker"]["env"]
env["EXTRA_MOZHARNESS_CONFIG"] = {
"artifact-reference": json.dumps(extra_config, sort_keys=True)
}
env["GECKO_HEAD_REV"] = "default"
env["MOZILLA_BUILD_URL"] = {
"artifact-reference": "<signing/{}>".format(apk)
}
env["MOZHARNESS_URL"] = {
"artifact-reference":
"<geckoview-nightly/public/build/en-US/mozharness.zip>"
}
worker = task["worker"]
worker.setdefault("mounts", []).append({
"content": {
"url":
"https://hg.mozilla.org/mozilla-central/raw-file/default/taskcluster/scripts/tester/test-linux.sh"
},
"file": "./test-linux.sh",
})
task["run"]["command"].append("--test={}".format(test_name))
task["run"]["command"].extend(task.pop("args", []))
yield task
def resolve_keys(config, tasks):
for task in tasks:
resolve_keyed_by(
task,
"treeherder.platform",
item_name=task["name"],
**{
"build-type": task["attributes"]["build-type"],
"level": config.params["level"],
},
)
yield task
def load_jobs(params):
with open(os.path.join(GOANNA, '.cron.yml'), 'rb') as f:
cron_yml = yaml.load(f)
schema.validate(cron_yml)
# resolve keyed_by fields in each job
jobs = cron_yml['jobs']
for job in jobs:
resolve_keyed_by(job, 'when', 'Cron job ' + job['name'],
project=params['project'])
return {j['name']: j for j in jobs}
def resolve_properties(config, tasks):
for task in tasks:
for property in ("REPACK_MANIFESTS_URL", ):
property = "worker.env.{}".format(property)
resolve_keyed_by(task, property, property, **config.params)
if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
task.setdefault('scopes', []).append(
'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'
.format(**config.params))
yield task
def resolve_keys(config, tasks):
for task in tasks:
resolve_keyed_by(
task,
"treeherder.job-symbol",
item_name=task["name"],
**{
'build-type': task["attributes"]["build-type"],
'level': config.params["level"],
}
)
yield task
def update_channel(config, jobs):
for job in jobs:
resolve_keyed_by(
job, 'run.update-channel', item_name=job['name'],
**{
'release-type': config.params['release_type'],
}
)
update_channel = job['run'].pop('update-channel', None)
if update_channel:
job['run'].setdefault('extra-config', {})['update_channel'] = update_channel
yield job
def get_partner_url_config(parameters, graph_config):
partner_url_config = deepcopy(graph_config['partner-urls'])
substitutions = {
'release-product': parameters['release_product'],
'release-level': release_level(parameters['project']),
'release-type': parameters["release_type"]
}
resolve_keyed_by(partner_url_config, 'release-eme-free-repack',
'eme-free manifest_url', **substitutions)
resolve_keyed_by(partner_url_config, 'release-partner-repack',
'partner manifest url', **substitutions)
return partner_url_config
def mozconfig(config, jobs):
for job in jobs:
resolve_keyed_by(
job, 'run.mozconfig-variant', item_name=job['name'],
**{
'release-type': config.params['release_type'],
}
)
mozconfig_variant = job['run'].pop('mozconfig-variant', None)
if mozconfig_variant:
job['run'].setdefault('extra-config', {})['mozconfig_variant'] = mozconfig_variant
yield job
def resolve_keys(config, jobs):
for job in jobs:
for field in ('worker-type', 'attributes.artifact_map'):
resolve_keyed_by(job,
field,
item_name=job['label'],
**{
'release-level':
config.params.release_level(),
'project': config.params['project']
})
yield job
def resolve_keys(config, tasks):
for task in tasks:
for key in ("worker.channel", "worker.commit", "worker.dep"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
'build-type':
task["attributes"]["build-type"],
'level': config.params["level"],
})
yield task
def should_run(job, params):
run_on_projects = job.get('run-on-projects', ['all'])
if not match_run_on_projects(params['project'], run_on_projects):
return False
# Resolve when key here, so we don't require it before we know that we
# actually want to run on this branch.
resolve_keyed_by(job, 'when', 'Cron job ' + job['name'],
project=params['project'])
if not any(match_utc(params, hour=sched.get('hour'), minute=sched.get('minute'))
for sched in job.get('when', [])):
return False
return True
def stub_installer(config, jobs):
for job in jobs:
resolve_keyed_by(
job, 'stub-installer', item_name=job['name'], project=config.params['project']
)
job.setdefault('attributes', {})
if job.get('stub-installer'):
job['attributes']['stub-installer'] = job['stub-installer']
job['worker']['env'].update({"USE_STUB_INSTALLER": "1"})
if 'stub-installer' in job:
del job['stub-installer']
yield job
def resolve_keys(config, tasks):
for task in tasks:
for key in ("run-on-tasks-for", ):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
'build-type':
task["attributes"]["build-type"],
'level': config.params["level"],
'tasks-for': config.params["tasks_for"],
})
yield task
def beetmover_task(config, tasks):
for task in tasks:
task["worker"]["max-run-time"] = 600
task["worker"]["version"] = get_version()
task["description"] = task["description"].format(
task["attributes"]["buildconfig"]["name"])
resolve_keyed_by(task,
"worker.bucket",
item_name=task["name"],
**{
"level": config.params["level"],
})
yield task
def resolve_keys(config, tasks):
for task in tasks:
for key in ("worker.github-project", "worker.is-prerelease", "worker.release-name"):
resolve_keyed_by(
task,
key,
item_name=task["name"],
**{
'build-type': task["attributes"]["build-type"],
'level': config.params["level"],
}
)
yield task
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by project, etc."""
fields = [
"run.config",
]
for job in jobs:
job = copy.deepcopy(job)
for field in fields:
resolve_keyed_by(item=job,
field=field,
item_name=job['name'],
project=config.params['project'])
yield job
def resolve_keys(config, tasks):
for task in tasks:
for key in ("index", "worker-type", "worker.signing-type",
"treeherder.job-symbol"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
"build-type":
task["attributes"]["build-type"],
"level": config.params["level"],
})
yield task
def resolve_keys(config, tasks):
for task in tasks:
for key in ("worker.push",):
resolve_keyed_by(
task,
key,
item_name=task["name"],
**{
"build-type": task["attributes"]["build-type"],
"level": config.params["level"],
}
)
yield task
def resolve_keys(config, jobs):
for job in jobs:
for field in ("worker-type", "attributes.artifact_map"):
resolve_keyed_by(job,
field,
item_name=job["label"],
**{
"release-level":
config.params.release_level(),
"release-type": config.params["release_type"],
"project": config.params["project"],
})
yield job
def mozconfig(config, jobs):
for job in jobs:
resolve_keyed_by(job,
"run.mozconfig-variant",
item_name=job["name"],
**{
"release-type": config.params["release_type"],
})
mozconfig_variant = job["run"].pop("mozconfig-variant", None)
if mozconfig_variant:
job["run"].setdefault("extra-config",
{})["mozconfig_variant"] = mozconfig_variant
yield job
def resolve_keys(config, tasks):
for task in tasks:
for key in ("worker.channel", "worker.dep", "worker.certificate-alias",
"routes"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
"build-type":
task["attributes"]["build-type"],
"level": config.params["level"],
})
yield task
def handle_keyed_by_app(config, tests):
fields = [
'activity',
'binary-path',
'cold',
'max-run-time',
'run-on-projects',
'target',
]
for test in tests:
for field in fields:
resolve_keyed_by(test, field, item_name=test['test-name'])
yield test
def resolve_keys(config, tasks):
for task in tasks:
for key in ("run-on-tasks-for", "signing-format", "notify"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
"build-type":
task["attributes"]["build-type"],
"level": config.params["level"],
"tasks-for": config.params["tasks_for"],
})
yield task
def resolve_keys(config, tasks):
for task in tasks:
for key in ("treeherder.job-symbol", "worker.bucket",
"worker.beetmover-application-name"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
"build-type":
task["attributes"]["build-type"],
"level": config.params["level"],
})
yield task
def resolve_keys(config, tasks):
for task in tasks:
for key in ("index", "worker-type", "worker.signing-type",
"signing-format"):
resolve_keyed_by(task,
key,
item_name=task["name"],
**{
'build-type':
task["attributes"]["build-type"],
'level': config.params["level"],
})
yield task
def set_tier(config, tests):
"""Set the tier based on policy for all test descriptions that do not
specify a tier otherwise."""
for test in tests:
if 'tier' in test:
resolve_keyed_by(test, 'tier', item_name=test['test-name'])
# only override if not set for the test
if 'tier' not in test or test['tier'] == 'default':
if test['test-platform'] in ['linux32/opt',
'linux32/debug',
'linux32-nightly/opt',
'linux32-devedition/opt',
'linux32-stylo-disabled/debug',
'linux32-stylo-disabled/opt',
'linux64/opt',
'linux64-nightly/opt',
'linux64/debug',
'linux64-pgo/opt',
'linux64-devedition/opt',
'linux64-asan/opt',
'linux64-stylo-disabled/debug',
'linux64-stylo-disabled/opt',
'windows7-32/debug',
'windows7-32/opt',
'windows7-32-pgo/opt',
'windows7-32-devedition/opt',
'windows7-32-nightly/opt',
'windows7-32-stylo-disabled/debug',
'windows7-32-stylo-disabled/opt',
'windows10-64/debug',
'windows10-64/opt',
'windows10-64-pgo/opt',
'windows10-64-devedition/opt',
'windows10-64-nightly/opt',
'windows10-64-stylo-disabled/debug',
'windows10-64-stylo-disabled/opt',
'macosx64/opt',
'macosx64/debug',
'macosx64-nightly/opt',
'macosx64-devedition/opt',
'macosx64-stylo-disabled/debug',
'macosx64-stylo-disabled/opt',
'android-4.3-arm7-api-16/opt',
'android-4.3-arm7-api-16/debug',
'android-4.2-x86/opt']:
test['tier'] = 1
else:
test['tier'] = 2
yield test
def set_tier(config, tests):
"""Set the tier based on policy for all test descriptions that do not
specify a tier otherwise."""
for test in tests:
if 'tier' in test:
resolve_keyed_by(test, 'tier', item_name=test['test-name'])
# only override if not set for the test
if 'tier' not in test or test['tier'] == 'default':
if test['test-platform'] in ['linux32/opt',
'linux32/debug',
'linux32-nightly/opt',
'linux32-devedition/opt',
'linux32-stylo-disabled/debug',
'linux32-stylo-disabled/opt',
'linux64/opt',
'linux64-nightly/opt',
'linux64/debug',
'linux64-pgo/opt',
'linux64-devedition/opt',
'linux64-asan/opt',
'linux64-stylo-disabled/debug',
'linux64-stylo-disabled/opt',
'windows7-32/debug',
'windows7-32/opt',
'windows7-32-pgo/opt',
'windows7-32-devedition/opt',
'windows7-32-nightly/opt',
'windows7-32-stylo-disabled/debug',
'windows7-32-stylo-disabled/opt',
'windows10-64/debug',
'windows10-64/opt',
'windows10-64-pgo/opt',
'windows10-64-devedition/opt',
'windows10-64-nightly/opt',
'windows10-64-stylo-disabled/debug',
'windows10-64-stylo-disabled/opt',
'macosx64/opt',
'macosx64/debug',
'macosx64-nightly/opt',
'macosx64-devedition/opt',
'macosx64-stylo-disabled/debug',
'macosx64-stylo-disabled/opt',
'android-4.3-arm7-api-16/opt',
'android-4.3-arm7-api-16/debug',
'android-4.2-x86/opt']:
test['tier'] = 1
else:
test['tier'] = 2
yield test
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}".format(dep_job.label)
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {
str(dep_job.kind): dep_job.label
}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
phase = get_phase(config)
resolve_keyed_by(
job, 'worker-type', item_name=label, project=config.params['project']
)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': phase,
}
yield task
def resolve_keys(config, tasks):
for task in tasks:
resolve_keyed_by(task,
"routes",
item_name=task["name"],
**{"tasks-for": config.params["tasks_for"]})
# TODO: Bug 1637695 - temp solution to unblock local building of
# appplication-services. Once we switch to new indexes, we should clean this up
if task['name'] in TOOLCHAIN_OLD_INDEX.keys():
sha = git_sha_for_directory("libs")
routes = task['routes']
routes.append(TOOLCHAIN_OLD_INDEX[task['name']].format(sha=sha))
yield task
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
resolve_keyed_by(
job, 'worker.google-play-track', item_name=job['name'],
project=config.params['project']
)
resolve_keyed_by(
job, 'worker.commit', item_name=job['name'],
project=config.params['project']
)
resolve_keyed_by(
job, 'worker.rollout-percentage', item_name=job['name'],
project=config.params['project']
)
job['scopes'] = [get_push_apk_scope(config)]
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
yield job
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
# Use the rc-google-play-track and rc-rollout-percentage in RC relpro flavors
if config.params['release_type'] == 'rc':
job['worker']['google-play-track'] = job['worker']['rc-google-play-track']
job['worker']['rollout-percentage'] = job['worker']['rc-rollout-percentage']
resolve_keyed_by(
job, 'worker.google-play-track', item_name=job['name'],
project=config.params['project']
)
resolve_keyed_by(
job, 'worker.commit', item_name=job['name'],
project=config.params['project']
)
resolve_keyed_by(
job, 'worker.rollout-percentage', item_name=job['name'],
project=config.params['project']
)
job['scopes'] = [get_push_apk_scope(config)]
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
yield job
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
job['worker']['upstream-artifacts'] = generate_upstream_artifacts(
job, job['dependencies'])
resolve_keyed_by(job,
'worker.google-play-track',
item_name=job['name'],
**{'release-type': config.params['release_type']})
resolve_keyed_by(job,
'worker.commit',
item_name=job['name'],
**{'release-level': config.params.release_level()})
resolve_keyed_by(job,
'worker.rollout-percentage',
item_name=job['name'],
**{'release-type': config.params['release_type']})
job['scopes'] = [get_push_apk_scope(config)]
resolve_keyed_by(job,
'worker-type',
item_name=job['name'],
**{'release-level': config.params.release_level()})
yield job
def handle_keyed_by(config, tasks):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
'worker.push',
'worker.bump-files',
'worker-type',
]
for task in tasks:
for field in fields:
resolve_keyed_by(task,
field,
item_name=task['name'],
project=config.params['project'])
yield task
def test_match_extra_value(self):
self.assertEqual(
resolve_keyed_by(
{'f': {'by-foo': {'x': 10, 'y': 20}}},
'f', 'n',
foo='y'),
{'f': 20})
def make_task_description(config, jobs):
release_config = get_release_config(config)
for job in jobs:
resolve_keyed_by(
job, 'worker-type', item_name=job['name'], project=config.params['project']
)
resolve_keyed_by(
job, 'scopes', item_name=job['name'], project=config.params['project']
)
job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
product=job['shipping-product'].capitalize(),
version=release_config['version'],
build_number=release_config['build_number']
)
yield job
def set_target(config, tests):
for test in tests:
build_platform = test['build-platform']
target = None
if 'target' in test:
resolve_keyed_by(test, 'target', item_name=test['test-name'])
target = test['target']
if not target:
if build_platform.startswith('macosx'):
target = 'target.dmg'
elif build_platform.startswith('android'):
target = 'target.apk'
elif build_platform.startswith('win'):
target = 'target.zip'
else:
target = 'target.tar.bz2'
test['mozharness']['build-artifact-name'] = 'public/build/' + target
yield test
def handle_keyed_by(config, tests):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
'instance-size',
'docker-image',
'max-run-time',
'chunks',
'e10s',
'suite',
'run-on-projects',
'os-groups',
'mozharness.chunked',
'mozharness.config',
'mozharness.extra-options',
]
for test in tests:
for field in fields:
resolve_keyed_by(test, field, item_name=test['test-name'],
project=config.params['project'])
yield test
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
job['worker']['payload'] = {} if 'human' in job['worker-type'] else {
'image': 'ubuntu:16.10',
'command': [
'/bin/bash',
'-c',
'echo "Dummy task while while bug 1351664 is implemented"'
],
'maxRunTime': 600,
}
yield job
def make_beetmover_cdns_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = (
"Beetmover push to cdns for '{product}'".format(
product=job['product']
)
)
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': job['worker-type'],
'scopes': [bucket_scope, action_scope],
'product': job['product'],
'dependencies': job['dependencies'],
'attributes': job.get('attributes', {}),
'run-on-projects': job.get('run-on-projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'push'),
'shipping-product': job.get('shipping-product'),
'notifications': job.get('notifications'),
}
yield task
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_")
)
if not task["worker"].get("env"):
task["worker"]["env"] = {}
task["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools &&" +
"hg up -r {} && cd release && ".format(
release_tag,
) +
"./final-verification.sh $FINAL_VERIFY_CONFIGS"
]
for thing in ("FINAL_VERIFY_CONFIGS", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)
yield task
def format(config, tasks):
""" Apply format substitution to worker.env and worker.command.
"""
format_params = {
'release_config': get_release_config(config),
'config_params': config.params,
}
for task in tasks:
format_params['task'] = task
command = task.get('worker', {}).get('command', [])
task['worker']['command'] = [x.format(**format_params) for x in command]
env = task.get('worker', {}).get('env', {})
for k in env.keys():
resolve_keyed_by(env, k, 'snap envs', project=config.params['project'])
task['worker']['env'][k] = env[k].format(**format_params)
resolve_keyed_by(task, 'scopes', 'snap scopes', project=config.params['project'])
yield task
def handle_platform(config, jobs):
"""
Handle the 'platform' property, setting up treeherder context as well as
try-related attributes.
"""
fields = [
'worker-type',
'worker',
]
for job in jobs:
platform = job['platform']
for field in fields:
resolve_keyed_by(job, field, item_name=job['name'])
if 'treeherder' in job:
job['treeherder']['platform'] = platform
if job.pop('require-build'):
add_build_dependency(config, job)
del job['platform']
yield job
def test_no_by(self):
self.assertEqual(
resolve_keyed_by({'x': 10}, 'z', 'n'),
{'x': 10})
def test_no_by_not_dict(self):
self.assertEqual(
resolve_keyed_by({'x': 10}, 'x.y', 'n'),
{'x': 10})
def test_no_by_dotted(self):
self.assertEqual(
resolve_keyed_by({'x': {'y': 10}}, 'x.z', 'n'),
{'x': {'y': 10}})
def test_no_by_not_by(self):
self.assertEqual(
resolve_keyed_by({'x': {'a': 10}}, 'x', 'n'),
{'x': {'a': 10}})
def test_no_by_empty_dict(self):
self.assertEqual(
resolve_keyed_by({'x': {}}, 'x', 'n'),
{'x': {}})
def test_no_by_not_only_by(self):
self.assertEqual(
resolve_keyed_by({'x': {'by-y': True, 'a': 10}}, 'x', 'n'),
{'x': {'by-y': True, 'a': 10}})
def test_match_nested_exact(self):
self.assertEqual(
resolve_keyed_by(
{'f': 'shoes', 'x': {'y': {'by-f': {'shoes': 'feet', 'gloves': 'hands'}}}},
'x.y', 'n'),
{'f': 'shoes', 'x': {'y': 'feet'}})
def test_match_regexp(self):
self.assertEqual(
resolve_keyed_by(
{'f': 'shoes', 'x': {'by-f': {'s?[hH]oes?': 'feet', 'gloves': 'hands'}}},
'x', 'n'),
{'f': 'shoes', 'x': 'feet'})
