def generate_update_line(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
release_config = get_release_config(config)
for job in jobs:
config_file = job.pop('whats-new-config')
update_config = load_yaml(config_file)
product = job['shipping-product']
# XXX On ESR68, even though they get filtered, we're building balrog tasks with the new
# Fennec numbers. This solution is not ideal, but works at the moment.
version = FennecVersion.parse(release_config['appVersion'])
if product == 'devedition':
product = 'firefox'
job['worker']['update-line'] = {}
for blob_type, suffix in [('wnp', ''), ('no-wnp', '-No-WNP')]:
context = {
'release-type': config.params['release_type'],
'product': product,
'version': version,
'blob-type': blob_type,
'build-id': config.params['moz_build_date'],
}
job['worker']['update-line'][suffix] = generate_update_properties(
context, update_config)
yield job
def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_")
)
for this_chunk in range(1, total_chunks+1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] += str(this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools && " +
"hg up -r {} && cd .. && ".format(
release_tag,
) +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(chunked, thing, thing, **config.params)
yield chunked
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
total_chunks = task["extra"]["chunks"]
platform = task["attributes"]["build_platform"]
# We git rid of -devedition (if it exists here) because we're
# only using this in the buildername, and buildbot doesn't use
# -devedition.
platform = platform.replace('-devedition', '')
product = task["shipping-product"]
buildername = "release-{branch}_" + product + "_" + platform + \
"_update_verify"
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_"))
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["scopes"] = [
"project:releng:buildbot-bridge:builder-name:{}".format(
# In scopes, "branch" is called "project"
buildername.replace("branch", "project"))
]
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks)
chunked["run"]["buildername"] = buildername
chunked.setdefault("worker", {}).setdefault("properties", {})
chunked["worker"]["properties"]["script_repo_revision"] = \
release_tag
chunked["worker"]["properties"]["THIS_CHUNK"] = str(this_chunk)
chunked["worker"]["properties"]["TOTAL_CHUNKS"] = str(total_chunks)
for thing in ("CHANNEL", "VERIFY_CONFIG"):
thing = "worker.properties.{}".format(thing)
resolve_keyed_by(chunked, thing, thing, **config.params)
yield chunked
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job['shipping-product']
current_version = release_config['version']
bouncer_products_per_alias = job['bouncer-products-per-alias']
entries = {
bouncer_alias: craft_bouncer_product_name(
product, bouncer_product, current_version,
)
for bouncer_alias, bouncer_product in bouncer_products_per_alias.items()
}
partner_bouncer_products_per_alias = job.get('partner-bouncer-products-per-alias')
if partner_bouncer_products_per_alias:
partners = get_partners_to_be_published(config)
for partner, sub_config_name, _ in partners:
entries.update({
bouncer_alias.replace('PARTNER', '{}-{}'.format(partner, sub_config_name)):
craft_partner_bouncer_product_name(
product, bouncer_product, current_version, partner, sub_config_name)
for bouncer_alias, bouncer_product in partner_bouncer_products_per_alias.items()
})
return entries
def add_command_arguments(config, tasks):
release_config = get_release_config(config)
all_locales = set()
for partner_class in config.params['release_partner_config'].values():
for partner in partner_class.values():
for sub_partner in partner.values():
all_locales.update(sub_partner.get('locales', []))
for task in tasks:
# add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
task['run']['options'] = [
'version={}'.format(release_config['version']),
'build-number={}'.format(release_config['build_number']),
'platform={}'.format(
task['attributes']['build_platform'].split('-')[0]),
]
if task['extra']['limit-locales']:
for locale in all_locales:
task['run']['options'].append('limit-locale={}'.format(locale))
if 'partner' in config.kind and config.params['release_partners']:
for partner in config.params['release_partners']:
task['run']['options'].append('p={}'.format(partner))
# The upstream taskIds are stored a special environment variable, because we want to use
# task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
# blocks that. It's space-separated string of ids in the end.
task['worker']['env']['UPSTREAM_TASKIDS'] = {
'task-reference':
' '.join(['<{}>'.format(dep) for dep in task['dependencies']])
}
yield task
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
branch = config.params['project']
product = run['product']
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
})
if 'channels' in run:
release_props['channels'] = run['channels']
resolve_keyed_by(release_props, 'channels', 'channels',
**config.params)
if product in ('devedition', 'firefox'):
release_props['balrog_api_root'] = _get_balrog_api_root(branch)
worker['properties'].update(release_props)
# Setting script_repo_revision to the gecko revision doesn't work for
# jobs that clone build/tools or other repos instead of gecko.
if 'script_repo_revision' not in worker['properties']:
worker['properties']['script_repo_revision'] = revision
def format(config, tasks):
"""Apply format substitution to worker.env and worker.command."""
format_params = {
"release_config": get_release_config(config),
"config_params": config.params,
}
for task in tasks:
format_params["task"] = task
command = task.get("worker", {}).get("command", [])
task["worker"]["command"] = [x.format(**format_params) for x in command]
env = task.get("worker", {}).get("env", {})
for k in env.keys():
resolve_keyed_by(
env,
k,
"flatpak envs",
**{
"release-level": config.params.release_level(),
"project": config.params["project"],
}
)
task["worker"]["env"][k] = env[k].format(**format_params)
yield task
def format(config, tasks):
""" Apply format substitution to worker.env and worker.command.
"""
format_params = {
'release_config': get_release_config(config),
'config_params': config.params,
}
for task in tasks:
format_params['task'] = task
command = task.get('worker', {}).get('command', [])
task['worker']['command'] = [
x.format(**format_params) for x in command
]
env = task.get('worker', {}).get('env', {})
for k in env.keys():
resolve_keyed_by(
env, k, 'flatpak envs', **{
'release-level': config.params.release_level(),
'project': config.params['project']
})
task['worker']['env'][k] = env[k].format(**format_params)
yield task
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
real_task = deepcopy(task)
real_task.setdefault("worker", {}).setdefault("properties", {})
real_task["worker"]["properties"]["version"] = release_config["version"]
real_task["worker"]["properties"]["appVersion"] = release_config["appVersion"]
real_task["worker"]["properties"]["build_number"] = release_config["build_number"]
real_task["worker"]["properties"]["partial_versions"] = release_config.get(
"partial_versions", ""
)
for thing in ("generate_bz2_blob", "balrog_api_root", "channels", "repo_path"):
thing = "worker.properties.{}".format(thing)
resolve_keyed_by(real_task, thing, thing, **config.params)
# Non-RC builds from mozilla-release shouldn't use the beta channel.
if config.params.get('project') == 'mozilla-release':
if config.params.get('desktop_release_type') != "rc":
real_task["worker"]["properties"]["channels"] = \
real_task["worker"]["properties"]["channels"].replace("beta,", "")
yield real_task
def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_"))
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] += str(this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash", "-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools && " +
"hg up -r {} && cd .. && ".format(release_tag, ) +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(chunked, thing, thing, **config.params)
yield chunked
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config, force=True)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
branch = config.params['project']
buildername = worker['buildername']
underscore_version = release_props['version'].replace('.', '_')
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
'script_repo_revision': revision,
})
worker['properties'].update(release_props)
# scopes
worker['scopes'] = [
"project:releng:buildbot-bridge:builder-name:{}".format(buildername)
]
# routes
if run.get('routes'):
worker['routes'] = []
repl_dict = {
'branch': branch,
'build_number': str(release_props['build_number']),
'revision': revision,
'underscore_version': underscore_version,
}
for route in run['routes']:
route = route.format(**repl_dict)
worker['routes'].append(route)
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
branch = config.params['project']
product = run['product']
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
})
if 'channels' in run:
release_props['channels'] = run['channels']
resolve_keyed_by(release_props, 'channels', 'channels', **config.params)
if product in ('devedition', 'firefox'):
release_props['balrog_api_root'] = _get_balrog_api_root(branch)
if run.get('release-eta'):
# TODO Use same property name when we move away from BuildBot
release_props['schedule_at'] = run['release-eta']
worker['properties'].update(release_props)
# Setting script_repo_revision to the gecko revision doesn't work for
# jobs that clone build/tools or other repos instead of gecko.
if 'script_repo_revision' not in worker['properties']:
worker['properties']['script_repo_revision'] = revision
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job["shipping-product"]
current_version = release_config["version"]
bouncer_products = job["bouncer-products"]
partners = get_partners_to_be_published(config)
entries = {}
for partner, sub_config_name, platforms in partners:
platforms = [PARTNER_PLATFORMS_TO_BOUNCER[p] for p in platforms]
entries.update({
craft_partner_bouncer_product_name(product, bouncer_product,
current_version, partner,
sub_config_name):
{
"options": {
"add_locales":
False, # partners may use different sets of locales
"ssl_only": craft_ssl_only(bouncer_product),
},
"paths_per_bouncer_platform":
craft_paths_per_bouncer_platform(
product,
bouncer_product,
platforms,
current_version,
partner,
sub_config_name,
),
}
for bouncer_product in bouncer_products
})
return entries
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
real_task = deepcopy(task)
real_task.setdefault("worker", {}).setdefault("properties", {})
real_task["worker"]["properties"]["version"] = release_config["version"]
real_task["worker"]["properties"]["appVersion"] = release_config["appVersion"]
real_task["worker"]["properties"]["build_number"] = release_config["build_number"]
real_task["worker"]["properties"]["partial_versions"] = release_config.get(
"partial_versions", ""
)
for thing in ("generate_bz2_blob", "balrog_api_root", "channels", "repo_path"):
thing = "worker.properties.{}".format(thing)
resolve_keyed_by(real_task, thing, thing, **config.params)
# Non-RC builds from mozilla-release shouldn't use the beta channel.
if config.params.get('project') == 'mozilla-release':
if config.params.get('release_type') != "rc":
real_task["worker"]["properties"]["channels"] = \
real_task["worker"]["properties"]["channels"].replace("beta,", "")
yield real_task
def interpolate(config, jobs):
release_config = get_release_config(config)
for job in jobs:
mh_options = list(job["run"]["options"])
job["run"]["options"] = [
option.format(version=release_config["version"],
build_number=release_config["build_number"])
for option in mh_options
]
yield job
def bb_release_worker(config, worker, run):
# props
release_props = get_release_config(config, force=True)
repo_path = urlparse(config.params['head_repository']).path.lstrip('/')
revision = config.params['head_rev']
release_props.update({
'release_promotion': True,
'repo_path': repo_path,
'revision': revision,
'script_repo_revision': revision,
})
worker['properties'].update(release_props)
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job["shipping-product"]
bouncer_platforms = job["bouncer-platforms"]
current_version = release_config["version"]
current_build_number = release_config["build_number"]
bouncer_products = job["bouncer-products"]
previous_versions_string = release_config.get("partial_versions", None)
if previous_versions_string:
previous_versions = previous_versions_string.split(", ")
else:
logger.warn(
'No partials defined! Bouncer submission task won\'t send any \
partial-related entry for "{}"'.format(
job["name"]
)
)
bouncer_products = [
bouncer_product
for bouncer_product in bouncer_products
if "partial" not in bouncer_product
]
previous_versions = [None]
project = config.params["project"]
return {
craft_bouncer_product_name(
product,
bouncer_product,
current_version,
current_build_number,
previous_version,
): {
"options": {
"add_locales": True,
"ssl_only": craft_ssl_only(bouncer_product, project),
},
"paths_per_bouncer_platform": craft_paths_per_bouncer_platform(
product,
bouncer_product,
bouncer_platforms,
current_version,
current_build_number,
previous_version,
),
}
for bouncer_product in bouncer_products
for previous_version in previous_versions
}
def add_command(config, jobs):
release_config = get_release_config(config)
version = release_config["version"]
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
command = [
"cd", "/builds/worker/checkouts/gecko", "&&",
"./mach", "python",
"testing/mozharness/scripts/release/bouncer_check.py",
"--version={}".format(version),
]
job["run"]["command"] = command
yield job
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
release_config = get_release_config(config)
task_def['payload'] = {
'maxRunTime': worker['max-run-time'],
'upload_date': config.params['build_date'],
'upstreamArtifacts': worker['upstream-artifacts']
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
if release_config:
task_def['payload'].update(release_config)
def add_previous_versions(config, jobs):
release_config = get_release_config(config)
if not release_config.get("partial_versions"):
for job in jobs:
yield job
else:
extra_params = []
for partial in release_config["partial_versions"].split(","):
extra_params.append("--previous-version={}".format(partial.split("build")[0].strip()))
for job in jobs:
job["run"]["mach"].extend(extra_params)
yield job
def build_beetmover_payload(config, task, task_def):
worker = task['worker']
release_config = get_release_config(config)
task_def['payload'] = {
'maxRunTime': worker['max-run-time'],
'upload_date': config.params['build_date'],
'upstreamArtifacts': worker['upstream-artifacts']
}
if worker.get('locale'):
task_def['payload']['locale'] = worker['locale']
if release_config:
task_def['payload'].update(release_config)
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job['shipping-product']
current_version = release_config['version']
bouncer_products_per_alias = job['bouncer-products-per-alias']
return {
bouncer_alias: craft_bouncer_product_name(
product, bouncer_product, current_version,
)
for bouncer_alias, bouncer_product in bouncer_products_per_alias.items()
}
def add_previous_versions(config, jobs):
release_config = get_release_config(config)
if not release_config.get("partial_versions"):
for job in jobs:
yield job
else:
extra_params = []
for partial in release_config["partial_versions"].split(","):
extra_params.append("--previous-version={}".format(partial.split("build")[0].strip()))
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
job["run"]["command"].extend(extra_params)
yield job
def make_task_description(config, jobs):
release_config = get_release_config(config)
for job in jobs:
resolve_keyed_by(
job, 'worker-type', item_name=job['name'], project=config.params['project']
)
resolve_keyed_by(
job, 'scopes', item_name=job['name'], project=config.params['project']
)
job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
product=job['shipping-product'].capitalize(),
version=release_config['version'],
build_number=release_config['build_number']
)
yield job
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job['shipping-product']
release_type = config.params['release_type']
# The release_type is defined but may point to None.
if not release_type:
release_type = ''
current_version = release_config['version']
bouncer_products_per_alias = job['bouncer-products-per-alias']
return {
bouncer_alias: craft_bouncer_product_name(
product, bouncer_product, current_version,
)
for bouncer_alias, bouncer_product in bouncer_products_per_alias.items()
}
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by project, etc."""
fields = [
"run.config",
"run.product-field",
"run.extra-config",
]
release_config = get_release_config(config)
version = release_config["version"]
for job in jobs:
for field in fields:
resolve_keyed_by(item=job,
field=field,
item_name=job['name'],
**{
'project': config.params['project'],
'release-level':
config.params.release_level(),
'release-type': config.params['release_type'],
})
for cfg in job["run"]["config"]:
job["run"]["mach"].extend(["--config", cfg])
if config.kind == "cron-bouncer-check":
job["run"]["mach"].extend([
"--product-field={}".format(job["run"]["product-field"]),
"--products-url={}".format(job["run"]["products-url"]),
])
del job["run"]["product-field"]
del job["run"]["products-url"]
elif config.kind == "release-bouncer-check":
job["run"]["mach"].append("--version={}".format(version))
del job["run"]["config"]
if 'extra-config' in job['run']:
env = job['worker'].setdefault('env', {})
env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(
job['run']['extra-config'])
del job["run"]["extra-config"]
yield job
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by project, etc."""
fields = [
"run.config",
"run.product-field",
"run.extra-config",
]
release_config = get_release_config(config)
version = release_config["version"]
for job in jobs:
for field in fields:
resolve_keyed_by(item=job,
field=field,
item_name=job["name"],
**{
"project": config.params["project"],
"release-level":
config.params.release_level(),
"release-type": config.params["release_type"],
})
for cfg in job["run"]["config"]:
job["run"]["mach"].extend(["--config", cfg])
if config.kind == "cron-bouncer-check":
job["run"]["mach"].extend([
"--product-field={}".format(job["run"]["product-field"]),
"--products-url={}".format(job["run"]["products-url"]),
])
del job["run"]["product-field"]
del job["run"]["products-url"]
elif config.kind == "release-bouncer-check":
job["run"]["mach"].append("--version={}".format(version))
del job["run"]["config"]
if "extra-config" in job["run"]:
env = job["worker"].setdefault("env", {})
env["EXTRA_MOZHARNESS_CONFIG"] = six.ensure_text(
json.dumps(job["run"]["extra-config"], sort_keys=True))
del job["run"]["extra-config"]
yield job
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job['shipping-product']
bouncer_platforms = job['bouncer-platforms']
current_version = release_config['version']
current_build_number = release_config['build_number']
bouncer_products = job['bouncer-products']
previous_versions_string = release_config.get('partial_versions', None)
if previous_versions_string:
previous_versions = previous_versions_string.split(', ')
else:
logger.warn(
'No partials defined! Bouncer submission task won\'t send any \
partial-related entry for "{}"'.format(job['name']))
bouncer_products = [
bouncer_product for bouncer_product in bouncer_products
if 'partial' not in bouncer_product
]
previous_versions = [None]
project = config.params['project']
return {
craft_bouncer_product_name(product, bouncer_product, current_version,
current_build_number, previous_version):
{
'options': {
'add_locales': craft_add_locales(product),
'check_uptake': craft_check_uptake(bouncer_product),
'ssl_only': craft_ssl_only(bouncer_product, project),
},
'paths_per_bouncer_platform':
craft_paths_per_bouncer_platform(product, bouncer_product,
bouncer_platforms,
current_version,
current_build_number,
previous_version),
}
for bouncer_product in bouncer_products
for previous_version in previous_versions
}
def add_notifications(config, jobs):
release_config = get_release_config(config)
for job in jobs:
label = "{}-{}".format(config.kind, job["name"])
notifications = job.pop("notifications", None)
if notifications:
resolve_keyed_by(
notifications, "emails", label, project=config.params["project"]
)
emails = notifications["emails"]
format_kwargs = dict(
task=job,
config=config.__dict__,
release_config=release_config,
)
subject = titleformatter.format(notifications["subject"], **format_kwargs)
message = titleformatter.format(notifications["message"], **format_kwargs)
emails = [email.format(**format_kwargs) for email in emails]
# By default, we only send mail on success to avoid messages like 'blah is in the
# candidates dir' when cancelling graphs, dummy job failure, etc
status_types = notifications.get("status-types", ["on-completed"])
for s in status_types:
job.setdefault("routes", []).extend(
["notify.email.{}.{}".format(email, s) for email in emails]
)
# Customize the email subject to include release name and build number
job.setdefault("extra", {}).update(
{
"notify": {
"email": {
"subject": subject,
}
}
}
)
if message:
job["extra"]["notify"]["email"]["content"] = message
yield job
def add_notifications(config, jobs):
release_config = get_release_config(config)
for job in jobs:
label = '{}-{}'.format(config.kind, job['name'])
notifications = job.pop('notifications', None)
if notifications:
resolve_keyed_by(notifications,
'emails',
label,
project=config.params['project'])
emails = notifications['emails']
format_kwargs = dict(
task=job,
config=config.__dict__,
release_config=release_config,
)
subject = titleformatter.format(notifications['subject'],
**format_kwargs)
message = titleformatter.format(notifications['message'],
**format_kwargs)
emails = [email.format(**format_kwargs) for email in emails]
# By default, we only send mail on success to avoid messages like 'blah is in the
# candidates dir' when cancelling graphs, dummy job failure, etc
status_types = notifications.get('status-types', ['on-completed'])
for s in status_types:
job.setdefault('routes', []).extend([
'notify.email.{}.{}'.format(email, s) for email in emails
])
# Customize the email subject to include release name and build number
job.setdefault('extra', {}).update(
{'notify': {
'email': {
'subject': subject,
}
}})
if message:
job['extra']['notify']['email']['content'] = message
yield job
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_"))
if not task["worker"].get("env"):
task["worker"]["env"] = {}
task["worker"]["command"] = [
"/bin/bash", "-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools &&" +
"hg up -r {} && cd release && ".format(release_tag, ) +
"./final-verification.sh $FINAL_VERIFY_CONFIGS"
]
for thing in ("FINAL_VERIFY_CONFIGS", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)
yield task
def add_command_arguments(config, tasks):
release_config = get_release_config(config)
# staging releases - pass reduced set of locales to the repacking script
all_locales = set()
partner_config = get_partner_config_by_kind(config, config.kind)
for partner in partner_config.values():
for sub_partner in partner.values():
all_locales.update(sub_partner.get("locales", []))
for task in tasks:
# add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
if not task["attributes"]["build_platform"].endswith("-shippable"):
raise Exception(
"Unexpected partner repack platform: {}".format(
task["attributes"]["build_platform"], ), )
platform = task["attributes"]["build_platform"].partition(
"-shippable")[0]
task["run"]["options"] = [
"version={}".format(release_config["version"]),
"build-number={}".format(release_config["build_number"]),
"platform={}".format(platform),
]
if task["extra"]["limit-locales"]:
for locale in all_locales:
task["run"]["options"].append("limit-locale={}".format(locale))
if "partner" in config.kind and config.params["release_partners"]:
for partner in config.params["release_partners"]:
task["run"]["options"].append("partner={}".format(partner))
# The upstream taskIds are stored a special environment variable, because we want to use
# task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
# blocks that. It's space-separated string of ids in the end.
task["worker"]["env"]["UPSTREAM_TASKIDS"] = {
"task-reference":
" ".join(["<{}>".format(dep) for dep in task["dependencies"]])
}
# Forward the release type for bouncer product construction
task["worker"]["env"]["RELEASE_TYPE"] = config.params["release_type"]
yield task
def add_command_arguments(config, tasks):
release_config = get_release_config(config)
# staging releases - pass reduced set of locales to the repacking script
all_locales = set()
partner_config = get_partner_config_by_kind(config, config.kind)
for partner in partner_config.values():
for sub_partner in partner.values():
all_locales.update(sub_partner.get('locales', []))
for task in tasks:
# add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
if not task['attributes']['build_platform'].endswith('-shippable'):
raise Exception(
"Unexpected partner repack platform: {}".format(
task['attributes']['build_platform'], ), )
platform = task['attributes']['build_platform'].partition(
'-shippable')[0]
task['run']['options'] = [
'version={}'.format(release_config['version']),
'build-number={}'.format(release_config['build_number']),
'platform={}'.format(platform),
]
if task['extra']['limit-locales']:
for locale in all_locales:
task['run']['options'].append('limit-locale={}'.format(locale))
if 'partner' in config.kind and config.params['release_partners']:
for partner in config.params['release_partners']:
task['run']['options'].append('partner={}'.format(partner))
# The upstream taskIds are stored a special environment variable, because we want to use
# task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
# blocks that. It's space-separated string of ids in the end.
task['worker']['env']['UPSTREAM_TASKIDS'] = {
'task-reference':
' '.join(['<{}>'.format(dep) for dep in task['dependencies']])
}
# Forward the release type for bouncer product construction
task['worker']['env']['RELEASE_TYPE'] = config.params['release_type']
yield task
def make_task_description(config, jobs):
release_config = get_release_config(config)
for job in jobs:
resolve_keyed_by(job,
"worker-type",
item_name=job["name"],
**{"release-level": config.params.release_level()})
resolve_keyed_by(job,
"scopes",
item_name=job["name"],
**{"release-level": config.params.release_level()})
job["worker"][
"release-name"] = "{product}-{version}-build{build_number}".format(
product=job["shipping-product"].capitalize(),
version=release_config["version"],
build_number=release_config["build_number"],
)
yield job
def format(config, tasks):
""" Apply format substitution to worker.env and worker.command.
"""
format_params = {
'release_config': get_release_config(config),
'config_params': config.params,
}
for task in tasks:
format_params['task'] = task
command = task.get('worker', {}).get('command', [])
task['worker']['command'] = [x.format(**format_params) for x in command]
env = task.get('worker', {}).get('env', {})
for k in env.keys():
resolve_keyed_by(env, k, 'snap envs', project=config.params['project'])
task['worker']['env'][k] = env[k].format(**format_params)
resolve_keyed_by(task, 'scopes', 'snap scopes', project=config.params['project'])
yield task
def add_command(config, tasks):
for task in tasks:
release_config = get_release_config(config)
release_tag = "{}_{}_RELEASE_RUNTIME".format(
task["shipping-product"].upper(),
release_config["version"].replace(".", "_")
)
if not task["worker"].get("env"):
task["worker"]["env"] = {}
task["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && cd tools &&" +
"hg up -r {} && cd release && ".format(
release_tag,
) +
"./final-verification.sh $FINAL_VERIFY_CONFIGS"
]
for thing in ("FINAL_VERIFY_CONFIGS", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)
yield task