def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
return {
'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
signed_prefix, tarball_extension
)},
'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
}
elif build_platform.startswith('win'):
task_env = {
'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)},
}
# Stub installer is only generated on win32
if '32' in build_platform:
task_env['SIGNED_SETUP_STUB'] = {
'task-reference': '{}setup-stub.exe'.format(signed_prefix),
}
return task_env
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None,
project=None):
mar_prefix = get_taskcluster_artifact_prefix(
task, build_task_ref, postfix='host/bin/', locale=None
)
signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale)
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
return {
'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
signed_prefix, tarball_extension
)},
'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
}
elif build_platform.startswith('win'):
task_env = {
'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)},
'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)},
'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)},
}
no_stub = ("mozilla-esr60", "jamun")
if project in no_stub:
# Stub installer is only generated on win32 and not on esr
task_env['NO_STUB_INSTALLER'] = '1'
else:
# Stub installer is only generated on win32
if '32' in build_platform:
task_env['SIGNED_SETUP_STUB'] = {
'task-reference': '{}setup-stub.exe'.format(signed_prefix),
}
return task_env
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def add_command(config, tasks):
for task in tasks:
if not task["worker"].get("env"):
task["worker"]["env"] = {}
final_verify_configs = []
for upstream in task.get("dependencies", {}).keys():
if 'update-verify-config' in upstream:
final_verify_configs.append("{}update-verify.cfg".format(
get_taskcluster_artifact_prefix(task,
"<{}>".format(upstream))))
task['run'] = {
'using': 'run-task',
'command': {
'task-reference':
'cd /builds/worker/checkouts/gecko && '
'tools/update-verify/release/final-verification.sh ' +
' '.join(final_verify_configs),
},
'sparse-profile': 'update-verify',
}
for thing in ("BUILD_TOOLS_REPO", ):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)
yield task
def _generate_task_env(build_platform, signing_task, signing_task_ref,
partner):
# Force private artifacts here, until we can populate our dependency map
# with actual task definitions rather than labels.
# (get_taskcluster_artifact_prefix requires the task definition to find
# the artifact_prefix attribute).
signed_prefix = get_taskcluster_artifact_prefix(signing_task,
signing_task_ref,
locale=partner,
force_private=True)
signed_prefix = signed_prefix.replace('public/build', 'releng/partner')
if build_platform.startswith('macosx'):
return {
'SIGNED_INPUT': {
'task-reference': '{}target.tar.gz'.format(signed_prefix)
},
}
elif build_platform.startswith('win'):
task_env = {
'SIGNED_ZIP': {
'task-reference': '{}target.zip'.format(signed_prefix)
},
'SIGNED_SETUP': {
'task-reference': '{}setup.exe'.format(signed_prefix)
},
}
return task_env
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform))
def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
for this_chunk in range(1, total_chunks+1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && " +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(
chunked, thing, thing,
**{
'project': config.params['project'],
'release-type': config.params['release_type'],
}
)
update_verify_config = None
for upstream in chunked.get("dependencies", {}).keys():
if 'update-verify-config' in upstream:
update_verify_config = "{}update-verify.cfg".format(
get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
)
if not update_verify_config:
raise Exception("Couldn't find upate verify config")
chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = {
"task-reference": update_verify_config
}
yield chunked
def add_command(config, tasks):
for task in tasks:
if not task["worker"].get("env"):
task["worker"]["env"] = {}
final_verify_configs = []
for upstream in task.get("dependencies", {}).keys():
if 'update-verify-config' in upstream:
final_verify_configs.append("{}update-verify.cfg".format(
get_taskcluster_artifact_prefix("<{}>".format(upstream))))
task["worker"]["command"] = [
"/bin/bash", "-c", {
"task-reference":
"hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " +
"./final-verification.sh " + " ".join(final_verify_configs)
}
]
for thing in ("BUILD_TOOLS_REPO", ):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(task, thing, thing, **config.params)
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
signing_task = None
for dependency in sorted(dependencies.keys()):
if 'repackage-signing' in dependency:
signing_task = dependency
break
signing_task_ref = '<{}>'.format(signing_task)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar')
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {
'task-reference': artifact_path
},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build][
'previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build][
'previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
cot = extra.setdefault('chainOfTrust', {})
cot.setdefault('inputs', {})['docker-image'] = {
"task-reference": "<docker-image>"
}
mar_channel_id = None
if config.params['project'] == 'mozilla-beta':
if 'devedition' in label:
mar_channel_id = 'firefox-mozilla-aurora'
else:
mar_channel_id = 'firefox-mozilla-beta'
elif config.params['project'] == 'mozilla-release':
mar_channel_id = 'firefox-mozilla-release'
elif 'esr' in config.params['project']:
mar_channel_id = 'firefox-mozilla-esr'
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'taskcluster-proxy': True,
'env': {
'SHA1_SIGNING_CERT':
'nightly_sha1',
'SHA384_SIGNING_CERT':
'nightly_sha384',
'DATADOG_API_SECRET':
'project/releng/gecko/build/level-3/datadog-api-key'
}
}
if mar_channel_id:
worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
level = config.params['level']
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies':
dependencies,
'scopes': [
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key'
% level,
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
],
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
attributes['shipping_phase'] = job['shipping-phase']
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
dep_task_ref = '<{}>'.format(dependent_kind)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(dep_job,
dep_task_ref,
locale=locale),
'target.complete.mar')
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {
'task-reference': artifact_path
},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build][
'previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build][
'previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
mar_channel_id = None
if config.params['project'] == 'mozilla-beta':
if 'devedition' in label:
mar_channel_id = 'firefox-mozilla-aurora'
else:
mar_channel_id = 'firefox-mozilla-beta'
elif config.params['project'] == 'mozilla-release':
mar_channel_id = 'firefox-mozilla-release'
elif 'esr' in config.params['project']:
mar_channel_id = 'firefox-mozilla-esr'
level = config.params['level']
worker = {
'artifacts':
_generate_task_output_files(dep_job, builds.keys(), locale),
'implementation':
'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os':
'linux',
'max-run-time':
3600,
'chain-of-trust':
True,
'taskcluster-proxy':
True,
'env': {
'SHA1_SIGNING_CERT':
'nightly_sha1',
'SHA384_SIGNING_CERT':
'nightly_sha384',
'DATADOG_API_SECRET':
'project/releng/gecko/build/level-{}/datadog-api-key'.format(
level),
}
}
if mar_channel_id:
worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
if config.params.release_level() == 'staging':
worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true'
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies':
dependencies,
'scopes': [
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key'
% level
],
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
# We only want caching on linux/windows due to bug 1436977
if level == 3 and any(
[platform in dep_th_platform
for platform in ['linux', 'windows']]):
task['scopes'].append(
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
)
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
signing_task = None
for dependency in sorted(dependencies.keys()):
if 'repackage-signing' in dependency:
signing_task = dependency
break
signing_task_ref = '<{}>'.format(signing_task)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar')
for build in builds:
extra['funsize']['partials'].append({
'locale':
build_locale,
'from_mar':
builds[build]['mar_url'],
'to_mar': {
'task-reference': artifact_path
},
'platform':
get_balrog_platform_name(dep_th_platform),
'branch':
config.params['project'],
'update_number':
update_number,
'dest_mar':
build,
})
update_number += 1
cot = extra.setdefault('chainOfTrust', {})
cot.setdefault('inputs', {})['docker-image'] = {
"task-reference": "<docker-image>"
}
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'env': {
'SHA1_SIGNING_CERT': 'nightly_sha1',
'SHA384_SIGNING_CERT': 'nightly_sha384'
}
}
level = config.params['level']
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
signing_task = None
for dependency in sorted(dependencies.keys()):
if 'repackage-signing' in dependency:
signing_task = dependency
break
signing_task_ref = '<{}>'.format(signing_task)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar'
)
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {'task-reference': artifact_path},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build]['previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
mar_channel_id = None
if config.params['project'] == 'mozilla-beta':
if 'devedition' in label:
mar_channel_id = 'firefox-mozilla-aurora'
else:
mar_channel_id = 'firefox-mozilla-beta'
elif config.params['project'] == 'mozilla-release':
mar_channel_id = 'firefox-mozilla-release'
elif 'esr' in config.params['project']:
mar_channel_id = 'firefox-mozilla-esr'
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {'in-tree': 'funsize-update-generator'},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'taskcluster-proxy': True,
'env': {
'SHA1_SIGNING_CERT': 'nightly_sha1',
'SHA384_SIGNING_CERT': 'nightly_sha384',
'DATADOG_API_SECRET': 'project/releng/gecko/build/level-3/datadog-api-key'
}
}
if mar_channel_id:
worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
level = config.params['level']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies': dependencies,
'scopes': [
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level,
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
],
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
'worker': worker,
}
yield task