def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None): mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None) signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale) if build_platform.startswith('linux') or build_platform.startswith('macosx'): tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz' return { 'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format( signed_prefix, tarball_extension )}, 'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)}, } elif build_platform.startswith('win'): task_env = { 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, 'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)}, } # Stub installer is only generated on win32 if '32' in build_platform: task_env['SIGNED_SETUP_STUB'] = { 'task-reference': '{}setup-stub.exe'.format(signed_prefix), } return task_env raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def _generate_task_env(task, build_platform, build_task_ref, signing_task_ref, locale=None, project=None): mar_prefix = get_taskcluster_artifact_prefix( task, build_task_ref, postfix='host/bin/', locale=None ) signed_prefix = get_taskcluster_artifact_prefix(task, signing_task_ref, locale=locale) if build_platform.startswith('linux') or build_platform.startswith('macosx'): tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz' return { 'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format( signed_prefix, tarball_extension )}, 'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)}, } elif build_platform.startswith('win'): task_env = { 'SIGNED_ZIP': {'task-reference': '{}target.zip'.format(signed_prefix)}, 'SIGNED_SETUP': {'task-reference': '{}setup.exe'.format(signed_prefix)}, 'UNSIGNED_MAR': {'task-reference': '{}mar.exe'.format(mar_prefix)}, } no_stub = ("mozilla-esr60", "jamun") if project in no_stub: # Stub installer is only generated on win32 and not on esr task_env['NO_STUB_INSTALLER'] = '1' else: # Stub installer is only generated on win32 if '32' in build_platform: task_env['SIGNED_SETUP_STUB'] = { 'task-reference': '{}setup-stub.exe'.format(signed_prefix), } return task_env raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def add_command(config, tasks): for task in tasks: if not task["worker"].get("env"): task["worker"]["env"] = {} final_verify_configs = [] for upstream in task.get("dependencies", {}).keys(): if 'update-verify-config' in upstream: final_verify_configs.append("{}update-verify.cfg".format( get_taskcluster_artifact_prefix(task, "<{}>".format(upstream)))) task['run'] = { 'using': 'run-task', 'command': { 'task-reference': 'cd /builds/worker/checkouts/gecko && ' 'tools/update-verify/release/final-verification.sh ' + ' '.join(final_verify_configs), }, 'sparse-profile': 'update-verify', } for thing in ("BUILD_TOOLS_REPO", ): thing = "worker.env.{}".format(thing) resolve_keyed_by(task, thing, thing, **config.params) yield task
def _generate_task_env(build_platform, signing_task, signing_task_ref, partner): # Force private artifacts here, until we can populate our dependency map # with actual task definitions rather than labels. # (get_taskcluster_artifact_prefix requires the task definition to find # the artifact_prefix attribute). signed_prefix = get_taskcluster_artifact_prefix(signing_task, signing_task_ref, locale=partner, force_private=True) signed_prefix = signed_prefix.replace('public/build', 'releng/partner') if build_platform.startswith('macosx'): return { 'SIGNED_INPUT': { 'task-reference': '{}target.tar.gz'.format(signed_prefix) }, } elif build_platform.startswith('win'): task_env = { 'SIGNED_ZIP': { 'task-reference': '{}target.zip'.format(signed_prefix) }, 'SIGNED_SETUP': { 'task-reference': '{}setup.exe'.format(signed_prefix) }, } return task_env raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform))
def add_command(config, tasks): for task in tasks: total_chunks = task["extra"]["chunks"] for this_chunk in range(1, total_chunks+1): chunked = deepcopy(task) chunked["treeherder"]["symbol"] = add_suffix( chunked["treeherder"]["symbol"], this_chunk) chunked["label"] = "release-update-verify-{}-{}/{}".format( chunked["name"], this_chunk, total_chunks ) if not chunked["worker"].get("env"): chunked["worker"]["env"] = {} chunked["worker"]["command"] = [ "/bin/bash", "-c", "hg clone $BUILD_TOOLS_REPO tools && " + "tools/scripts/release/updates/chunked-verify.sh " + "UNUSED UNUSED {} {}".format( total_chunks, this_chunk, ) ] for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"): thing = "worker.env.{}".format(thing) resolve_keyed_by( chunked, thing, thing, **{ 'project': config.params['project'], 'release-type': config.params['release_type'], } ) update_verify_config = None for upstream in chunked.get("dependencies", {}).keys(): if 'update-verify-config' in upstream: update_verify_config = "{}update-verify.cfg".format( get_taskcluster_artifact_prefix(task, "<{}>".format(upstream)) ) if not update_verify_config: raise Exception("Couldn't find upate verify config") chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = { "task-reference": update_verify_config } yield chunked
def add_command(config, tasks): for task in tasks: if not task["worker"].get("env"): task["worker"]["env"] = {} final_verify_configs = [] for upstream in task.get("dependencies", {}).keys(): if 'update-verify-config' in upstream: final_verify_configs.append("{}update-verify.cfg".format( get_taskcluster_artifact_prefix("<{}>".format(upstream)))) task["worker"]["command"] = [ "/bin/bash", "-c", { "task-reference": "hg clone $BUILD_TOOLS_REPO tools && cd tools/release && " + "./final-verification.sh " + " ".join(final_verify_configs) } ] for thing in ("BUILD_TOOLS_REPO", ): thing = "worker.env.{}".format(thing) resolve_keyed_by(task, thing, thing, **config.params) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue signing_task = None for dependency in sorted(dependencies.keys()): if 'repackage-signing' in dependency: signing_task = dependency break signing_task_ref = '<{}>'.format(signing_task) extra = {'funsize': {'partials': list()}} update_number = 1 artifact_path = "{}{}".format( get_taskcluster_artifact_prefix(signing_task_ref, locale=locale), 'target.complete.mar') for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'task-reference': artifact_path }, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 cot = extra.setdefault('chainOfTrust', {}) cot.setdefault('inputs', {})['docker-image'] = { "task-reference": "<docker-image>" } mar_channel_id = None if config.params['project'] == 'mozilla-beta': if 'devedition' in label: mar_channel_id = 'firefox-mozilla-aurora' else: mar_channel_id = 'firefox-mozilla-beta' elif config.params['project'] == 'mozilla-release': mar_channel_id = 'firefox-mozilla-release' elif 'esr' in config.params['project']: mar_channel_id = 'firefox-mozilla-esr' worker = { 'artifacts': _generate_task_output_files(builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384', 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-3/datadog-api-key' } } if mar_channel_id: worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id level = config.params['level'] task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'scopes': [ 'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level, 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['primary-dependency'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) attributes['shipping_phase'] = job['shipping-phase'] build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue dep_task_ref = '<{}>'.format(dependent_kind) extra = {'funsize': {'partials': list()}} update_number = 1 artifact_path = "{}{}".format( get_taskcluster_artifact_prefix(dep_job, dep_task_ref, locale=locale), 'target.complete.mar') for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'task-reference': artifact_path }, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 mar_channel_id = None if config.params['project'] == 'mozilla-beta': if 'devedition' in label: mar_channel_id = 'firefox-mozilla-aurora' else: mar_channel_id = 'firefox-mozilla-beta' elif config.params['project'] == 'mozilla-release': mar_channel_id = 'firefox-mozilla-release' elif 'esr' in config.params['project']: mar_channel_id = 'firefox-mozilla-esr' level = config.params['level'] worker = { 'artifacts': _generate_task_output_files(dep_job, builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384', 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-{}/datadog-api-key'.format( level), } } if mar_channel_id: worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id if config.params.release_level() == 'staging': worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true' task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'scopes': [ 'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level ], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } # We only want caching on linux/windows due to bug 1436977 if level == 3 and any( [platform in dep_th_platform for platform in ['linux', 'windows']]): task['scopes'].append( 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue signing_task = None for dependency in sorted(dependencies.keys()): if 'repackage-signing' in dependency: signing_task = dependency break signing_task_ref = '<{}>'.format(signing_task) extra = {'funsize': {'partials': list()}} update_number = 1 artifact_path = "{}{}".format( get_taskcluster_artifact_prefix(signing_task_ref, locale=locale), 'target.complete.mar') for build in builds: extra['funsize']['partials'].append({ 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'task-reference': artifact_path }, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, }) update_number += 1 cot = extra.setdefault('chainOfTrust', {}) cot.setdefault('inputs', {})['docker-image'] = { "task-reference": "<docker-image>" } worker = { 'artifacts': _generate_task_output_files(builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600, 'chain-of-trust': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384' } } level = config.params['level'] task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue signing_task = None for dependency in sorted(dependencies.keys()): if 'repackage-signing' in dependency: signing_task = dependency break signing_task_ref = '<{}>'.format(signing_task) extra = {'funsize': {'partials': list()}} update_number = 1 artifact_path = "{}{}".format( get_taskcluster_artifact_prefix(signing_task_ref, locale=locale), 'target.complete.mar' ) for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': {'task-reference': artifact_path}, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build]['previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build]['previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 mar_channel_id = None if config.params['project'] == 'mozilla-beta': if 'devedition' in label: mar_channel_id = 'firefox-mozilla-aurora' else: mar_channel_id = 'firefox-mozilla-beta' elif config.params['project'] == 'mozilla-release': mar_channel_id = 'firefox-mozilla-release' elif 'esr' in config.params['project']: mar_channel_id = 'firefox-mozilla-esr' worker = { 'artifacts': _generate_task_output_files(builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': {'in-tree': 'funsize-update-generator'}, 'os': 'linux', 'max-run-time': 3600, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384', 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-3/datadog-api-key' } } if mar_channel_id: worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id level = config.params['level'] task = { 'label': label, 'description': "{} Partials".format( dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'scopes': [ 'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level, 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } yield task