def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] dependencies = {dep_job.attributes.get('kind'): dep_job.label} if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi': # repackage-signing can end up with multiple deps... raise NotImplementedError( "Can't repackage a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies # This is so we get the build task in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' locale = attributes.get('locale', job.get('locale')) if locale: attributes['locale'] = locale treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'Nr') else: treeherder.setdefault('symbol', 'Rpk') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') if config.kind == 'repackage-msi': treeherder['symbol'] = 'MSI({})'.format(locale or 'N') build_task = None signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'signing' in dependency: signing_task = dependency else: build_task = dependency _fetch_subst_locale = 'en-US' if locale: # XXXCallek: todo: rewrite dependency finding # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt' # We need this additional dependency to support finding the mar binary # Which is needed in order to generate a new complete.mar dependencies['build'] = "build-{}/opt".format( dependencies[build_task][13:dependencies[build_task].rfind('-')]) build_task = 'build' _fetch_subst_locale = locale level = config.params['level'] build_platform = attributes['build_platform'] use_stub = attributes.get('stub-installer') repackage_config = [] package_formats = job.get('package-formats') if use_stub and not repackage_signing_task: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ['installer-stub'] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), '_locale': _fetch_subst_locale, 'architecture': architecture(build_platform), 'version_display': config.params['version'], } # Allow us to replace args a well, but specifying things expanded in mozharness # Without breaking .format and without allowing unknown through substs.update({name: '{{{}}}'.format(name) for name in MOZHARNESS_EXPANSIONS}) command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } command['args'] = [ arg.format(**substs) for arg in command['args'] ] if 'installer' in format and 'aarch64' not in build_platform: command['args'].append('--use-upx') repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, } if locale: # Make sure we specify the locale-specific upload dir worker.setdefault('env', {}).update(LOCALE=locale) if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith(('linux', 'macosx')): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform) ) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(worker_type), repackage_config=repackage_config, locale=locale, ) description = ( "Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, build_task, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get('fetches')), 'release-artifacts': [artifact['name'] for artifact in worker['artifacts']] } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] dependencies = {dep_job.attributes.get('kind'): dep_job.label} if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi': # repackage-signing can end up with multiple deps... raise NotImplementedError( "Can't repackage a signing task with multiple dependencies") attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' locale = attributes.get('locale', job.get('locale')) if locale: attributes['locale'] = locale treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'Nr') else: treeherder.setdefault('symbol', 'Rpk') dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform') treeherder.setdefault('platform', dep_th_platform) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') if config.kind == 'repackage-msi': treeherder['symbol'] = 'MSI({})'.format(locale or 'N') signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'signing' in dependency: signing_task = dependency _fetch_subst_locale = 'en-US' if locale: _fetch_subst_locale = locale worker_type = job['worker-type'] build_platform = attributes['build_platform'] use_stub = attributes.get('stub-installer') repackage_config = [] package_formats = job.get('package-formats') if use_stub and not repackage_signing_task: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ['installer-stub'] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), '_locale': _fetch_subst_locale, 'architecture': architecture(build_platform), 'version_display': config.params['version'], 'mar-channel-id': attributes['mar-channel-id'], } # Allow us to replace args a well, but specifying things expanded in mozharness # Without breaking .format and without allowing unknown through substs.update({name: '{{{}}}'.format(name) for name in MOZHARNESS_EXPANSIONS}) command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } command['args'] = [ arg.format(**substs) for arg in command['args'] ] if 'installer' in format and 'aarch64' not in build_platform: command['args'].append('--use-upx') repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = job.get('worker', {}) worker.update({ 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, }) if locale: # Make sure we specify the locale-specific upload dir worker.setdefault('env', {}).update(LOCALE=locale) worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config=repackage_config, locale=locale, ) description = ( "Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get('fetches')), 'release-artifacts': [artifact['name'] for artifact in worker['artifacts']] } if build_platform.startswith('macosx'): task.setdefault('fetches', {}).setdefault('toolchain', []).extend([ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ]) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['primary-dependency'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) attributes['shipping_phase'] = job['shipping-phase'] build_locale = locale or 'en-US' builds = get_builds(config.params['release_history'], dep_th_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {'funsize': {'partials': list()}} update_number = 1 locale_suffix = '' if locale: locale_suffix = '{}/'.format(locale) artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'artifact-reference': artifact_path }, 'platform': get_balrog_platform_name(dep_th_platform), 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 mar_channel_id = None if config.params['project'] == 'mozilla-beta': if 'devedition' in label: mar_channel_id = 'firefox-mozilla-aurora' else: mar_channel_id = 'firefox-mozilla-beta' elif config.params['project'] == 'mozilla-release': mar_channel_id = 'firefox-mozilla-release' elif 'esr' in config.params['project']: mar_channel_id = 'firefox-mozilla-esr' level = config.params['level'] worker = { 'artifacts': _generate_task_output_files(dep_job, builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600 if 'asan' in dep_job.label else 600, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SHA1_SIGNING_CERT': 'nightly_sha1', 'SHA384_SIGNING_CERT': 'nightly_sha384', 'DATADOG_API_SECRET': 'project/releng/gecko/build/level-{}/datadog-api-key'.format( level), 'EXTRA_PARAMS': '--arch={}'.format(architecture(attributes['build_platform'])), } } if mar_channel_id: worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id if config.params.release_level() == 'staging': worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true' task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level, 'dependencies': dependencies, 'scopes': [ 'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level ], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 \ and any([platform in dep_th_platform for platform in ['linux', 'windows']]): task['scopes'].append( 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get('release_history'): return for job in jobs: dep_job = job['primary-dependency'] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault('symbol', 'p(N)') label = job.get('label', "partials-{}".format(dep_job.label)) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = "p({})".format(locale) attributes['shipping_phase'] = job['shipping-phase'] build_locale = locale or 'en-US' build_platform = attributes['build_platform'] builds = get_builds(config.params['release_history'], build_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {'funsize': {'partials': list()}} update_number = 1 locale_suffix = '' if locale: locale_suffix = '{}/'.format(locale) artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { 'locale': build_locale, 'from_mar': builds[build]['mar_url'], 'to_mar': { 'artifact-reference': artifact_path }, 'branch': config.params['project'], 'update_number': update_number, 'dest_mar': build, } if 'product' in builds[build]: partial_info['product'] = builds[build]['product'] if 'previousVersion' in builds[build]: partial_info['previousVersion'] = builds[build][ 'previousVersion'] if 'previousBuildNumber' in builds[build]: partial_info['previousBuildNumber'] = builds[build][ 'previousBuildNumber'] extra['funsize']['partials'].append(partial_info) update_number += 1 level = config.params['level'] worker = { 'artifacts': _generate_task_output_files(dep_job, builds.keys(), locale), 'implementation': 'docker-worker', 'docker-image': { 'in-tree': 'funsize-update-generator' }, 'os': 'linux', 'max-run-time': 3600 if 'asan' in dep_job.label else 900, 'chain-of-trust': True, 'taskcluster-proxy': True, 'env': { 'SIGNING_CERT': identify_desired_signing_keys( config.params["project"], config.params['release_product']), 'EXTRA_PARAMS': '--arch={}'.format(architecture(build_platform)), 'MAR_CHANNEL_ID': attributes['mar-channel-id'] } } if config.params.release_level() == 'staging': worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true' task = { 'label': label, 'description': "{} Partials".format(dep_job.task["metadata"]["description"]), 'worker-type': 'b-linux', 'dependencies': dependencies, 'scopes': [], 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, 'worker': worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 \ and any([build_platform.startswith(prefix) for prefix in ['linux', 'win']]): task['scopes'].append( 'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/' ) yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes["repackage_type"] = "repackage" locale = attributes.get("locale", job.get("locale")) if locale: attributes["locale"] = locale treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "Rpk") dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform") treeherder.setdefault("platform", dep_th_platform) treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") if config.kind == "repackage-msi": treeherder["symbol"] = "MSI({})".format(locale or "N") signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if "repackage-signing" in dependency: repackage_signing_task = dependency elif "signing" in dependency: signing_task = dependency _fetch_subst_locale = "en-US" if locale: _fetch_subst_locale = locale worker_type = job["worker-type"] build_platform = attributes["build_platform"] use_stub = attributes.get("stub-installer") repackage_config = [] package_formats = job.get("package-formats") if use_stub and not repackage_signing_task: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ["installer-stub"] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "_locale": _fetch_subst_locale, "architecture": architecture(build_platform), "version_display": config.params["version"], "mar-channel-id": attributes["mar-channel-id"], } # Allow us to replace args a well, but specifying things expanded in mozharness # Without breaking .format and without allowing unknown through substs.update({ name: "{{{}}}".format(name) for name in MOZHARNESS_EXPANSIONS }) command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } command["args"] = [arg.format(**substs) for arg in command["args"]] if "installer" in format and "aarch64" not in build_platform: command["args"].append("--use-upx") repackage_config.append(command) run = job.get("mozharness", {}) run.update({ "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, }) worker = job.get("worker", {}) worker.update({ "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, # Don't add generic artifact directory. "skip-artifacts": True, }) if locale: # Make sure we specify the locale-specific upload dir worker.setdefault("env", {})["LOCALE"] = locale worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config=repackage_config, locale=locale, ) description = ("Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "if-dependencies": [dep_job.kind], "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "optimization": dep_job.optimization, "treeherder": treeherder, "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get("fetches"), ), "release-artifacts": [artifact["name"] for artifact in worker["artifacts"]], } if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ]) yield task
def make_task_description(config, jobs): # If no balrog release history, then don't generate partials if not config.params.get("release_history"): return for job in jobs: dep_job = job["primary-dependency"] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault("symbol", "p(N)") label = job.get("label", "partials-{}".format(dep_job.label)) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get("locale") if locale: attributes["locale"] = locale treeherder["symbol"] = "p({})".format(locale) attributes["shipping_phase"] = job["shipping-phase"] build_locale = locale or "en-US" build_platform = attributes["build_platform"] builds = get_builds(config.params["release_history"], build_platform, build_locale) # If the list is empty there's no available history for this platform # and locale combination, so we can't build any partials. if not builds: continue extra = {"funsize": {"partials": list()}} update_number = 1 locale_suffix = "" if locale: locale_suffix = "{}/".format(locale) artifact_path = "<{}/{}/{}target.complete.mar>".format( dep_job.kind, get_artifact_prefix(dep_job), locale_suffix, ) for build in sorted(builds): partial_info = { "locale": build_locale, "from_mar": builds[build]["mar_url"], "to_mar": { "artifact-reference": artifact_path }, "branch": config.params["project"], "update_number": update_number, "dest_mar": build, } if "product" in builds[build]: partial_info["product"] = builds[build]["product"] if "previousVersion" in builds[build]: partial_info["previousVersion"] = builds[build][ "previousVersion"] if "previousBuildNumber" in builds[build]: partial_info["previousBuildNumber"] = builds[build][ "previousBuildNumber"] extra["funsize"]["partials"].append(partial_info) update_number += 1 level = config.params["level"] worker = { "artifacts": _generate_task_output_files(dep_job, builds.keys(), locale), "implementation": "docker-worker", "docker-image": { "in-tree": "funsize-update-generator" }, "os": "linux", "max-run-time": 3600 if "asan" in dep_job.label else 900, "chain-of-trust": True, "taskcluster-proxy": True, "env": { "SIGNING_CERT": identify_desired_signing_keys( config.params["project"], config.params["release_product"]), "EXTRA_PARAMS": "--arch={}".format(architecture(build_platform)), "MAR_CHANNEL_ID": attributes["mar-channel-id"], }, } if config.params.release_level() == "staging": worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true" task = { "label": label, "description": "{} Partials".format(dep_job.description), "worker-type": "b-linux", "dependencies": dependencies, "scopes": [], "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, "worker": worker, } # We only want caching on linux/windows due to bug 1436977 if int(level) == 3 and any( [build_platform.startswith(prefix) for prefix in ["linux", "win"]]): task["scopes"].append( "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" ) yield task