def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency attributes['repackage_type'] = 'repackage' level = config.params['level'] repack_id = job['extra']['repack_id'] repackage_config = [] for format in job.get('package-formats'): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), } command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, } if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith('macosx'): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform)) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"]), } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] dependencies = {dep_job.attributes.get('kind'): dep_job.label} if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi': # repackage-signing can end up with multiple deps... raise NotImplementedError( "Can't repackage a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies # This is so we get the build task in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' locale = attributes.get('locale', job.get('locale')) if locale: attributes['locale'] = locale treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'Nr') else: treeherder.setdefault('symbol', 'Rpk') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') if config.kind == 'repackage-msi': treeherder['symbol'] = 'MSI({})'.format(locale or 'N') build_task = None signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'signing' in dependency: signing_task = dependency else: build_task = dependency _fetch_subst_locale = 'en-US' if locale: # XXXCallek: todo: rewrite dependency finding # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt' # We need this additional dependency to support finding the mar binary # Which is needed in order to generate a new complete.mar dependencies['build'] = "build-{}/opt".format( dependencies[build_task][13:dependencies[build_task].rfind('-')]) build_task = 'build' _fetch_subst_locale = locale level = config.params['level'] build_platform = attributes['build_platform'] use_stub = attributes.get('stub-installer') repackage_config = [] package_formats = job.get('package-formats') if use_stub and not repackage_signing_task: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ['installer-stub'] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), '_locale': _fetch_subst_locale, 'architecture': architecture(build_platform), 'version_display': config.params['version'], } # Allow us to replace args a well, but specifying things expanded in mozharness # Without breaking .format and without allowing unknown through substs.update({name: '{{{}}}'.format(name) for name in MOZHARNESS_EXPANSIONS}) command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } command['args'] = [ arg.format(**substs) for arg in command['args'] ] if 'installer' in format and 'aarch64' not in build_platform: command['args'].append('--use-upx') repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, } if locale: # Make sure we specify the locale-specific upload dir worker.setdefault('env', {}).update(LOCALE=locale) if build_platform.startswith('win'): worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level run['use-magic-mh-args'] = False else: if build_platform.startswith(('linux', 'macosx')): worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level else: raise NotImplementedError( 'Unsupported build_platform: "{}"'.format(build_platform) ) run['tooltool-downloads'] = 'internal' worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(worker_type), repackage_config=repackage_config, locale=locale, ) description = ( "Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, build_task, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get('fetches')), 'release-artifacts': [artifact['name'] for artifact in worker['artifacts']] } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes["build_platform"] if job["build-platform"].startswith("win"): if dep_job.kind.endswith("signing"): continue if job["build-platform"].startswith("macosx"): if dep_job.kind.endswith("repack"): continue dependencies = {dep_job.attributes.get("kind"): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith("macosx") and dependency.endswith( "signing"): signing_task = dependency elif build_platform.startswith("win") and dependency.endswith( "repack"): signing_task = dependency attributes["repackage_type"] = "repackage" repack_id = job["extra"]["repack_id"] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split("/") repack_stub_installer = partner_config[partner][subpartner].get( "repack_stub_installer") if build_platform.startswith("win32") and repack_stub_installer: job["package-formats"].append("installer-stub") repackage_config = [] for format in job.get("package-formats"): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { "archive_format": archive_format(build_platform), "executable_extension": executable_extension(build_platform), } command["inputs"] = { name: filename.format(**substs) for name, filename in command["inputs"].items() } repackage_config.append(command) run = job.get("mozharness", {}) run.update({ "using": "mozharness", "script": "mozharness/scripts/repackage.py", "job-script": "taskcluster/scripts/builder/repackage.sh", "actions": ["setup", "repackage"], "extra-config": { "repackage_config": repackage_config, }, }) worker = { "chain-of-trust": True, "max-run-time": 7200 if build_platform.startswith("win") else 3600, "taskcluster-proxy": True if get_artifact_prefix(dep_job) else False, "env": { "REPACK_ID": repack_id, }, # Don't add generic artifact directory. "skip-artifacts": True, } worker_type = "b-linux" worker["docker-image"] = {"in-tree": "debian10-amd64-build"} worker["artifacts"] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job["extra"]["repack_id"], build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) task = { "label": job["label"], "description": description, "worker-type": worker_type, "dependencies": dependencies, "attributes": attributes, "scopes": ["queue:get-artifact:releng/partner/*"], "run-on-projects": dep_job.attributes.get("run_on_projects"), "routes": job.get("routes", []), "extra": job.get("extra", {}), "worker": worker, "run": run, "fetches": _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer, ), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get("priority"): task["priority"] = job["priority"] if build_platform.startswith("macosx"): task.setdefault("fetches", {}).setdefault("toolchain", []).extend([ "linux64-libdmg", "linux64-hfsplus", "linux64-node", ]) yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) build_platform = attributes['build_platform'] if job['build-platform'].startswith('win'): if dep_job.kind.endswith('signing'): continue if job['build-platform'].startswith('macosx'): if dep_job.kind.endswith('repack'): continue dependencies = {dep_job.attributes.get('kind'): dep_job.label} dependencies.update(dep_job.dependencies) signing_task = None for dependency in dependencies.keys(): if build_platform.startswith('macosx') and dependency.endswith( 'signing'): signing_task = dependency elif build_platform.startswith('win') and dependency.endswith( 'repack'): signing_task = dependency attributes['repackage_type'] = 'repackage' repack_id = job['extra']['repack_id'] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: job['package-formats'].append('installer-stub') repackage_config = [] for format in job.get('package-formats'): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), } command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-config': { 'repackage_config': repackage_config, }, }) worker = { 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, 'taskcluster-proxy': True if get_artifact_prefix(dep_job) else False, 'env': { 'REPACK_ID': repack_id, }, # Don't add generic artifact directory. 'skip-artifacts': True, } worker_type = 'b-linux' worker['docker-image'] = {"in-tree": "debian8-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config, partner=repack_id, ) description = ("Repackaging for repack_id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=job['extra']['repack_id'], build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'scopes': ['queue:get-artifact:releng/partner/*'], 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config( dep_job, build_platform, signing_task, partner=repack_id, project=config.params["project"], repack_stub_installer=repack_stub_installer), } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] if build_platform.startswith('macosx'): task.setdefault('fetches', {}).setdefault('toolchain', []).extend([ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ]) yield task
def make_job_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] dependencies = {dep_job.attributes.get('kind'): dep_job.label} if len(dep_job.dependencies ) > 1 and not config.kind == 'repackage-msi': # repackage-signing can end up with multiple deps... raise NotImplementedError( "Can't repackage a signing task with multiple dependencies") attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' locale = attributes.get('locale', job.get('locale')) if locale: attributes['locale'] = locale treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'Nr') else: treeherder.setdefault('symbol', 'Rpk') dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform') treeherder.setdefault('platform', dep_th_platform) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') if config.kind == 'repackage-msi': treeherder['symbol'] = 'MSI({})'.format(locale or 'N') signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): if 'repackage-signing' in dependency: repackage_signing_task = dependency elif 'signing' in dependency: signing_task = dependency _fetch_subst_locale = 'en-US' if locale: _fetch_subst_locale = locale worker_type = job['worker-type'] build_platform = attributes['build_platform'] use_stub = attributes.get('stub-installer') repackage_config = [] package_formats = job.get('package-formats') if use_stub and not repackage_signing_task: # if repackage_signing_task doesn't exists, generate the stub installer package_formats += ['installer-stub'] for format in package_formats: command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), 'executable_extension': executable_extension(build_platform), '_locale': _fetch_subst_locale, 'architecture': architecture(build_platform), 'version_display': config.params['version'], } # Allow us to replace args a well, but specifying things expanded in mozharness # Without breaking .format and without allowing unknown through substs.update({ name: '{{{}}}'.format(name) for name in MOZHARNESS_EXPANSIONS }) command['inputs'] = { name: filename.format(**substs) for name, filename in command['inputs'].items() } command['args'] = [arg.format(**substs) for arg in command['args']] if 'installer' in format and 'aarch64' not in build_platform: command['args'].append('--use-upx') repackage_config.append(command) run = job.get('mozharness', {}) run.update({ 'using': 'mozharness', 'script': 'mozharness/scripts/repackage.py', 'job-script': 'taskcluster/scripts/builder/repackage.sh', 'actions': ['setup', 'repackage'], 'extra-workspace-cache-key': 'repackage', 'extra-config': { 'repackage_config': repackage_config, }, }) worker = job.get('worker', {}) worker.update({ 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, }) if locale: # Make sure we specify the locale-specific upload dir worker.setdefault('env', {}).update(LOCALE=locale) worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config=repackage_config, locale=locale, ) description = ("Repackaging for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) task = { 'label': job['label'], 'description': description, 'worker-type': worker_type, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'routes': job.get('routes', []), 'extra': job.get('extra', {}), 'worker': worker, 'run': run, 'fetches': _generate_download_config(dep_job, build_platform, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], existing_fetch=job.get('fetches')), 'release-artifacts': [artifact['name'] for artifact in worker['artifacts']] } if build_platform.startswith('macosx'): task['toolchains'] = [ 'linux64-libdmg', 'linux64-hfsplus', 'linux64-node', ] yield task