def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'c-Up(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') attributes = copy_attributes_from_dependent_job(dep_job) treeherder_job_symbol = dep_job.task['extra']['treeherder']['symbol'] treeherder['symbol'] = replace_group(treeherder_job_symbol, 'c-Up') if dep_job.attributes.get('locale'): attributes['locale'] = dep_job.attributes.get('locale') label = job['label'] description = ("Balrog submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": ["public/manifest.json"], }] task = { 'label': label, 'description': description, 'worker-type': 'balrog', 'worker': { 'implementation': 'balrog', 'upstream-artifacts': upstream_artifacts, 'balrog-action': 'submit-locale', 'suffixes': ['', '-No-WNP'] if job.get('update-no-wnp') else [''], }, 'dependencies': { 'beetmover': dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job.get('shipping-phase', 'promote'), 'shipping-product': job.get('shipping-product'), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {dep_job.kind: dep_job.label} # XXX release snap-repackage has a variable number of dependencies, depending on how many # "post-beetmover-dummy" jobs there are in the graph. if dep_job.kind != 'release-snap-repackage' and len( dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope( config, job_release_type=attributes.get('release-type')) action_scope = get_beetmover_action_scope( config, job_release_type=attributes.get('release-type')) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], 'shipping-product': job.get('shipping-product'), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) upstream_symbol = dep_job.task['extra']['treeherder']['symbol'] if 'build' in job['dependent-tasks']: upstream_symbol = job['dependent-tasks']['build'].task['extra'][ 'treeherder']['symbol'] treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) upstream_deps = job['dependent-tasks'] signing_name = "build-signing" build_name = "build" repackage_name = "repackage" repackage_signing_name = "repackage-signing" msi_signing_name = "repackage-signing-msi" mar_signing_name = "mar-signing" if job.get('locale'): signing_name = "nightly-l10n-signing" build_name = "nightly-l10n" repackage_name = "repackage-l10n" repackage_signing_name = "repackage-signing-l10n" mar_signing_name = "mar-signing-l10n" dependencies = { "build": upstream_deps[build_name], "repackage": upstream_deps[repackage_name], "signing": upstream_deps[signing_name], "mar-signing": upstream_deps[mar_signing_name], } if 'partials-signing' in upstream_deps: dependencies['partials-signing'] = upstream_deps[ 'partials-signing'] if msi_signing_name in upstream_deps: dependencies[msi_signing_name] = upstream_deps[msi_signing_name] if repackage_signing_name in upstream_deps: dependencies["repackage-signing"] = upstream_deps[ repackage_signing_name] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], 'shipping-product': job.get('shipping-product'), } yield task
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of Checksums file for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'cs({})'.format( dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target.checksums", ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg'), ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BMcs') ) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('kind', 'build') label = job['label'] build_platform = attributes.get('build_platform') description = ( "Beetmover submission of checksums for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) extra = {} if build_platform.startswith("android"): extra['product'] = 'fennec' elif 'devedition' in build_platform: extra['product'] = 'devedition' else: extra['product'] = 'firefox' dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, } if 'shipping-phase' in job: task['shipping-phase'] = job['shipping-phase'] if 'shipping-product' in job: task['shipping-product'] = job['shipping-product'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "c-Up(N)") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", "{}/opt".format(dep_th_platform)) treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)) treeherder.setdefault("kind", "build") attributes = copy_attributes_from_dependent_job(dep_job) treeherder_job_symbol = dep_job.task["extra"]["treeherder"]["symbol"] treeherder["symbol"] = replace_group(treeherder_job_symbol, "c-Up") if dep_job.attributes.get("locale"): attributes["locale"] = dep_job.attributes.get("locale") label = job["label"] description = ("Balrog submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": ["public/manifest.json"], }] dependencies = {"beetmover": dep_job.label} for kind_dep in config.kind_dependencies_tasks.values(): if (kind_dep.kind == "startup-test" and kind_dep.attributes["build_platform"] == attributes.get("build_platform") and kind_dep.attributes["build_type"] == attributes.get("build_type") and kind_dep.attributes.get("shipping_product") == job.get("shipping-product")): dependencies["startup-test"] = kind_dep.label task = { "label": label, "description": description, "worker-type": "balrog", "worker": { "implementation": "balrog", "upstream-artifacts": upstream_artifacts, "balrog-action": "v2-submit-locale", "suffixes": ["", "-No-WNP"] if job.get("update-no-wnp") else [""], }, "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job.get("shipping-phase", "promote"), "shipping-product": job.get("shipping-product"), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = inherit_treeherder_from_dep(job, dep_job) upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"] if "build" in job["dependent-tasks"]: upstream_symbol = job["dependent-tasks"]["build"].task["extra"][ "treeherder"]["symbol"] treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR")) label = job["label"] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), )) upstream_deps = job["dependent-tasks"] signing_name = "build-signing" build_name = "build" repackage_name = "repackage" repackage_signing_name = "repackage-signing" msi_signing_name = "repackage-signing-msi" mar_signing_name = "mar-signing" if job.get("locale"): signing_name = "shippable-l10n-signing" build_name = "shippable-l10n" repackage_name = "repackage-l10n" repackage_signing_name = "repackage-signing-l10n" mar_signing_name = "mar-signing-l10n" dependencies = { "build": upstream_deps[build_name], "repackage": upstream_deps[repackage_name], "signing": upstream_deps[signing_name], "mar-signing": upstream_deps[mar_signing_name], } if "partials-signing" in upstream_deps: dependencies["partials-signing"] = upstream_deps[ "partials-signing"] if msi_signing_name in upstream_deps: dependencies[msi_signing_name] = upstream_deps[msi_signing_name] if repackage_signing_name in upstream_deps: dependencies["repackage-signing"] = upstream_deps[ repackage_signing_name] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if job.get("locale"): attributes["locale"] = job["locale"] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job["shipping-phase"], "shipping-product": job.get("shipping-product"), } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault( "symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"), ) dep_th_platform = ( dep_job.task.get("extra", {}) .get("treeherder", {}) .get("machine", {}) .get("platform", "") ) treeherder.setdefault("platform", "{}/opt".format(dep_th_platform)) treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) ) treeherder.setdefault("kind", "build") label = job["label"] build_platform = attributes.get("build_platform") description = ( "Beetmover submission of checksums for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=build_platform, build_type=attributes.get("build_type"), ) ) extra = {} if "devedition" in build_platform: extra["product"] = "devedition" else: extra["product"] = "firefox" dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if dep_job.attributes.get("locale"): treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale")) attributes["locale"] = dep_job.attributes.get("locale") bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "extra": extra, } if "shipping-phase" in job: task["shipping-phase"] = job["shipping-phase"] if "shipping-product" in job: task["shipping-product"] = job["shipping-product"] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault( "symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM") ) dep_th_platform = ( dep_job.task.get("extra", {}) .get("treeherder", {}) .get("machine", {}) .get("platform", "") ) treeherder.setdefault("platform", "{}/opt".format(dep_th_platform)) treeherder.setdefault( "tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1) ) treeherder.setdefault("kind", "build") label = job["label"] description = ( "Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get("locale", "en-US"), build_platform=attributes.get("build_platform"), build_type=attributes.get("build_type"), ) ) dependencies = {dep_job.kind: dep_job.label} # XXX release snap-repackage has a variable number of dependencies, depending on how many # "post-beetmover-dummy" jobs there are in the graph. if dep_job.kind != "release-snap-repackage" and len(dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies" ) signing_dependencies = dep_job.dependencies dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get("attributes", {})) if job.get("locale"): attributes["locale"] = job["locale"] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { "label": label, "description": description, "worker-type": "beetmover", "scopes": [bucket_scope, action_scope], "dependencies": dependencies, "attributes": attributes, "run-on-projects": dep_job.attributes.get("run_on_projects"), "treeherder": treeherder, "shipping-phase": job["shipping-phase"], } yield task