def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'cs(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of Checksums file for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'cs({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') upstream_artifacts = [{ "taskId": {"task-reference": "<beetmover>"}, "taskType": "beetmover", "paths": [ "public/target.checksums", ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'scopes': [ signing_cert_scope, "project:releng:signing:format:gpg" ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', 'BMcslang(N{})'.format(attributes.get('l10n_chunk', ''))) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] build_platform = attributes.get('build_platform') description = "Beetmover submission of checksums for langpack files" extra = {} if 'devedition' in build_platform: extra['product'] = 'devedition' else: extra['product'] = 'firefox' dependencies = {dep_job.kind: dep_job.label} for k, v in dep_job.dependencies.items(): if k.startswith('beetmover'): dependencies[k] = v attributes = copy_attributes_from_dependent_job(dep_job) if 'chunk_locales' in dep_job.attributes: attributes['chunk_locales'] = dep_job.attributes['chunk_locales'] attributes.update(job.get('attributes', {})) bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, } if 'shipping-phase' in job: task['shipping-phase'] = job['shipping-phase'] if 'shipping-product' in job: task['shipping-product'] = job['shipping-product'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Gd(s)') treeherder.setdefault( 'platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}/{}'".format( attributes.get('build_platform'), attributes.get('build_type'), ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'ps(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') label = job.get('label', "partials-signing-{}".format(dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) locale = dep_job.attributes.get('locale') if locale: attributes['locale'] = locale treeherder['symbol'] = 'ps({})'.format(locale) balrog_platform = get_balrog_platform_name(dep_th_platform) upstream_artifacts = generate_upstream_artifacts( dep_job, config.params['release_history'], balrog_platform, locale) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384'] task = { 'label': label, 'description': "{} Partials".format( dep_job.task["metadata"]["description"]), 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600}, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {dep_job.kind: dep_job.label} # XXX release snap-repackage has a variable number of dependencies, depending on how many # "post-beetmover-dummy" jobs there are in the graph. if dep_job.kind != 'release-snap-repackage' and len( dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], } yield task
def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False): action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner') task = deepcopy(job) task['scopes'] = [bucket_scope, action_scope] task['worker-type'] = get_worker_type_for_scope(config, bucket_scope) task['partner_public'] = partner_public if partner_public: task['label'] = "{}-public".format(task['label']) return task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-gv') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform)) treeherder.setdefault('tier', 2) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for geckoview" "{build_platform}/{build_type}'".format( build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) if job.get('locale'): attributes['locale'] = job['locale'] resolve_keyed_by(job, 'bucket-scope', item_name=job['label'], **{'release-level': config.params.release_level()}) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, job['bucket-scope']), 'scopes': [ job['bucket-scope'], 'project:releng:beetmover:action:push-to-maven' ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': ['mozilla-central'], 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-gv') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', '{}/opt'.format(dep_th_platform)) treeherder.setdefault('tier', 2) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for geckoview" "{build_platform}/{build_type}'".format( build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {dep_job.kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if job.get('locale'): attributes['locale'] = job['locale'] attributes['run_on_hg_branches'] = job['run-on-hg-branches'] task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, job['bucket-scope']), 'scopes': [ job['bucket-scope'], 'project:releng:beetmover:action:push-to-maven' ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': job['run-on-projects'], 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], 'shipping-product': job.get('shipping-product'), } yield task
def make_release_generate_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'SGenChcks') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') job_template = "{}-{}".format(dep_job.label, "signing") label = job.get("label", job_template) description = "Signing of the overall release-related checksums" dependencies = {"build": dep_job.label} upstream_artifacts = [{ "taskId": { "task-reference": "<build>" }, "taskType": "build", "paths": [ get_artifact_path(dep_job, "SHA256SUMS"), get_artifact_path(dep_job, "SHA512SUMS"), ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [signing_cert_scope, "project:releng:signing:format:gpg"], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def add_balrog_scopes(config, jobs): for job in jobs: worker = job['worker'] server_scope = get_balrog_server_scope(config) action_scope = get_balrog_action_scope(config, action=worker['balrog-action']) job['scopes'] = [server_scope, action_scope] job['worker-type'] = get_worker_type_for_scope(config, server_scope) yield job
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'css(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = "Signing of release-source checksums file" dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target-source.checksums", ], "formats": ["autograph_gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [ signing_cert_scope, ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-S') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} if len(dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") signing_dependencies = dep_job.dependencies dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) phase = get_phase(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': phase, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-SGenChcks') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') job_template = "{}".format(dep_job.label) label = job_template.replace("signing", "beetmover") description = "Transfer *SUMS and *SUMMARY checksums file to S3." # first dependency is the signing task for the *SUMS files dependencies = { str(dep_job.kind): dep_job.label } if len(dep_job.dependencies) > 1: raise NotImplementedError( "Can't beetmove a signing task with multiple dependencies") # update the dependencies with the dependencies of the signing task dependencies.update(dep_job.dependencies) bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': 'promote', } yield task
def make_beetmover_push_to_release_description(config, jobs): for job in jobs: treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Rel(BM-C)') treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') treeherder.setdefault('platform', job['treeherder-platform']) label = job['name'] description = ("Beetmover push to release for '{product}'".format( product=job['product'])) bucket_scope = get_beetmover_bucket_scope(config, job_release_type=job.get( 'attributes', {}).get('release-type')) action_scope = add_scope_prefix(config, 'beetmover:action:push-to-releases') task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'product': job['product'], 'dependencies': job['dependencies'], 'attributes': job.get('attributes', {}), 'run-on-projects': job.get('run-on-projects'), 'treeherder': treeherder, 'shipping-phase': job.get('shipping-phase', 'push'), 'shipping-product': job.get('shipping-product'), 'routes': job.get('routes', []), 'extra': job.get('extra', {}), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-R') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependent_kind = dep_job.kind if dependent_kind == 'repackage-signing-l10n': dependent_kind = "repackage-signing" dependencies = {dependent_kind: dep_job} signing_name = "build-signing" if job.get('locale'): signing_name = "nightly-l10n-signing" dependencies['signing'] = job['grandparent-tasks'][signing_name] build_name = "build" if job.get('locale'): build_name = "unsigned-repack" dependencies["build"] = job['grandparent-tasks'][build_name] # repackage-l10n actually uses the repackage depname here dependencies["repackage"] = job['grandparent-tasks']["repackage"] # If this isn't a direct dependency, it won't be in there. if 'repackage-signing' not in dependencies: repackage_signing_name = "repackage-signing" if job.get('locale'): repackage_signing_name = "repackage-signing-l10n" dependencies["repackage-signing"] = job['grandparent-tasks'][ repackage_signing_name] attributes = copy_attributes_from_dependent_job(dep_job) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) phase = get_phase(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job.get('shipping-phase', phase), 'shipping-product': job.get('shipping-product'), } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get('locale', dep_job.attributes.get('locale')) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) if attributes.get('nightly'): treeherder.setdefault('symbol', 'rs(N)') else: treeherder.setdefault('symbol', 'rs(B)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') if locale: treeherder['symbol'] = 'rs({})'.format(locale) if config.kind == 'repackage-signing-msi': treeherder['symbol'] = 'MSIs({})'.format(locale or 'N') label = job['label'] dep_kind = dep_job.kind if 'l10n' in dep_kind: dep_kind = 'repackage' dependencies = {dep_kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] upstream_artifacts = [] for artifact in dep_job.release_artifacts: basename = os.path.basename(artifact) if basename in SIGNING_FORMATS: upstream_artifacts.append({ "taskId": { "task-reference": "<{}>".format(dep_kind) }, "taskType": "repackage", "paths": [artifact], "formats": SIGNING_FORMATS[os.path.basename(artifact)], }) scopes += list({ add_scope_prefix(config, 'signing:format:{}'.format(format)) for artifact in upstream_artifacts for format in artifact['formats'] }) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_beetmover_checksums_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BMcs(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('kind', 'build') label = job['label'] build_platform = attributes.get('build_platform') description = ( "Beetmover submission of checksums for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) extra = {} if build_platform.startswith("android"): extra['product'] = 'fennec' elif 'devedition' in build_platform: extra['product'] = 'devedition' else: extra['product'] = 'firefox' dependent_kind = str(dep_job.kind) dependencies = {dependent_kind: dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'extra': extra, } if 'shipping-phase' in job: task['shipping-phase'] = job['shipping-phase'] if 'shipping-product' in job: task['shipping-product'] = job['shipping-product'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg') ] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["sha2signcode", "gpg"] }] scopes.append( add_scope_prefix(config, "signing:format:sha2signcode")) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["gpg"] }] task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] locale = dep_job.attributes.get('locale') treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', join_symbol(job.get('treeherder-group', 'ms'), locale or 'N')) label = job.get('label', "{}-{}".format(config.kind, dep_job.label)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('kind', 'build') treeherder.setdefault('tier', 1) dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['required_signoffs'] = sorted_unique_list( attributes.get('required_signoffs', []), job.pop('required_signoffs')) attributes['shipping_phase'] = job['shipping-phase'] if locale: attributes['locale'] = locale balrog_platform = get_balrog_platform_name(dep_th_platform) if config.kind == 'partials-signing': upstream_artifacts = generate_partials_artifacts( dep_job, config.params['release_history'], balrog_platform, locale) else: upstream_artifacts = generate_complete_artifacts( dep_job, config.kind) build_platform = dep_job.attributes.get('build_platform') is_nightly = job.get( 'nightly', # First check current job dep_job.attributes.get( 'nightly', # Then dep job for 'nightly' dep_job.attributes.get( 'shippable'))) # lastly dep job for 'shippable' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] task = { 'label': label, 'description': "{} {}".format(dep_job.task["metadata"]["description"], job['description-suffix']), 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'dependencies': dependencies, 'attributes': attributes, 'scopes': scopes, 'run-on-projects': job.get('run-on-projects', dep_job.attributes.get('run_on_projects')), 'treeherder': treeherder, } yield task
def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault( 'symbol', replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of Checksums file for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"beetmover": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) if dep_job.attributes.get('locale'): treeherder['symbol'] = 'cs({})'.format( dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": [ "public/target.checksums", ], "formats": ["gpg"] }] signing_cert_scope = get_signing_cert_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': [ signing_cert_scope, add_scope_prefix(config, 'signing:format:gpg'), ], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') )) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config, job_release_type=dep_job.attributes.get('release-type') ) worker_type_alias = get_worker_type_for_scope(config, signing_cert_scope) mac_behavior = None task = { 'label': label, 'description': description, 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600)}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if 'macosx' in build_platform: worker_type_alias_map = { 'linux-depsigning': 'mac-depsigning', 'linux-signing': 'mac-signing', } assert worker_type_alias in worker_type_alias_map, \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type_alias) ) worker_type_alias = worker_type_alias_map[worker_type_alias] mac_behavior = evaluate_keyed_by( config.graph_config['mac-notarization']['mac-behavior'], 'mac behavior', { 'release-type': config.params['release_type'], 'platform': build_platform, }, ) task['worker']['mac-behavior'] = mac_behavior if job.get('entitlements-url'): task['worker']['entitlements-url'] = job['entitlements-url'] task['worker-type'] = worker_type_alias if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) treeherder = job.get('treeherder', {}) is_nightly = dep_job.attributes.get('nightly', False) treeherder.setdefault('symbol', _generate_treeherder_symbol(is_nightly)) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') build_platform = dep_job.attributes.get('build_platform') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) treeherder.setdefault('tier', 1 if '-ccov' not in build_platform else 2) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( dep_job.attributes.get('build_platform'), is_nightly, config ) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': 3600}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'treeherder': treeherder, 'routes': job.get('routes', []), } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'rs(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Signing of repackaged artifacts for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) dependencies = {"repackage": dep_job.label} signing_dependencies = dep_job.dependencies # This is so we get the build task etc in our dependencies to # have better beetmover support. dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' locale_str = "" if dep_job.attributes.get('locale'): treeherder['symbol'] = 'rs({})'.format( dep_job.attributes.get('locale')) attributes['locale'] = dep_job.attributes.get('locale') locale_str = "{}/".format(dep_job.attributes.get('locale')) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get('nightly') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [ signing_cert_scope, 'project:releng:signing:format:mar_sha384' ] upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.complete.mar".format(locale_str), ], "formats": ["mar_sha384"] }] if 'win' in build_platform: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.installer.exe".format(locale_str), ], "formats": ["sha2signcode"] }) scopes.append("project:releng:signing:format:sha2signcode") # Stub installer is only generated on win32 if '32' in build_platform: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ "public/build/{}target.stub-installer.exe".format( locale_str), ], "formats": ["sha2signcodestub"] }) scopes.append("project:releng:signing:format:sha2signcodestub") task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-") description = ( "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( repack_id=repack_id, build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) if 'linux' in build_platform: # we want the repack job, via the dependencies for the the chunking-dummy dep_job for dep in dep_job.dependencies.values(): if dep.startswith('release-partner-repack'): dependencies = {"repack": dep} break else: # we have a genuine repackage job as our parent dependencies = {"repackage": dep_job.label} attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }] partner_config = get_partner_config_by_kind(config, config.kind) partner, subpartner, _ = repack_id.split('/') repack_stub_installer = partner_config[partner][subpartner].get( 'repack_stub_installer') if build_platform.startswith('win32') and repack_stub_installer: upstream_artifacts.append({ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path( dep_job, "{}/target.stub-installer.exe".format(repack_id)), ], "formats": ["autograph_authenticode", "autograph_gpg"] }) elif 'mac' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repackage>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.dmg".format(repack_id)), ], "formats": ["autograph_gpg"] }] elif 'linux' in build_platform: upstream_artifacts = [{ "taskId": { "task-reference": "<repack>" }, "taskType": "repackage", "paths": [ get_artifact_path(dep_job, "{}/target.tar.bz2".format(repack_id)), ], "formats": ["autograph_gpg"] }] task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'extra': { 'repack_id': repack_id, } } # we may have reduced the priority for partner jobs, otherwise task.py will set it if job.get('priority'): task['priority'] = job['priority'] yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_nightly = True # cert_scope_per_platform uses this to choose the right cert description = ("Signing of OpenH264 Binaries for '" "{build_platform}/{build_type}'".format( build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) # we have a genuine repackage job as our parent dependencies = {"openh264": dep_job.label} my_attributes = copy_attributes_from_dependent_job(dep_job) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) scopes = [signing_cert_scope] if 'win' in build_platform: # job['primary-dependency'].task['payload']['command'] scopes.append( add_scope_prefix(config, "signing:format:sha2signcode")) formats = ['sha2signcode'] else: formats = ['autograph_gpg'] rev = attributes['openh264_rev'] upstream_artifacts = [{ "taskId": { "task-reference": "<openh264>" }, "taskType": "build", "paths": [ "private/openh264/openh264-{}-{}.zip".format( build_platform, rev), ], "formats": formats }] treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( 'symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol'))) task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, 'max-run-time': 3600 }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': my_attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append( add_scope_prefix(config, 'signing:format:{}'.format(format)) ) is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable', False)) build_platform = dep_job.attributes.get('build_platform') treeherder = None if 'partner' not in config.kind and 'eme-free' not in config.kind: treeherder = job.get('treeherder', {}) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') build_type = dep_job.attributes.get('build_type') treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) treeherder.setdefault('symbol', _generate_treeherder_symbol( dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') )) treeherder.setdefault('kind', 'build') label = job['label'] description = ( "Initial Signing for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=build_platform, build_type=attributes.get('build_type') ) ) attributes = copy_attributes_from_dependent_job(dep_job) attributes['signed'] = True if dep_job.attributes.get('chunk_locales'): # Used for l10n attribute passthrough attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales') signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config ) worker_type = get_worker_type_for_scope(config, signing_cert_scope) task = { 'label': label, 'description': description, 'worker-type': worker_type, 'worker': {'implementation': 'scriptworker-signing', 'upstream-artifacts': job['upstream-artifacts'], 'max-run-time': job.get('max-run-time', 3600)}, 'scopes': [signing_cert_scope] + signing_format_scopes, 'dependencies': {job['depname']: dep_job.label}, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'optimization': dep_job.optimization, 'routes': job.get('routes', []), 'shipping-product': job.get('shipping-product'), 'shipping-phase': job.get('shipping-phase'), } if treeherder: task['treeherder'] = treeherder if job.get('extra'): task['extra'] = job['extra'] yield task
def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage-signing' treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Gd(s)') treeherder.setdefault( 'platform', dep_job.task.get('extra', {}).get('treeherder-platform')) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') dependencies = {dep_job.kind: dep_job.label} signing_dependencies = dep_job.dependencies dependencies.update({ k: v for k, v in signing_dependencies.items() if k != 'docker-image' }) description = "Signing Geckodriver for build '{}/{}'".format( attributes.get('build_platform'), attributes.get('build_type'), ) build_platform = dep_job.attributes.get('build_platform') is_nightly = dep_job.attributes.get( 'nightly', dep_job.attributes.get('shippable')) signing_cert_scope = get_signing_cert_scope_per_platform( build_platform, is_nightly, config) upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform) scopes = [signing_cert_scope] task = { 'label': job['label'], 'description': description, 'worker-type': get_worker_type_for_scope(config, signing_cert_scope), 'worker': { 'implementation': 'scriptworker-signing', 'upstream-artifacts': upstream_artifacts, }, 'scopes': scopes, 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, } if build_platform.startswith('macosx'): worker_type = task['worker-type'] worker_type_alias_map = { 'linux-depsigning': 'mac-depsigning', 'linux-signing': 'mac-signing', } assert worker_type in worker_type_alias_map, \ ( "Make sure to adjust the below worker_type_alias logic for " "mac if you change the signing workerType aliases!" " ({} not found in mapping)".format(worker_type) ) worker_type = worker_type_alias_map[worker_type] task['worker-type'] = worker_type_alias_map[task['worker-type']] task['worker']['mac-behavior'] = 'mac_geckodriver' yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) upstream_symbol = dep_job.task['extra']['treeherder']['symbol'] if 'build' in job['dependent-tasks']: upstream_symbol = job['dependent-tasks']['build'].task['extra'][ 'treeherder']['symbol'] treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR')) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') label = job['label'] description = ("Beetmover submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) upstream_deps = job['dependent-tasks'] signing_name = "build-signing" build_name = "build" repackage_name = "repackage" repackage_signing_name = "repackage-signing" msi_signing_name = "repackage-signing-msi" mar_signing_name = "mar-signing" if job.get('locale'): signing_name = "nightly-l10n-signing" build_name = "nightly-l10n" repackage_name = "repackage-l10n" repackage_signing_name = "repackage-signing-l10n" mar_signing_name = "mar-signing-l10n" dependencies = { "build": upstream_deps[build_name], "repackage": upstream_deps[repackage_name], "signing": upstream_deps[signing_name], "mar-signing": upstream_deps[mar_signing_name], } if 'partials-signing' in upstream_deps: dependencies['partials-signing'] = upstream_deps[ 'partials-signing'] if msi_signing_name in upstream_deps: dependencies[msi_signing_name] = upstream_deps[msi_signing_name] if repackage_signing_name in upstream_deps: dependencies["repackage-signing"] = upstream_deps[ repackage_signing_name] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) if job.get('locale'): attributes['locale'] = job['locale'] bucket_scope = get_beetmover_bucket_scope(config) action_scope = get_beetmover_action_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, bucket_scope), 'scopes': [bucket_scope, action_scope], 'dependencies': dependencies, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job['shipping-phase'], 'shipping-product': job.get('shipping-product'), } yield task
def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'c-Up(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault( 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)) treeherder.setdefault('kind', 'build') attributes = copy_attributes_from_dependent_job(dep_job) treeherder_job_symbol = dep_job.task['extra']['treeherder']['symbol'] treeherder['symbol'] = replace_group(treeherder_job_symbol, 'c-Up') if dep_job.attributes.get('locale'): attributes['locale'] = dep_job.attributes.get('locale') label = job['label'] description = ("Balrog submission for locale '{locale}' for build '" "{build_platform}/{build_type}'".format( locale=attributes.get('locale', 'en-US'), build_platform=attributes.get('build_platform'), build_type=attributes.get('build_type'))) upstream_artifacts = [{ "taskId": { "task-reference": "<beetmover>" }, "taskType": "beetmover", "paths": ["public/manifest.json"], }] server_scope = get_balrog_server_scope(config) task = { 'label': label, 'description': description, 'worker-type': get_worker_type_for_scope(config, server_scope), 'worker': { 'implementation': 'balrog', 'upstream-artifacts': upstream_artifacts, 'balrog-action': 'submit-locale', 'suffixes': ['', '-No-WNP'] if job.get('update-no-wnp') else [''], }, 'dependencies': { 'beetmover': dep_job.label }, 'attributes': attributes, 'run-on-projects': dep_job.attributes.get('run_on_projects'), 'treeherder': treeherder, 'shipping-phase': job.get('shipping-phase', 'promote'), 'shipping-product': job.get('shipping-product'), } yield task