def populate_repack_manifests_url(config, tasks):
for task in tasks:
partner_url_config = get_partner_url_config(config.params,
config.graph_config)
for k in partner_url_config:
if config.kind.startswith(k):
task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
partner_url_config[k]
break
else:
raise Exception("Can't find partner REPACK_MANIFESTS_URL")
for property in ("limit-locales", ):
property = "extra.{}".format(property)
resolve_keyed_by(
task, property, property,
**{'release-level': config.params.release_level()})
if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
task.setdefault('scopes', []).append(
'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'
.format(**config.params))
yield task
def populate_repack_manifests_url(config, tasks):
for task in tasks:
partner_url_config = get_partner_url_config(config.params,
config.graph_config)
for k in partner_url_config:
if config.kind.startswith(k):
task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
partner_url_config[k]
break
else:
raise Exception("Can't find partner REPACK_MANIFESTS_URL")
if task['worker']['env']['REPACK_MANIFESTS_URL'].startswith('git@'):
task.setdefault('scopes', []).append(
'secrets:get:project/releng/gecko/build/level-{level}/partner-github-ssh'
.format(**config.params))
yield task
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][
release_promotion_flavor]
release_history = {}
product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if promotion_config.get('version-bump', False):
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get('partial-updates', False):
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix,
parameters['project'],
partial_updates=input['partial_updates'])
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds',
promotion_config.get('rebuild-kinds', []))
do_not_optimize = input.get('do_not_optimize',
promotion_config.get('do-not-optimize', []))
release_enable_partners = input.get(
'release_enable_partners', parameters['project'] in PARTNER_BRANCHES
and product in ('firefox', ))
release_enable_emefree = input.get(
'release_enable_emefree', parameters['project'] in EMEFREE_BRANCHES
and product in ('firefox', ))
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
parameters['pushlog_id'] = parameters['pushlog_id'] or \
find_hg_revision_pushlog_id(parameters, graph_config, revision)
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['release_type'] = promotion_config.get('release-type', '')
parameters['release_eta'] = input.get('release_eta', '')
parameters['release_enable_partners'] = release_enable_partners
parameters['release_partners'] = input.get('release_partners')
parameters['release_enable_emefree'] = release_enable_emefree
parameters['release_product'] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
partner_config = input.get('release_partner_config')
if not partner_config and (release_enable_emefree
or release_enable_partners):
partner_url_config = get_partner_url_config(
parameters,
graph_config,
enable_emefree=release_enable_emefree,
enable_partners=release_enable_partners)
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if input.get('release_partner_build_number'):
parameters['release_partner_build_number'] = input[
'release_partner_build_number']
if partner_config:
parameters['release_partner_config'] = fix_partner_config(
partner_config)
if input['version']:
parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][
release_promotion_flavor]
release_history = {}
product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if promotion_config.get('version-bump', False):
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get('partial-updates', False):
partial_updates = input.get('partial_updates', {})
if not partial_updates and release_level(
parameters['project']) == 'production':
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = json.dumps(partial_updates)
release_history = populate_release_history(
balrog_prefix,
parameters['project'],
partial_updates=partial_updates)
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds',
promotion_config.get('rebuild-kinds', []))
do_not_optimize = input.get('do_not_optimize',
promotion_config.get('do-not-optimize', []))
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
if not parameters['pushlog_id']:
repo_param = '{}head_repository'.format(
graph_config['project-repo-param-prefix'])
push_info = find_hg_revision_push_info(
repository=parameters[repo_param], revision=revision)
parameters['pushlog_id'] = push_info['pushid']
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
if promotion_config.get('is-rc'):
parameters['release_type'] += '-rc'
parameters['release_eta'] = input.get('release_eta', '')
parameters['release_product'] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
# Partner/EMEfree are enabled by default when get_partner_url_config() returns a non-null url
# The action input may override by sending False. It's an error to send True with no url found
partner_url_config = get_partner_url_config(parameters, graph_config)
release_enable_partners = partner_url_config[
'release-partner-repack'] is not None
release_enable_emefree = partner_url_config[
'release-eme-free-repack'] is not None
if input.get('release_enable_partners') is False:
release_enable_partners = False
elif input.get(
'release_enable_partners') is True and not release_enable_partners:
raise Exception(
"Can't enable partner repacks when no config url found")
if input.get('release_enable_emefree') is False:
release_enable_emefree = False
elif input.get(
'release_enable_emefree') is True and not release_enable_emefree:
raise Exception("Can't enable EMEfree when no config url found")
parameters['release_enable_partners'] = release_enable_partners
parameters['release_enable_emefree'] = release_enable_emefree
partner_config = input.get('release_partner_config')
if not partner_config and (release_enable_emefree
or release_enable_partners):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters['release_partner_config'] = fix_partner_config(
partner_config)
parameters['release_partners'] = input.get('release_partners')
if input.get('release_partner_build_number'):
parameters['release_partner_build_number'] = input[
'release_partner_build_number']
if input['version']:
parameters['version'] = input['version']
parameters['required_signoffs'] = get_required_signoffs(input, parameters)
parameters['signoff_urls'] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id):
release_promotion_flavor = input["release_promotion_flavor"]
promotion_config = graph_config["release-promotion"]["flavors"][
release_promotion_flavor]
release_history = {}
product = promotion_config["product"]
next_version = str(input.get("next_version") or "")
if promotion_config.get("version-bump", False):
# We force str() the input, hence the 'None'
if next_version in ["", "None"]:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get("partial-updates", False):
partial_updates = input.get("partial_updates", {})
if not partial_updates and release_level(
parameters["project"]) == "production":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates,
sort_keys=True)
release_history = populate_release_history(
balrog_prefix,
parameters["project"],
partial_updates=partial_updates)
target_tasks_method = promotion_config["target-tasks-method"].format(
project=parameters["project"])
rebuild_kinds = input.get("rebuild_kinds",
promotion_config.get("rebuild-kinds", []))
do_not_optimize = input.get("do_not_optimize",
promotion_config.get("do-not-optimize", []))
# Build previous_graph_ids from ``previous_graph_ids``, ``revision``,
# or the action parameters.
previous_graph_ids = input.get("previous_graph_ids")
if not previous_graph_ids:
revision = input.get("revision")
if revision:
head_rev_param = "{}head_rev".format(
graph_config["project-repo-param-prefix"])
push_parameters = {
head_rev_param: revision,
"project": parameters["project"],
}
else:
push_parameters = parameters
previous_graph_ids = [
find_decision_task(push_parameters, graph_config)
]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters["do_not_optimize"] = do_not_optimize
parameters["target_tasks_method"] = target_tasks_method
parameters["build_number"] = int(input["build_number"])
parameters["next_version"] = next_version
parameters["release_history"] = release_history
if promotion_config.get("is-rc"):
parameters["release_type"] += "-rc"
parameters["release_eta"] = input.get("release_eta", "")
parameters["release_product"] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters["optimize_target_tasks"] = True
if release_promotion_flavor == "promote_firefox_partner_repack":
release_enable_partner_repack = True
release_enable_partner_attribution = False
release_enable_emefree = False
elif release_promotion_flavor == "promote_firefox_partner_attribution":
release_enable_partner_repack = False
release_enable_partner_attribution = True
release_enable_emefree = False
else:
# for promotion or ship phases, we use the action input to turn the repacks/attribution off
release_enable_partner_repack = input.get(
"release_enable_partner_repack", True)
release_enable_partner_attribution = input.get(
"release_enable_partner_attribution", True)
release_enable_emefree = input.get("release_enable_emefree", True)
partner_url_config = get_partner_url_config(parameters, graph_config)
if (release_enable_partner_repack
and not partner_url_config["release-partner-repack"]):
raise Exception(
"Can't enable partner repacks when no config url found")
if (release_enable_partner_attribution
and not partner_url_config["release-partner-attribution"]):
raise Exception(
"Can't enable partner attribution when no config url found")
if release_enable_emefree and not partner_url_config[
"release-eme-free-repack"]:
raise Exception(
"Can't enable EMEfree repacks when no config url found")
parameters["release_enable_partner_repack"] = release_enable_partner_repack
parameters[
"release_enable_partner_attribution"] = release_enable_partner_attribution
parameters["release_enable_emefree"] = release_enable_emefree
partner_config = input.get("release_partner_config")
if not partner_config and any([
release_enable_partner_repack,
release_enable_partner_attribution,
release_enable_emefree,
]):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters["release_partner_config"] = fix_partner_config(
partner_config)
parameters["release_partners"] = input.get("release_partners")
if input.get("release_partner_build_number"):
parameters["release_partner_build_number"] = input[
"release_partner_build_number"]
if input["version"]:
parameters["version"] = input["version"]
parameters["required_signoffs"] = get_required_signoffs(input, parameters)
parameters["signoff_urls"] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
