def generate_partials_builds(self, product, branch):
from taskgraph.util.partials import populate_release_history
try:
import yaml
release_history = {'release_history': populate_release_history(product, branch)}
print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
except Exception:
traceback.print_exc()
sys.exit(1)
def generate_partials_builds(self, product, branch):
from taskgraph.util.partials import populate_release_history
try:
import yaml
release_history = {'release_history': populate_release_history(product, branch)}
print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
except Exception:
traceback.print_exc()
sys.exit(1)
def get_decision_parameters(graph_config, parameters):
logger.info("{}.get_decision_parameters called".format(__name__))
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
project = parameters["project"]
parameters.setdefault("release_history", dict())
if "nightly" in parameters.get("target_tasks_method", ""):
parameters["release_history"] = populate_release_history(
BALROG_PRODUCT, project)
def get_decision_parameters(options):
"""
Load parameters from the command-line options for 'taskgraph decision'.
This also applies per-project parameters, based on the given project.
"""
parameters = {
n: options[n]
for n in [
'base_repository',
'head_repository',
'head_rev',
'head_ref',
'message',
'project',
'pushlog_id',
'pushdate',
'owner',
'level',
'target_tasks_method',
] if n in options
}
for n in (
'comm_base_repository',
'comm_head_repository',
'comm_head_rev',
'comm_head_ref',
):
if n in options and options[n] is not None:
parameters[n] = options[n]
# Define default filter list, as most configurations shouldn't need
# custom filters.
parameters['filters'] = [
'check_servo',
'target_tasks_method',
]
parameters['existing_tasks'] = {}
parameters['do_not_optimize'] = []
parameters['build_number'] = 1
parameters['version'] = get_version()
parameters['app_version'] = get_app_version()
parameters['next_version'] = None
parameters['release_type'] = ''
parameters['release_eta'] = ''
parameters['release_enable_partners'] = False
parameters['release_partners'] = []
parameters['release_partner_config'] = {}
parameters['release_partner_build_number'] = 1
parameters['release_enable_emefree'] = False
parameters['release_product'] = None
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
# use the pushdate as build_date if given, else use current time
parameters['build_date'] = parameters['pushdate'] or int(time.time())
# moz_build_date is the build identifier based on build_date
parameters['moz_build_date'] = time.strftime(
"%Y%m%d%H%M%S", time.gmtime(parameters['build_date']))
project = parameters['project']
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
"for this project".format(project, __file__))
parameters.update(PER_PROJECT_PARAMETERS['default'])
# `target_tasks_method` has higher precedence than `project` parameters
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history(
'Firefox', project)
if options.get('try_task_config_file'):
task_config_file = os.path.abspath(options.get('try_task_config_file'))
else:
# if try_task_config.json is present, load it
task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
# load try settings
if 'try' in project:
parameters['try_mode'] = None
if os.path.isfile(task_config_file):
logger.info("using try tasks from {}".format(task_config_file))
parameters['try_mode'] = 'try_task_config'
with open(task_config_file, 'r') as fh:
parameters['try_task_config'] = json.load(fh)
else:
parameters['try_task_config'] = None
if 'try:' in parameters['message']:
parameters['try_mode'] = 'try_option_syntax'
args = parse_message(parameters['message'])
parameters['try_options'] = args
else:
parameters['try_options'] = None
if parameters['try_mode']:
# The user has explicitly requested a set of jobs, so run them all
# regardless of optimization. Their dependencies can be optimized,
# though.
parameters['optimize_target_tasks'] = False
else:
# For a try push with no task selection, apply the default optimization
# process to all of the tasks.
parameters['optimize_target_tasks'] = True
else:
parameters['try_mode'] = None
parameters['try_task_config'] = None
parameters['try_options'] = None
result = Parameters(**parameters)
result.check()
return result
def get_decision_parameters(config, options):
"""
Load parameters from the command-line options for 'taskgraph decision'.
This also applies per-project parameters, based on the given project.
"""
product_dir = config['product-dir']
parameters = {
n: options[n]
for n in [
'base_repository',
'head_repository',
'head_rev',
'head_ref',
'message',
'project',
'pushlog_id',
'pushdate',
'owner',
'level',
'target_tasks_method',
] if n in options
}
for n in (
'comm_base_repository',
'comm_head_repository',
'comm_head_rev',
'comm_head_ref',
):
if n in options and options[n] is not None:
parameters[n] = options[n]
# Define default filter list, as most configurations shouldn't need
# custom filters.
parameters['filters'] = [
'target_tasks_method',
]
parameters['existing_tasks'] = {}
parameters['do_not_optimize'] = []
parameters['build_number'] = 1
parameters['version'] = get_version(product_dir)
parameters['app_version'] = get_app_version(product_dir)
parameters['hg_branch'] = get_hg_revision_branch(
GECKO, revision=parameters['head_rev'])
parameters['next_version'] = None
parameters['release_type'] = ''
parameters['release_eta'] = ''
parameters['release_enable_partners'] = False
parameters['release_partners'] = []
parameters['release_partner_config'] = {}
parameters['release_partner_build_number'] = 1
parameters['release_enable_emefree'] = False
parameters['release_product'] = None
parameters['required_signoffs'] = []
parameters['signoff_urls'] = {}
parameters['try_mode'] = None
parameters['try_task_config'] = None
parameters['try_options'] = None
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
# use the pushdate as build_date if given, else use current time
parameters['build_date'] = parameters['pushdate'] or int(time.time())
# moz_build_date is the build identifier based on build_date
parameters['moz_build_date'] = time.strftime(
"%Y%m%d%H%M%S", time.gmtime(parameters['build_date']))
project = parameters['project']
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
"for this project".format(project, __file__))
parameters.update(PER_PROJECT_PARAMETERS['default'])
# `target_tasks_method` has higher precedence than `project` parameters
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history(
'Firefox', project)
if options.get('try_task_config_file'):
task_config_file = os.path.abspath(options.get('try_task_config_file'))
else:
# if try_task_config.json is present, load it
task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
# load try settings
if 'try' in project:
set_try_config(parameters, task_config_file)
result = Parameters(**parameters)
result.check()
return result
def release_promotion_action(parameters, input, task_group_id, task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
release_history = {}
product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if release_promotion_flavor in VERSION_BUMP_FLAVORS:
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for %s "
"targets." % ', '.join(VERSION_BUMP_FLAVORS))
if product in ('firefox', 'devedition'):
if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS:
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for %s "
"targets." % ', '.join(PARTIAL_UPDATES_FLAVORS))
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix,
parameters['project'],
partial_updates=input['partial_updates'])
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
target_tasks_method = promotion_config['target_tasks_method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds',
promotion_config.get('rebuild_kinds', []))
do_not_optimize = input.get('do_not_optimize',
promotion_config.get('do_not_optimize', []))
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
parameters['pushlog_id'] = parameters['pushlog_id'] or \
find_hg_revision_pushlog_id(parameters, revision)
previous_graph_ids = [find_decision_task(parameters)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['release_type'] = promotion_config.get('release_type', '')
parameters['release_eta'] = input.get('release_eta', '')
if input['version']:
parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][
release_promotion_flavor]
release_history = {}
product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if promotion_config.get('version-bump', False):
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get('partial-updates', False):
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix,
parameters['project'],
partial_updates=input['partial_updates'])
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds',
promotion_config.get('rebuild-kinds', []))
do_not_optimize = input.get('do_not_optimize',
promotion_config.get('do-not-optimize', []))
release_enable_partners = input.get(
'release_enable_partners', parameters['project'] in PARTNER_BRANCHES
and product in ('firefox', ))
release_enable_emefree = input.get(
'release_enable_emefree', parameters['project'] in EMEFREE_BRANCHES
and product in ('firefox', ))
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
parameters['pushlog_id'] = parameters['pushlog_id'] or \
find_hg_revision_pushlog_id(parameters, graph_config, revision)
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['release_type'] = promotion_config.get('release-type', '')
parameters['release_eta'] = input.get('release_eta', '')
parameters['release_enable_partners'] = release_enable_partners
parameters['release_partners'] = input.get('release_partners')
parameters['release_enable_emefree'] = release_enable_emefree
parameters['release_product'] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
partner_config = input.get('release_partner_config')
if not partner_config and (release_enable_emefree
or release_enable_partners):
partner_url_config = get_partner_url_config(
parameters,
graph_config,
enable_emefree=release_enable_emefree,
enable_partners=release_enable_partners)
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if input.get('release_partner_build_number'):
parameters['release_partner_build_number'] = input[
'release_partner_build_number']
if partner_config:
parameters['release_partner_config'] = fix_partner_config(
partner_config)
if input['version']:
parameters['version'] = input['version']
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def get_decision_parameters(options):
"""
Load parameters from the command-line options for 'taskgraph decision'.
This also applies per-project parameters, based on the given project.
"""
parameters = {
n: options[n]
for n in [
'base_repository',
'head_repository',
'head_rev',
'head_ref',
'message',
'project',
'pushlog_id',
'pushdate',
'owner',
'level',
'target_tasks_method',
] if n in options
}
# Define default filter list, as most configurations shouldn't need
# custom filters.
parameters['filters'] = [
'check_servo',
'target_tasks_method',
]
parameters['target_task_labels'] = []
parameters['morph_templates'] = {}
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
# use the pushdate as build_date if given, else use current time
parameters['build_date'] = parameters['pushdate'] or int(time.time())
# moz_build_date is the build identifier based on build_date
parameters['moz_build_date'] = time.strftime(
"%Y%m%d%H%M%S", time.gmtime(parameters['build_date']))
project = parameters['project']
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
"for this project".format(project, __file__))
parameters.update(PER_PROJECT_PARAMETERS['default'])
# morph_templates and target_task_labels are only used on try, so don't
# bother loading them elsewhere
task_config_file = os.path.join(GECKO, 'try_task_config.json')
if project == 'try' and os.path.isfile(task_config_file):
with open(task_config_file, 'r') as fh:
task_config = json.load(fh)
parameters['morph_templates'] = task_config.get('templates', {})
parameters['target_task_labels'] = task_config.get('tasks')
# `target_tasks_method` has higher precedence than `project` parameters
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history(
'Firefox', project)
return Parameters(parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][
release_promotion_flavor]
release_history = {}
product = promotion_config['product']
next_version = str(input.get('next_version') or '')
if promotion_config.get('version-bump', False):
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get('partial-updates', False):
partial_updates = input.get('partial_updates', {})
if not partial_updates and release_level(
parameters['project']) == 'production':
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ['PARTIAL_UPDATES'] = json.dumps(partial_updates)
release_history = populate_release_history(
balrog_prefix,
parameters['project'],
partial_updates=partial_updates)
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project'])
rebuild_kinds = input.get('rebuild_kinds',
promotion_config.get('rebuild-kinds', []))
do_not_optimize = input.get('do_not_optimize',
promotion_config.get('do-not-optimize', []))
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
if not parameters['pushlog_id']:
repo_param = '{}head_repository'.format(
graph_config['project-repo-param-prefix'])
push_info = find_hg_revision_push_info(
repository=parameters[repo_param], revision=revision)
parameters['pushlog_id'] = push_info['pushid']
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
if promotion_config.get('is-rc'):
parameters['release_type'] += '-rc'
parameters['release_eta'] = input.get('release_eta', '')
parameters['release_product'] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
# Partner/EMEfree are enabled by default when get_partner_url_config() returns a non-null url
# The action input may override by sending False. It's an error to send True with no url found
partner_url_config = get_partner_url_config(parameters, graph_config)
release_enable_partners = partner_url_config[
'release-partner-repack'] is not None
release_enable_emefree = partner_url_config[
'release-eme-free-repack'] is not None
if input.get('release_enable_partners') is False:
release_enable_partners = False
elif input.get(
'release_enable_partners') is True and not release_enable_partners:
raise Exception(
"Can't enable partner repacks when no config url found")
if input.get('release_enable_emefree') is False:
release_enable_emefree = False
elif input.get(
'release_enable_emefree') is True and not release_enable_emefree:
raise Exception("Can't enable EMEfree when no config url found")
parameters['release_enable_partners'] = release_enable_partners
parameters['release_enable_emefree'] = release_enable_emefree
partner_config = input.get('release_partner_config')
if not partner_config and (release_enable_emefree
or release_enable_partners):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters['release_partner_config'] = fix_partner_config(
partner_config)
parameters['release_partners'] = input.get('release_partners')
if input.get('release_partner_build_number'):
parameters['release_partner_build_number'] = input[
'release_partner_build_number']
if input['version']:
parameters['version'] = input['version']
parameters['required_signoffs'] = get_required_signoffs(input, parameters)
parameters['signoff_urls'] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id,
task_id):
release_promotion_flavor = input["release_promotion_flavor"]
promotion_config = graph_config["release-promotion"]["flavors"][
release_promotion_flavor]
release_history = {}
product = promotion_config["product"]
next_version = str(input.get("next_version") or "")
if promotion_config.get("version-bump", False):
# We force str() the input, hence the 'None'
if next_version in ["", "None"]:
raise Exception(
"`next_version` property needs to be provided for `{}` "
"target.".format(release_promotion_flavor))
if promotion_config.get("partial-updates", False):
partial_updates = input.get("partial_updates", {})
if not partial_updates and release_level(
parameters["project"]) == "production":
raise Exception(
"`partial_updates` property needs to be provided for `{}`"
"target.".format(release_promotion_flavor))
balrog_prefix = product.title()
os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates,
sort_keys=True)
release_history = populate_release_history(
balrog_prefix,
parameters["project"],
partial_updates=partial_updates)
target_tasks_method = promotion_config["target-tasks-method"].format(
project=parameters["project"])
rebuild_kinds = input.get("rebuild_kinds",
promotion_config.get("rebuild-kinds", []))
do_not_optimize = input.get("do_not_optimize",
promotion_config.get("do-not-optimize", []))
# Build previous_graph_ids from ``previous_graph_ids``, ``revision``,
# or the action parameters.
previous_graph_ids = input.get("previous_graph_ids")
if not previous_graph_ids:
revision = input.get("revision")
if revision:
head_rev_param = "{}head_rev".format(
graph_config["project-repo-param-prefix"])
push_parameters = {
head_rev_param: revision,
"project": parameters["project"],
}
else:
push_parameters = parameters
previous_graph_ids = [
find_decision_task(push_parameters, graph_config)
]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds)
parameters["do_not_optimize"] = do_not_optimize
parameters["target_tasks_method"] = target_tasks_method
parameters["build_number"] = int(input["build_number"])
parameters["next_version"] = next_version
parameters["release_history"] = release_history
if promotion_config.get("is-rc"):
parameters["release_type"] += "-rc"
parameters["release_eta"] = input.get("release_eta", "")
parameters["release_product"] = product
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters["optimize_target_tasks"] = True
if release_promotion_flavor == "promote_firefox_partner_repack":
release_enable_partner_repack = True
release_enable_partner_attribution = False
release_enable_emefree = False
elif release_promotion_flavor == "promote_firefox_partner_attribution":
release_enable_partner_repack = False
release_enable_partner_attribution = True
release_enable_emefree = False
else:
# for promotion or ship phases, we use the action input to turn the repacks/attribution off
release_enable_partner_repack = input.get(
"release_enable_partner_repack", True)
release_enable_partner_attribution = input.get(
"release_enable_partner_attribution", True)
release_enable_emefree = input.get("release_enable_emefree", True)
partner_url_config = get_partner_url_config(parameters, graph_config)
if (release_enable_partner_repack
and not partner_url_config["release-partner-repack"]):
raise Exception(
"Can't enable partner repacks when no config url found")
if (release_enable_partner_attribution
and not partner_url_config["release-partner-attribution"]):
raise Exception(
"Can't enable partner attribution when no config url found")
if release_enable_emefree and not partner_url_config[
"release-eme-free-repack"]:
raise Exception(
"Can't enable EMEfree repacks when no config url found")
parameters["release_enable_partner_repack"] = release_enable_partner_repack
parameters[
"release_enable_partner_attribution"] = release_enable_partner_attribution
parameters["release_enable_emefree"] = release_enable_emefree
partner_config = input.get("release_partner_config")
if not partner_config and any([
release_enable_partner_repack,
release_enable_partner_attribution,
release_enable_emefree,
]):
github_token = get_token(parameters)
partner_config = get_partner_config(partner_url_config, github_token)
if partner_config:
parameters["release_partner_config"] = fix_partner_config(
partner_config)
parameters["release_partners"] = input.get("release_partners")
if input.get("release_partner_build_number"):
parameters["release_partner_build_number"] = input[
"release_partner_build_number"]
if input["version"]:
parameters["version"] = input["version"]
parameters["required_signoffs"] = get_required_signoffs(input, parameters)
parameters["signoff_urls"] = get_signoff_urls(input, parameters)
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, input, task_group_id, task_id, task):
release_promotion_flavor = input['release_promotion_flavor']
release_history = {}
desktop_release_type = None
next_version = str(input.get('next_version') or '')
if release_promotion_flavor in VERSION_BUMP_FLAVORS:
# We force str() the input, hence the 'None'
if next_version in ['', 'None']:
raise Exception(
"`next_version` property needs to be provided for %s "
"targets." % ', '.join(VERSION_BUMP_FLAVORS)
)
if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS:
desktop_release_type = input.get('desktop_release_type', None)
if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES:
raise Exception("`desktop_release_type` must be one of: %s" %
", ".join(VALID_DESKTOP_RELEASE_TYPES))
if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS:
partial_updates = json.dumps(input.get('partial_updates', {}))
if partial_updates == "{}":
raise Exception(
"`partial_updates` property needs to be provided for %s "
"targets." % ', '.join(PARTIAL_UPDATES_FLAVORS)
)
balrog_prefix = 'Firefox'
if desktop_release_type == 'devedition':
balrog_prefix = 'Devedition'
os.environ['PARTIAL_UPDATES'] = partial_updates
release_history = populate_release_history(
balrog_prefix, parameters['project'],
partial_updates=input['partial_updates']
)
if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS:
uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', []))
if partial_updates == "[]":
raise Exception(
"`uptake_monitoring_platforms` property needs to be provided for %s "
"targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS)
)
os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms
promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor]
target_tasks_method = input.get(
'target_tasks_method',
promotion_config['target_tasks_method'].format(project=parameters['project'])
)
rebuild_kinds = input.get(
'rebuild_kinds', promotion_config.get('rebuild_kinds', [])
)
do_not_optimize = input.get(
'do_not_optimize', promotion_config.get('do_not_optimize', [])
)
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
# or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
revision = input.get('revision')
parameters['pushlog_id'] = parameters['pushlog_id'] or \
find_hg_revision_pushlog_id(parameters, revision)
previous_graph_ids = [find_decision_task(parameters)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
parameters['next_version'] = next_version
parameters['release_history'] = release_history
parameters['desktop_release_type'] = desktop_release_type
parameters['release_eta'] = input.get('release_eta', '')
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({}, parameters=parameters)
def get_decision_parameters(options):
"""
Load parameters from the command-line options for 'taskgraph decision'.
This also applies per-project parameters, based on the given project.
"""
parameters = {n: options[n] for n in [
'base_repository',
'head_repository',
'head_rev',
'head_ref',
'message',
'project',
'pushlog_id',
'pushdate',
'owner',
'level',
'target_tasks_method',
] if n in options}
for n in (
'comm_base_repository',
'comm_head_repository',
'comm_head_rev',
'comm_head_ref',
):
if n in options and options[n] is not None:
parameters[n] = options[n]
# Define default filter list, as most configurations shouldn't need
# custom filters.
parameters['filters'] = [
'check_servo',
'target_tasks_method',
]
parameters['existing_tasks'] = {}
parameters['do_not_optimize'] = []
parameters['build_number'] = 1
parameters['next_version'] = None
parameters['desktop_release_type'] = ''
parameters['release_eta'] = ''
# owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
# case, fake it
if '@' not in parameters['owner']:
parameters['owner'] += '@noreply.mozilla.org'
# use the pushdate as build_date if given, else use current time
parameters['build_date'] = parameters['pushdate'] or int(time.time())
# moz_build_date is the build identifier based on build_date
parameters['moz_build_date'] = time.strftime("%Y%m%d%H%M%S",
time.gmtime(parameters['build_date']))
project = parameters['project']
try:
parameters.update(PER_PROJECT_PARAMETERS[project])
except KeyError:
logger.warning("using default project parameters; add {} to "
"PER_PROJECT_PARAMETERS in {} to customize behavior "
"for this project".format(project, __file__))
parameters.update(PER_PROJECT_PARAMETERS['default'])
# `target_tasks_method` has higher precedence than `project` parameters
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history('Firefox', project)
if options.get('try_task_config_file'):
task_config_file = os.path.abspath(options.get('try_task_config_file'))
else:
# if try_task_config.json is present, load it
task_config_file = os.path.join(os.getcwd(), 'try_task_config.json')
# load try settings
if 'try' in project:
parameters['try_mode'] = None
if os.path.isfile(task_config_file):
logger.info("using try tasks from {}".format(task_config_file))
parameters['try_mode'] = 'try_task_config'
with open(task_config_file, 'r') as fh:
parameters['try_task_config'] = json.load(fh)
else:
parameters['try_task_config'] = None
if 'try:' in parameters['message']:
parameters['try_mode'] = 'try_option_syntax'
args = parse_message(parameters['message'])
parameters['try_options'] = args
else:
parameters['try_options'] = None
if parameters['try_mode']:
# The user has explicitly requested a set of jobs, so run them all
# regardless of optimization. Their dependencies can be optimized,
# though.
parameters['optimize_target_tasks'] = False
else:
# For a try push with no task selection, apply the default optimization
# process to all of the tasks.
parameters['optimize_target_tasks'] = True
else:
parameters['try_mode'] = None
parameters['try_task_config'] = None
parameters['try_options'] = None
result = Parameters(**parameters)
result.check()
return result
