def generate_partials_builds(self, product, branch): from taskgraph.util.partials import populate_release_history try: import yaml release_history = {'release_history': populate_release_history(product, branch)} print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False)) except Exception: traceback.print_exc() sys.exit(1)
def generate_partials_builds(self, product, branch): from taskgraph.util.partials import populate_release_history try: import yaml release_history = {'release_history': populate_release_history(product, branch)} print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False)) except Exception: traceback.print_exc() sys.exit(1)
def get_decision_parameters(graph_config, parameters): logger.info("{}.get_decision_parameters called".format(__name__)) # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built project = parameters["project"] parameters.setdefault("release_history", dict()) if "nightly" in parameters.get("target_tasks_method", ""): parameters["release_history"] = populate_release_history( BALROG_PRODUCT, project)
def get_decision_parameters(options): """ Load parameters from the command-line options for 'taskgraph decision'. This also applies per-project parameters, based on the given project. """ parameters = { n: options[n] for n in [ 'base_repository', 'head_repository', 'head_rev', 'head_ref', 'message', 'project', 'pushlog_id', 'pushdate', 'owner', 'level', 'target_tasks_method', ] if n in options } for n in ( 'comm_base_repository', 'comm_head_repository', 'comm_head_rev', 'comm_head_ref', ): if n in options and options[n] is not None: parameters[n] = options[n] # Define default filter list, as most configurations shouldn't need # custom filters. parameters['filters'] = [ 'check_servo', 'target_tasks_method', ] parameters['existing_tasks'] = {} parameters['do_not_optimize'] = [] parameters['build_number'] = 1 parameters['version'] = get_version() parameters['app_version'] = get_app_version() parameters['next_version'] = None parameters['release_type'] = '' parameters['release_eta'] = '' parameters['release_enable_partners'] = False parameters['release_partners'] = [] parameters['release_partner_config'] = {} parameters['release_partner_build_number'] = 1 parameters['release_enable_emefree'] = False parameters['release_product'] = None # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which # case, fake it if '@' not in parameters['owner']: parameters['owner'] += '@noreply.mozilla.org' # use the pushdate as build_date if given, else use current time parameters['build_date'] = parameters['pushdate'] or int(time.time()) # moz_build_date is the build identifier based on build_date parameters['moz_build_date'] = time.strftime( "%Y%m%d%H%M%S", time.gmtime(parameters['build_date'])) project = parameters['project'] try: parameters.update(PER_PROJECT_PARAMETERS[project]) except KeyError: logger.warning("using default project parameters; add {} to " "PER_PROJECT_PARAMETERS in {} to customize behavior " "for this project".format(project, __file__)) parameters.update(PER_PROJECT_PARAMETERS['default']) # `target_tasks_method` has higher precedence than `project` parameters if options.get('target_tasks_method'): parameters['target_tasks_method'] = options['target_tasks_method'] # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built parameters.setdefault('release_history', dict()) if 'nightly' in parameters.get('target_tasks_method', ''): parameters['release_history'] = populate_release_history( 'Firefox', project) if options.get('try_task_config_file'): task_config_file = os.path.abspath(options.get('try_task_config_file')) else: # if try_task_config.json is present, load it task_config_file = os.path.join(os.getcwd(), 'try_task_config.json') # load try settings if 'try' in project: parameters['try_mode'] = None if os.path.isfile(task_config_file): logger.info("using try tasks from {}".format(task_config_file)) parameters['try_mode'] = 'try_task_config' with open(task_config_file, 'r') as fh: parameters['try_task_config'] = json.load(fh) else: parameters['try_task_config'] = None if 'try:' in parameters['message']: parameters['try_mode'] = 'try_option_syntax' args = parse_message(parameters['message']) parameters['try_options'] = args else: parameters['try_options'] = None if parameters['try_mode']: # The user has explicitly requested a set of jobs, so run them all # regardless of optimization. Their dependencies can be optimized, # though. parameters['optimize_target_tasks'] = False else: # For a try push with no task selection, apply the default optimization # process to all of the tasks. parameters['optimize_target_tasks'] = True else: parameters['try_mode'] = None parameters['try_task_config'] = None parameters['try_options'] = None result = Parameters(**parameters) result.check() return result
def get_decision_parameters(config, options): """ Load parameters from the command-line options for 'taskgraph decision'. This also applies per-project parameters, based on the given project. """ product_dir = config['product-dir'] parameters = { n: options[n] for n in [ 'base_repository', 'head_repository', 'head_rev', 'head_ref', 'message', 'project', 'pushlog_id', 'pushdate', 'owner', 'level', 'target_tasks_method', ] if n in options } for n in ( 'comm_base_repository', 'comm_head_repository', 'comm_head_rev', 'comm_head_ref', ): if n in options and options[n] is not None: parameters[n] = options[n] # Define default filter list, as most configurations shouldn't need # custom filters. parameters['filters'] = [ 'target_tasks_method', ] parameters['existing_tasks'] = {} parameters['do_not_optimize'] = [] parameters['build_number'] = 1 parameters['version'] = get_version(product_dir) parameters['app_version'] = get_app_version(product_dir) parameters['hg_branch'] = get_hg_revision_branch( GECKO, revision=parameters['head_rev']) parameters['next_version'] = None parameters['release_type'] = '' parameters['release_eta'] = '' parameters['release_enable_partners'] = False parameters['release_partners'] = [] parameters['release_partner_config'] = {} parameters['release_partner_build_number'] = 1 parameters['release_enable_emefree'] = False parameters['release_product'] = None parameters['required_signoffs'] = [] parameters['signoff_urls'] = {} parameters['try_mode'] = None parameters['try_task_config'] = None parameters['try_options'] = None # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which # case, fake it if '@' not in parameters['owner']: parameters['owner'] += '@noreply.mozilla.org' # use the pushdate as build_date if given, else use current time parameters['build_date'] = parameters['pushdate'] or int(time.time()) # moz_build_date is the build identifier based on build_date parameters['moz_build_date'] = time.strftime( "%Y%m%d%H%M%S", time.gmtime(parameters['build_date'])) project = parameters['project'] try: parameters.update(PER_PROJECT_PARAMETERS[project]) except KeyError: logger.warning("using default project parameters; add {} to " "PER_PROJECT_PARAMETERS in {} to customize behavior " "for this project".format(project, __file__)) parameters.update(PER_PROJECT_PARAMETERS['default']) # `target_tasks_method` has higher precedence than `project` parameters if options.get('target_tasks_method'): parameters['target_tasks_method'] = options['target_tasks_method'] # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built parameters.setdefault('release_history', dict()) if 'nightly' in parameters.get('target_tasks_method', ''): parameters['release_history'] = populate_release_history( 'Firefox', project) if options.get('try_task_config_file'): task_config_file = os.path.abspath(options.get('try_task_config_file')) else: # if try_task_config.json is present, load it task_config_file = os.path.join(os.getcwd(), 'try_task_config.json') # load try settings if 'try' in project: set_try_config(parameters, task_config_file) result = Parameters(**parameters) result.check() return result
def release_promotion_action(parameters, input, task_group_id, task_id, task): release_promotion_flavor = input['release_promotion_flavor'] promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor] release_history = {} product = promotion_config['product'] next_version = str(input.get('next_version') or '') if release_promotion_flavor in VERSION_BUMP_FLAVORS: # We force str() the input, hence the 'None' if next_version in ['', 'None']: raise Exception( "`next_version` property needs to be provided for %s " "targets." % ', '.join(VERSION_BUMP_FLAVORS)) if product in ('firefox', 'devedition'): if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS: partial_updates = json.dumps(input.get('partial_updates', {})) if partial_updates == "{}": raise Exception( "`partial_updates` property needs to be provided for %s " "targets." % ', '.join(PARTIAL_UPDATES_FLAVORS)) balrog_prefix = product.title() os.environ['PARTIAL_UPDATES'] = partial_updates release_history = populate_release_history( balrog_prefix, parameters['project'], partial_updates=input['partial_updates']) promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor] target_tasks_method = promotion_config['target_tasks_method'].format( project=parameters['project']) rebuild_kinds = input.get('rebuild_kinds', promotion_config.get('rebuild_kinds', [])) do_not_optimize = input.get('do_not_optimize', promotion_config.get('do_not_optimize', [])) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``, # or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: revision = input.get('revision') parameters['pushlog_id'] = parameters['pushlog_id'] or \ find_hg_revision_pushlog_id(parameters, revision) previous_graph_ids = [find_decision_task(parameters)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) parameters['next_version'] = next_version parameters['release_history'] = release_history parameters['release_type'] = promotion_config.get('release_type', '') parameters['release_eta'] = input.get('release_eta', '') if input['version']: parameters['version'] = input['version'] # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id, task): release_promotion_flavor = input['release_promotion_flavor'] promotion_config = graph_config['release-promotion']['flavors'][ release_promotion_flavor] release_history = {} product = promotion_config['product'] next_version = str(input.get('next_version') or '') if promotion_config.get('version-bump', False): # We force str() the input, hence the 'None' if next_version in ['', 'None']: raise Exception( "`next_version` property needs to be provided for `{}` " "target.".format(release_promotion_flavor)) if promotion_config.get('partial-updates', False): partial_updates = json.dumps(input.get('partial_updates', {})) if partial_updates == "{}": raise Exception( "`partial_updates` property needs to be provided for `{}`" "target.".format(release_promotion_flavor)) balrog_prefix = product.title() os.environ['PARTIAL_UPDATES'] = partial_updates release_history = populate_release_history( balrog_prefix, parameters['project'], partial_updates=input['partial_updates']) target_tasks_method = promotion_config['target-tasks-method'].format( project=parameters['project']) rebuild_kinds = input.get('rebuild_kinds', promotion_config.get('rebuild-kinds', [])) do_not_optimize = input.get('do_not_optimize', promotion_config.get('do-not-optimize', [])) release_enable_partners = input.get( 'release_enable_partners', parameters['project'] in PARTNER_BRANCHES and product in ('firefox', )) release_enable_emefree = input.get( 'release_enable_emefree', parameters['project'] in EMEFREE_BRANCHES and product in ('firefox', )) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``, # or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: revision = input.get('revision') parameters['pushlog_id'] = parameters['pushlog_id'] or \ find_hg_revision_pushlog_id(parameters, graph_config, revision) previous_graph_ids = [find_decision_task(parameters, graph_config)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) parameters['next_version'] = next_version parameters['release_history'] = release_history parameters['release_type'] = promotion_config.get('release-type', '') parameters['release_eta'] = input.get('release_eta', '') parameters['release_enable_partners'] = release_enable_partners parameters['release_partners'] = input.get('release_partners') parameters['release_enable_emefree'] = release_enable_emefree parameters['release_product'] = product # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters['optimize_target_tasks'] = True partner_config = input.get('release_partner_config') if not partner_config and (release_enable_emefree or release_enable_partners): partner_url_config = get_partner_url_config( parameters, graph_config, enable_emefree=release_enable_emefree, enable_partners=release_enable_partners) github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if input.get('release_partner_build_number'): parameters['release_partner_build_number'] = input[ 'release_partner_build_number'] if partner_config: parameters['release_partner_config'] = fix_partner_config( partner_config) if input['version']: parameters['version'] = input['version'] # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def get_decision_parameters(options): """ Load parameters from the command-line options for 'taskgraph decision'. This also applies per-project parameters, based on the given project. """ parameters = { n: options[n] for n in [ 'base_repository', 'head_repository', 'head_rev', 'head_ref', 'message', 'project', 'pushlog_id', 'pushdate', 'owner', 'level', 'target_tasks_method', ] if n in options } # Define default filter list, as most configurations shouldn't need # custom filters. parameters['filters'] = [ 'check_servo', 'target_tasks_method', ] parameters['target_task_labels'] = [] parameters['morph_templates'] = {} # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which # case, fake it if '@' not in parameters['owner']: parameters['owner'] += '@noreply.mozilla.org' # use the pushdate as build_date if given, else use current time parameters['build_date'] = parameters['pushdate'] or int(time.time()) # moz_build_date is the build identifier based on build_date parameters['moz_build_date'] = time.strftime( "%Y%m%d%H%M%S", time.gmtime(parameters['build_date'])) project = parameters['project'] try: parameters.update(PER_PROJECT_PARAMETERS[project]) except KeyError: logger.warning("using default project parameters; add {} to " "PER_PROJECT_PARAMETERS in {} to customize behavior " "for this project".format(project, __file__)) parameters.update(PER_PROJECT_PARAMETERS['default']) # morph_templates and target_task_labels are only used on try, so don't # bother loading them elsewhere task_config_file = os.path.join(GECKO, 'try_task_config.json') if project == 'try' and os.path.isfile(task_config_file): with open(task_config_file, 'r') as fh: task_config = json.load(fh) parameters['morph_templates'] = task_config.get('templates', {}) parameters['target_task_labels'] = task_config.get('tasks') # `target_tasks_method` has higher precedence than `project` parameters if options.get('target_tasks_method'): parameters['target_tasks_method'] = options['target_tasks_method'] # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built parameters.setdefault('release_history', dict()) if 'nightly' in parameters.get('target_tasks_method', ''): parameters['release_history'] = populate_release_history( 'Firefox', project) return Parameters(parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): release_promotion_flavor = input['release_promotion_flavor'] promotion_config = graph_config['release-promotion']['flavors'][ release_promotion_flavor] release_history = {} product = promotion_config['product'] next_version = str(input.get('next_version') or '') if promotion_config.get('version-bump', False): # We force str() the input, hence the 'None' if next_version in ['', 'None']: raise Exception( "`next_version` property needs to be provided for `{}` " "target.".format(release_promotion_flavor)) if promotion_config.get('partial-updates', False): partial_updates = input.get('partial_updates', {}) if not partial_updates and release_level( parameters['project']) == 'production': raise Exception( "`partial_updates` property needs to be provided for `{}`" "target.".format(release_promotion_flavor)) balrog_prefix = product.title() os.environ['PARTIAL_UPDATES'] = json.dumps(partial_updates) release_history = populate_release_history( balrog_prefix, parameters['project'], partial_updates=partial_updates) target_tasks_method = promotion_config['target-tasks-method'].format( project=parameters['project']) rebuild_kinds = input.get('rebuild_kinds', promotion_config.get('rebuild-kinds', [])) do_not_optimize = input.get('do_not_optimize', promotion_config.get('do-not-optimize', [])) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``, # or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: revision = input.get('revision') if not parameters['pushlog_id']: repo_param = '{}head_repository'.format( graph_config['project-repo-param-prefix']) push_info = find_hg_revision_push_info( repository=parameters[repo_param], revision=revision) parameters['pushlog_id'] = push_info['pushid'] previous_graph_ids = [find_decision_task(parameters, graph_config)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) parameters['next_version'] = next_version parameters['release_history'] = release_history if promotion_config.get('is-rc'): parameters['release_type'] += '-rc' parameters['release_eta'] = input.get('release_eta', '') parameters['release_product'] = product # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters['optimize_target_tasks'] = True # Partner/EMEfree are enabled by default when get_partner_url_config() returns a non-null url # The action input may override by sending False. It's an error to send True with no url found partner_url_config = get_partner_url_config(parameters, graph_config) release_enable_partners = partner_url_config[ 'release-partner-repack'] is not None release_enable_emefree = partner_url_config[ 'release-eme-free-repack'] is not None if input.get('release_enable_partners') is False: release_enable_partners = False elif input.get( 'release_enable_partners') is True and not release_enable_partners: raise Exception( "Can't enable partner repacks when no config url found") if input.get('release_enable_emefree') is False: release_enable_emefree = False elif input.get( 'release_enable_emefree') is True and not release_enable_emefree: raise Exception("Can't enable EMEfree when no config url found") parameters['release_enable_partners'] = release_enable_partners parameters['release_enable_emefree'] = release_enable_emefree partner_config = input.get('release_partner_config') if not partner_config and (release_enable_emefree or release_enable_partners): github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if partner_config: parameters['release_partner_config'] = fix_partner_config( partner_config) parameters['release_partners'] = input.get('release_partners') if input.get('release_partner_build_number'): parameters['release_partner_build_number'] = input[ 'release_partner_build_number'] if input['version']: parameters['version'] = input['version'] parameters['required_signoffs'] = get_required_signoffs(input, parameters) parameters['signoff_urls'] = get_signoff_urls(input, parameters) # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): release_promotion_flavor = input["release_promotion_flavor"] promotion_config = graph_config["release-promotion"]["flavors"][ release_promotion_flavor] release_history = {} product = promotion_config["product"] next_version = str(input.get("next_version") or "") if promotion_config.get("version-bump", False): # We force str() the input, hence the 'None' if next_version in ["", "None"]: raise Exception( "`next_version` property needs to be provided for `{}` " "target.".format(release_promotion_flavor)) if promotion_config.get("partial-updates", False): partial_updates = input.get("partial_updates", {}) if not partial_updates and release_level( parameters["project"]) == "production": raise Exception( "`partial_updates` property needs to be provided for `{}`" "target.".format(release_promotion_flavor)) balrog_prefix = product.title() os.environ["PARTIAL_UPDATES"] = json.dumps(partial_updates, sort_keys=True) release_history = populate_release_history( balrog_prefix, parameters["project"], partial_updates=partial_updates) target_tasks_method = promotion_config["target-tasks-method"].format( project=parameters["project"]) rebuild_kinds = input.get("rebuild_kinds", promotion_config.get("rebuild-kinds", [])) do_not_optimize = input.get("do_not_optimize", promotion_config.get("do-not-optimize", [])) # Build previous_graph_ids from ``previous_graph_ids``, ``revision``, # or the action parameters. previous_graph_ids = input.get("previous_graph_ids") if not previous_graph_ids: revision = input.get("revision") if revision: head_rev_param = "{}head_rev".format( graph_config["project-repo-param-prefix"]) push_parameters = { head_rev_param: revision, "project": parameters["project"], } else: push_parameters = parameters previous_graph_ids = [ find_decision_task(push_parameters, graph_config) ] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds) parameters["do_not_optimize"] = do_not_optimize parameters["target_tasks_method"] = target_tasks_method parameters["build_number"] = int(input["build_number"]) parameters["next_version"] = next_version parameters["release_history"] = release_history if promotion_config.get("is-rc"): parameters["release_type"] += "-rc" parameters["release_eta"] = input.get("release_eta", "") parameters["release_product"] = product # When doing staging releases on try, we still want to re-use tasks from # previous graphs. parameters["optimize_target_tasks"] = True if release_promotion_flavor == "promote_firefox_partner_repack": release_enable_partner_repack = True release_enable_partner_attribution = False release_enable_emefree = False elif release_promotion_flavor == "promote_firefox_partner_attribution": release_enable_partner_repack = False release_enable_partner_attribution = True release_enable_emefree = False else: # for promotion or ship phases, we use the action input to turn the repacks/attribution off release_enable_partner_repack = input.get( "release_enable_partner_repack", True) release_enable_partner_attribution = input.get( "release_enable_partner_attribution", True) release_enable_emefree = input.get("release_enable_emefree", True) partner_url_config = get_partner_url_config(parameters, graph_config) if (release_enable_partner_repack and not partner_url_config["release-partner-repack"]): raise Exception( "Can't enable partner repacks when no config url found") if (release_enable_partner_attribution and not partner_url_config["release-partner-attribution"]): raise Exception( "Can't enable partner attribution when no config url found") if release_enable_emefree and not partner_url_config[ "release-eme-free-repack"]: raise Exception( "Can't enable EMEfree repacks when no config url found") parameters["release_enable_partner_repack"] = release_enable_partner_repack parameters[ "release_enable_partner_attribution"] = release_enable_partner_attribution parameters["release_enable_emefree"] = release_enable_emefree partner_config = input.get("release_partner_config") if not partner_config and any([ release_enable_partner_repack, release_enable_partner_attribution, release_enable_emefree, ]): github_token = get_token(parameters) partner_config = get_partner_config(partner_url_config, github_token) if partner_config: parameters["release_partner_config"] = fix_partner_config( partner_config) parameters["release_partners"] = input.get("release_partners") if input.get("release_partner_build_number"): parameters["release_partner_build_number"] = input[ "release_partner_build_number"] if input["version"]: parameters["version"] = input["version"] parameters["required_signoffs"] = get_required_signoffs(input, parameters) parameters["signoff_urls"] = get_signoff_urls(input, parameters) # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({"root": graph_config.root_dir}, parameters=parameters)
def release_promotion_action(parameters, input, task_group_id, task_id, task): release_promotion_flavor = input['release_promotion_flavor'] release_history = {} desktop_release_type = None next_version = str(input.get('next_version') or '') if release_promotion_flavor in VERSION_BUMP_FLAVORS: # We force str() the input, hence the 'None' if next_version in ['', 'None']: raise Exception( "`next_version` property needs to be provided for %s " "targets." % ', '.join(VERSION_BUMP_FLAVORS) ) if release_promotion_flavor in DESKTOP_RELEASE_TYPE_FLAVORS: desktop_release_type = input.get('desktop_release_type', None) if desktop_release_type not in VALID_DESKTOP_RELEASE_TYPES: raise Exception("`desktop_release_type` must be one of: %s" % ", ".join(VALID_DESKTOP_RELEASE_TYPES)) if release_promotion_flavor in PARTIAL_UPDATES_FLAVORS: partial_updates = json.dumps(input.get('partial_updates', {})) if partial_updates == "{}": raise Exception( "`partial_updates` property needs to be provided for %s " "targets." % ', '.join(PARTIAL_UPDATES_FLAVORS) ) balrog_prefix = 'Firefox' if desktop_release_type == 'devedition': balrog_prefix = 'Devedition' os.environ['PARTIAL_UPDATES'] = partial_updates release_history = populate_release_history( balrog_prefix, parameters['project'], partial_updates=input['partial_updates'] ) if release_promotion_flavor in UPTAKE_MONITORING_PLATFORMS_FLAVORS: uptake_monitoring_platforms = json.dumps(input.get('uptake_monitoring_platforms', [])) if partial_updates == "[]": raise Exception( "`uptake_monitoring_platforms` property needs to be provided for %s " "targets." % ', '.join(UPTAKE_MONITORING_PLATFORMS_FLAVORS) ) os.environ['UPTAKE_MONITORING_PLATFORMS'] = uptake_monitoring_platforms promotion_config = RELEASE_PROMOTION_CONFIG[release_promotion_flavor] target_tasks_method = input.get( 'target_tasks_method', promotion_config['target_tasks_method'].format(project=parameters['project']) ) rebuild_kinds = input.get( 'rebuild_kinds', promotion_config.get('rebuild_kinds', []) ) do_not_optimize = input.get( 'do_not_optimize', promotion_config.get('do_not_optimize', []) ) # make parameters read-write parameters = dict(parameters) # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``, # or ``revision``. previous_graph_ids = input.get('previous_graph_ids') if not previous_graph_ids: revision = input.get('revision') parameters['pushlog_id'] = parameters['pushlog_id'] or \ find_hg_revision_pushlog_id(parameters, revision) previous_graph_ids = [find_decision_task(parameters)] # Download parameters from the first decision task parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") # Download and combine full task graphs from each of the previous_graph_ids. # Sometimes previous relpro action tasks will add tasks, like partials, # that didn't exist in the first full_task_graph, so combining them is # important. The rightmost graph should take precedence in the case of # conflicts. combined_full_task_graph = {} for graph_id in previous_graph_ids: full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") combined_full_task_graph.update(full_task_graph) _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( combined_full_task_graph, previous_graph_ids, rebuild_kinds ) parameters['do_not_optimize'] = do_not_optimize parameters['target_tasks_method'] = target_tasks_method parameters['build_number'] = int(input['build_number']) parameters['next_version'] = next_version parameters['release_history'] = release_history parameters['desktop_release_type'] = desktop_release_type parameters['release_eta'] = input.get('release_eta', '') # make parameters read-only parameters = Parameters(**parameters) taskgraph_decision({}, parameters=parameters)
def get_decision_parameters(options): """ Load parameters from the command-line options for 'taskgraph decision'. This also applies per-project parameters, based on the given project. """ parameters = {n: options[n] for n in [ 'base_repository', 'head_repository', 'head_rev', 'head_ref', 'message', 'project', 'pushlog_id', 'pushdate', 'owner', 'level', 'target_tasks_method', ] if n in options} for n in ( 'comm_base_repository', 'comm_head_repository', 'comm_head_rev', 'comm_head_ref', ): if n in options and options[n] is not None: parameters[n] = options[n] # Define default filter list, as most configurations shouldn't need # custom filters. parameters['filters'] = [ 'check_servo', 'target_tasks_method', ] parameters['existing_tasks'] = {} parameters['do_not_optimize'] = [] parameters['build_number'] = 1 parameters['next_version'] = None parameters['desktop_release_type'] = '' parameters['release_eta'] = '' # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which # case, fake it if '@' not in parameters['owner']: parameters['owner'] += '@noreply.mozilla.org' # use the pushdate as build_date if given, else use current time parameters['build_date'] = parameters['pushdate'] or int(time.time()) # moz_build_date is the build identifier based on build_date parameters['moz_build_date'] = time.strftime("%Y%m%d%H%M%S", time.gmtime(parameters['build_date'])) project = parameters['project'] try: parameters.update(PER_PROJECT_PARAMETERS[project]) except KeyError: logger.warning("using default project parameters; add {} to " "PER_PROJECT_PARAMETERS in {} to customize behavior " "for this project".format(project, __file__)) parameters.update(PER_PROJECT_PARAMETERS['default']) # `target_tasks_method` has higher precedence than `project` parameters if options.get('target_tasks_method'): parameters['target_tasks_method'] = options['target_tasks_method'] # If the target method is nightly, we should build partials. This means # knowing what has been released previously. # An empty release_history is fine, it just means no partials will be built parameters.setdefault('release_history', dict()) if 'nightly' in parameters.get('target_tasks_method', ''): parameters['release_history'] = populate_release_history('Firefox', project) if options.get('try_task_config_file'): task_config_file = os.path.abspath(options.get('try_task_config_file')) else: # if try_task_config.json is present, load it task_config_file = os.path.join(os.getcwd(), 'try_task_config.json') # load try settings if 'try' in project: parameters['try_mode'] = None if os.path.isfile(task_config_file): logger.info("using try tasks from {}".format(task_config_file)) parameters['try_mode'] = 'try_task_config' with open(task_config_file, 'r') as fh: parameters['try_task_config'] = json.load(fh) else: parameters['try_task_config'] = None if 'try:' in parameters['message']: parameters['try_mode'] = 'try_option_syntax' args = parse_message(parameters['message']) parameters['try_options'] = args else: parameters['try_options'] = None if parameters['try_mode']: # The user has explicitly requested a set of jobs, so run them all # regardless of optimization. Their dependencies can be optimized, # though. parameters['optimize_target_tasks'] = False else: # For a try push with no task selection, apply the default optimization # process to all of the tasks. parameters['optimize_target_tasks'] = True else: parameters['try_mode'] = None parameters['try_task_config'] = None parameters['try_options'] = None result = Parameters(**parameters) result.check() return result