def group(value_component_type, process_val_func) -> list[RankedValue]: # We group any values that are merged together, so that the history can reflect # merges vs. replacements in a useful way. E.g., if we merge [a, b] and [c], # and then replace it with [d, e], the history will contain: # - [d, e] (from command-line flag) # - [a, b, c] (from env var, from config) # And similarly for dicts. grouped: list[list[RankedValue]] = [[]] for ranked_val in ranked_vals: if ranked_val.value and ranked_val.value.action == value_component_type.REPLACE: grouped.append([]) grouped[-1].append(ranked_val) return [ RankedValue( grp[-1].rank, process_val_func( value_component_type.merge( rv.value for rv in grp if rv.value is not None).val), ", ".join(rv.details for rv in grp if rv.details), ) for grp in grouped if grp ]
def format_option(self, ohi: OptionHelpInfo) -> List[str]: """Format the help output for a single option. :param ohi: Extracted information for option to print :return: Formatted help text for this option """ def maybe_parens(s: Optional[str]) -> str: return f" ({s})" if s else "" def format_value(val: RankedValue, prefix: str, left_padding: str) -> List[str]: if isinstance(val.value, (list, dict)): val_lines = json.dumps(val.value, sort_keys=True, indent=4).split("\n") else: val_lines = [to_help_str(val.value)] val_lines[0] = f"{prefix}{val_lines[0]}" val_lines[-1] = f"{val_lines[-1]}{maybe_parens(val.details)}" val_lines = [self.maybe_cyan(f"{left_padding}{line}") for line in val_lines] return val_lines indent = " " arg_lines = [f" {self.maybe_magenta(args)}" for args in ohi.display_args] arg_lines.append(self.maybe_magenta(f" {ohi.env_var}")) arg_lines.append(self.maybe_magenta(f" {ohi.config_key}")) choices = "" if ohi.choices is None else f"one of: [{', '.join(ohi.choices)}]" choices_lines = [ f"{indent}{' ' if i != 0 else ''}{self.maybe_cyan(s)}" for i, s in enumerate(wrap(f"{choices}", 96)) ] default_lines = format_value(RankedValue(Rank.HARDCODED, ohi.default), "default: ", indent) if not ohi.value_history: # Should never happen, but this keeps mypy happy. raise ValueError("No value history - options not parsed.") final_val = ohi.value_history.final_value curr_value_lines = format_value(final_val, "current value: ", indent) interesting_ranked_values = [ rv for rv in reversed(ohi.value_history.ranked_values) if rv.rank not in (Rank.NONE, Rank.HARDCODED, final_val.rank) ] value_derivation_lines = [ line for rv in interesting_ranked_values for line in format_value(rv, "overrode: ", f"{indent} ") ] description_lines = ohi.help.splitlines() # wrap() returns [] for an empty line, but we want to emit those, hence the "or [line]". description_lines = [ f"{indent}{s}" for line in description_lines for s in wrap(line, 96) or [line] ] lines = [ *arg_lines, *choices_lines, *default_lines, *curr_value_lines, *value_derivation_lines, *description_lines, ] if ohi.deprecated_message: lines.append(self.maybe_red(f"{indent}{ohi.deprecated_message}.")) if ohi.removal_hint: lines.append(self.maybe_red(f"{indent}{ohi.removal_hint}")) return lines
def _compute_value(self, dest, kwargs, flag_vals): """Compute the value to use for an option. The source of the default value is chosen according to the ranking in RankedValue. """ is_fromfile = kwargs.get('fromfile', False) action = kwargs.get('action') if is_fromfile and action and action != 'append': raise ParseError('Cannot fromfile {} with an action ({}) in scope {}' .format(dest, action, self._scope)) config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope udest = dest.upper() if self._scope == GLOBAL_SCOPE: # For convenience, we allow three forms of env var for global scope options. # The fully-specified env var is PANTS_DEFAULT_FOO, which is uniform with PANTS_<SCOPE>_FOO # for all the other scopes. However we also allow simply PANTS_FOO. And if the option name # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of # PANTS_PANTS_WORKDIR or PANTS_DEFAULT_PANTS_WORKDIR. We take the first specified value we # find, in this order: PANTS_DEFAULT_FOO, PANTS_FOO, FOO. env_vars = ['PANTS_DEFAULT_{0}'.format(udest), 'PANTS_{0}'.format(udest)] if udest.startswith('PANTS_'): env_vars.append(udest) else: sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', config_section.upper()) env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)] value_type = self.str_to_bool if is_boolean_flag(kwargs) else kwargs.get('type', str) env_val_str = None if self._env: for env_var in env_vars: if env_var in self._env: env_val_str = self._env.get(env_var) break config_val_str = self._config.get(config_section, dest, default=None) config_source_file = self._config.get_source_for_option(config_section, dest) if config_source_file is not None: config_source_file = os.path.relpath(config_source_file) def expand(val_str): if is_fromfile and val_str and val_str.startswith('@') and not val_str.startswith('@@'): fromfile = val_str[1:] try: with open(fromfile) as fp: return fp.read().strip() except IOError as e: raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format( dest, self._scope_str(), fromfile, e)) else: # Support a literal @ for fromfile values via @@. return val_str[1:] if is_fromfile and val_str.startswith('@@') else val_str def parse_typed_list(val_str): return None if val_str is None else [value_type(x) for x in list_option(expand(val_str))] def parse_typed_item(val_str): return None if val_str is None else value_type(expand(val_str)) flag_val = None if flag_vals: if action == 'append': flag_val = [parse_typed_item(v) for v in flag_vals] elif len(flag_vals) > 1: raise ParseError('Multiple cmd line flags specified for option {} in {}'.format( dest, self._scope_str())) else: flag_val = parse_typed_item(flag_vals[0]) default, parse = ([], parse_typed_list) if action == 'append' else (None, parse_typed_item) config_val = parse(config_val_str) env_val = parse(env_val_str) hardcoded_val = kwargs.get('default') config_details = 'in {}'.format(config_source_file) if config_source_file else None # Note: ranked_vals is guaranteed to have at least one element, and none of the values # of any of its elements will be None. ranked_vals = list(reversed(list(RankedValue.prioritized_iter( flag_val, env_val, config_val, hardcoded_val, default)))) choices = kwargs.get('choices') for ranked_val in ranked_vals: details = config_details if ranked_val.rank == RankedValue.CONFIG else None self._option_tracker.record_option(scope=self._scope, option=dest, value=ranked_val.value, rank=ranked_val.rank, deprecation_version=kwargs.get('deprecated_version'), details=details) def check(val): if choices is not None and val is not None and val not in choices: raise ParseError('{} is not an allowed value for option {} in {}. ' 'Must be one of: {}'.format( val, dest, self._scope_str(), choices )) return val if action == 'append': merged_rank = ranked_vals[-1].rank merged_val = [check(val) for vals in ranked_vals for val in vals.value] return RankedValue(merged_rank, merged_val) else: map(lambda rv: check(rv.value), ranked_vals) return ranked_vals[-1]
def _compute_value(self, dest, kwargs, flag_val_strs): """Compute the value to use for an option. The source of the default value is chosen according to the ranking in RankedValue. """ # Helper function to convert a string to a value of the option's type. def to_value_type(val_str): if val_str is None: return None elif kwargs.get('type') == bool: return self._ensure_bool(val_str) else: return self._wrap_type(kwargs.get('type', str))(val_str) # Helper function to expand a fromfile=True value string, if needed. def expand(val_str): if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'): if val_str.startswith( '@@' ): # Support a literal @ for fromfile values via @@. return val_str[1:] else: fromfile = val_str[1:] try: with open(fromfile) as fp: return fp.read().strip() except IOError as e: raise self.FromfileError( 'Failed to read {} in {} from file {}: {}'.format( dest, self._scope_str(), fromfile, e)) else: return val_str # Get value from config files, and capture details about its derivation. config_details = None config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope config_val_str = expand( self._config.get(config_section, dest, default=None)) config_source_file = self._config.get_source_for_option( config_section, dest) if config_source_file is not None: config_source_file = os.path.relpath(config_source_file) config_details = 'in {}'.format(config_source_file) # Get value from environment, and capture details about its derivation. udest = dest.upper() if self._scope == GLOBAL_SCOPE: # For convenience, we allow three forms of env var for global scope options. # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO # for all the other scopes. However we also allow simply PANTS_FOO. And if the option name # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO. env_vars = [ 'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest) ] if udest.startswith('PANTS_'): env_vars.append(udest) else: sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub( '_', self._scope.upper()) env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)] env_val_str = None env_details = None if self._env: for env_var in env_vars: if env_var in self._env: env_val_str = expand(self._env.get(env_var)) env_details = 'from env var {}'.format(env_var) break # Get value from cmd-line flags. flag_vals = [to_value_type(expand(x)) for x in flag_val_strs] if is_list_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to []. flag_val = ListValueComponent.merge( flag_vals) if flag_vals else None elif is_dict_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to {}. flag_val = DictValueComponent.merge( flag_vals) if flag_vals else None elif len(flag_vals) > 1: raise ParseError( 'Multiple cmd line flags specified for option {} in {}'.format( dest, self._scope_str())) elif len(flag_vals) == 1: flag_val = flag_vals[0] else: flag_val = None # Rank all available values. # Note that some of these values may already be of the value type, but type conversion # is idempotent, so this is OK. values_to_rank = [ to_value_type(x) for x in [ flag_val, env_val_str, config_val_str, kwargs.get('default'), None ] ] # Note that ranked_vals will always have at least one element, and all elements will be # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list( reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) # Record info about the derivation of each of the values. for ranked_val in ranked_vals: if ranked_val.rank == RankedValue.CONFIG: details = config_details elif ranked_val.rank == RankedValue.ENVIRONMENT: details = env_details else: details = None self._option_tracker.record_option( scope=self._scope, option=dest, value=ranked_val.value, rank=ranked_val.rank, deprecation_version=kwargs.get('removal_version'), details=details) # Helper function to check various validity constraints on final option values. def check(val): if val is not None: choices = kwargs.get('choices') if choices is not None and val not in choices: raise ParseError( '`{}` is not an allowed value for option {} in {}. ' 'Must be one of: {}'.format(val, dest, self._scope_str(), choices)) elif kwargs.get( 'type') == dir_option and not os.path.isdir(val): raise ParseError( 'Directory value `{}` for option {} in {} does not exist.' .format(val, dest, self._scope_str())) elif kwargs.get( 'type') == file_option and not os.path.isfile(val): raise ParseError( 'File value `{}` for option {} in {} does not exist.'. format(val, dest, self._scope_str())) # Generate the final value from all available values, and check that it (or its members, # if a list) are in the set of allowed choices. if is_list_option(kwargs): merged_rank = ranked_vals[-1].rank merged_val = ListValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val merged_val = [ self._convert_member_type(kwargs.get('member_type', str), x) for x in merged_val ] map(check, merged_val) ret = RankedValue(merged_rank, merged_val) elif is_dict_option(kwargs): merged_rank = ranked_vals[-1].rank merged_val = DictValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val map(check, merged_val) ret = RankedValue(merged_rank, merged_val) else: ret = ranked_vals[-1] check(ret.value) # All done! return ret
def test_rsc_dep_for_scala_java_and_test_targets(self): self.set_options(workflow=RankedValue( value=RscCompile.JvmCompileWorkflowType.rsc_and_zinc, rank=RankedValue.CONFIG, )) self.init_dependencies_for_scala_libraries() scala_dep = self.make_target( 'scala/classpath:scala_dep', target_type=ScalaLibrary, sources=['com/example/Bar.scala'] ) java_target = self.make_target( 'java/classpath:java_lib', target_type=JavaLibrary, sources=['com/example/Foo.java'], dependencies=[scala_dep], tags={'use-compiler:zinc-only'} ) scala_target = self.make_target( 'scala/classpath:scala_lib', target_type=ScalaLibrary, sources=['com/example/Foo.scala'], dependencies=[scala_dep] ) test_target = self.make_target( 'scala/classpath:scala_test', target_type=JUnitTests, sources=['com/example/Test.scala'], dependencies=[scala_target], tags={'use-compiler:zinc-only'} ) with temporary_dir() as tmp_dir: invalid_targets = [java_target, scala_target, scala_dep, test_target] task = self.create_task_with_target_roots( target_roots=[java_target, scala_target, test_target] ) jobs = task._create_compile_jobs( compile_contexts=self.create_compile_contexts(invalid_targets, task, tmp_dir), invalid_targets=invalid_targets, invalid_vts=self.wrap_in_vts(invalid_targets), classpath_product=None) dependee_graph = self.construct_dependee_graph_str(jobs, task) self.assertEqual(dedent(""" zinc[zinc-java](java/classpath:java_lib) <- {} rsc(scala/classpath:scala_lib) <- { zinc[zinc-only](scala/classpath:scala_test) } zinc[rsc-and-zinc](scala/classpath:scala_lib) <- {} rsc(scala/classpath:scala_dep) <- { rsc(scala/classpath:scala_lib), zinc[rsc-and-zinc](scala/classpath:scala_lib), zinc[zinc-only](scala/classpath:scala_test) } zinc[rsc-and-zinc](scala/classpath:scala_dep) <- { zinc[zinc-java](java/classpath:java_lib) } zinc[zinc-only](scala/classpath:scala_test) <- {}""").strip(), dependee_graph)
def format_option(self, ohi: OptionHelpInfo) -> List[str]: """Format the help output for a single option. :param ohi: Extracted information for option to print :return: Formatted help text for this option """ def maybe_parens(s: Optional[str]) -> str: return f" ({s})" if s else "" def format_value(ranked_val: RankedValue, prefix: str, left_padding: str) -> List[str]: if isinstance(ranked_val.value, (list, dict)): is_enum_list = (isinstance(ranked_val.value, list) and len(ranked_val.value) > 0 and isinstance(ranked_val.value[0], Enum)) normalized_val = ([ enum_elmt.value for enum_elmt in ranked_val.value ] if is_enum_list else ranked_val.value) val_lines = json.dumps(normalized_val, sort_keys=True, indent=4).split("\n") else: val_lines = [to_help_str(ranked_val.value)] val_lines[0] = f"{prefix}{val_lines[0]}" val_lines[ -1] = f"{val_lines[-1]}{maybe_parens(ranked_val.details)}" val_lines = [ self.maybe_cyan(f"{left_padding}{line}") for line in val_lines ] return val_lines indent = " " arg_lines = [ f" {self.maybe_magenta(args)}" for args in ohi.display_args ] arg_lines.append(self.maybe_magenta(f" {ohi.env_var}")) arg_lines.append(self.maybe_magenta(f" {ohi.config_key}")) choices = "" if ohi.choices is None else f"one of: [{', '.join(ohi.choices)}]" choices_lines = [ f"{indent}{' ' if i != 0 else ''}{self.maybe_cyan(s)}" for i, s in enumerate(textwrap.wrap(f"{choices}", self._width)) ] default_lines = format_value(RankedValue(Rank.HARDCODED, ohi.default), "default: ", indent) if not ohi.value_history: # Should never happen, but this keeps mypy happy. raise ValueError("No value history - options not parsed.") final_val = ohi.value_history.final_value curr_value_lines = format_value(final_val, "current value: ", indent) interesting_ranked_values = [ rv for rv in reversed(ohi.value_history.ranked_values) if rv.rank not in (Rank.NONE, Rank.HARDCODED, final_val.rank) ] value_derivation_lines = [ line for rv in interesting_ranked_values for line in format_value(rv, "overrode: ", f"{indent} ") ] description_lines = hard_wrap(ohi.help, indent=len(indent), width=self._width) lines = [ *arg_lines, *choices_lines, *default_lines, *curr_value_lines, *value_derivation_lines, *description_lines, ] if ohi.deprecated_message: maybe_colorize = self.maybe_red if ohi.deprecation_active else self.maybe_yellow lines.append(maybe_colorize(f"{indent}{ohi.deprecated_message}")) if ohi.removal_hint: lines.append(maybe_colorize(f"{indent}{ohi.removal_hint}")) return lines
def _compute_value(self, dest, kwargs, flag_val_strs): """Compute the value to use for an option. The source of the default value is chosen according to the ranking in RankedValue. """ # Helper function to convert a string to a value of the option's type. def to_value_type(val_str): if val_str is None: return None if kwargs.get("type") == bool: return self._ensure_bool(val_str) type_arg = kwargs.get("type", str) try: return self._wrap_type(type_arg)(val_str) except (TypeError, ValueError) as e: raise ParseError( f"Error applying type '{type_arg.__name__}' to option value '{val_str}', for option " f"'--{dest}' in {self._scope_str()}: {e}") # Helper function to expand a fromfile=True value string, if needed. # May return a string or a dict/list decoded from a json/yaml file. def expand(val_or_str): if (kwargs.get("fromfile", True) and isinstance(val_or_str, str) and val_or_str.startswith("@")): if val_or_str.startswith( "@@" ): # Support a literal @ for fromfile values via @@. return val_or_str[1:] else: fromfile = val_or_str[1:] try: with open(fromfile, "r") as fp: s = fp.read().strip() if fromfile.endswith(".json"): return json.loads(s) elif fromfile.endswith( ".yml") or fromfile.endswith(".yaml"): return yaml.safe_load(s) else: return s except (IOError, ValueError, yaml.YAMLError) as e: raise FromfileError( f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}" ) else: return val_or_str # Get value from config files, and capture details about its derivation. config_details = None config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope config_default_val_or_str = expand( self._config.get(Config.DEFAULT_SECTION, dest, default=None)) config_val_or_str = expand( self._config.get(config_section, dest, default=None)) config_source_file = self._config.get_source_for_option( config_section, dest) or self._config.get_source_for_option( Config.DEFAULT_SECTION, dest) if config_source_file is not None: config_source_file = os.path.relpath(config_source_file) config_details = f"in {config_source_file}" # Get value from environment, and capture details about its derivation. udest = dest.upper() if self._scope == GLOBAL_SCOPE: # For convenience, we allow three forms of env var for global scope options. # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO # for all the other scopes. However we also allow simply PANTS_FOO. And if the option name # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO. env_vars = [f"PANTS_GLOBAL_{udest}", f"PANTS_{udest}"] if udest.startswith("PANTS_"): env_vars.append(udest) else: sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub( "_", self._scope.upper()) env_vars = [f"PANTS_{sanitized_env_var_scope}_{udest}"] env_val_or_str = None env_details = None if self._env: for env_var in env_vars: if env_var in self._env: env_val_or_str = expand(self._env.get(env_var)) env_details = f"from env var {env_var}" break # Get value from cmd-line flags. flag_vals = [to_value_type(expand(x)) for x in flag_val_strs] if is_list_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to []. flag_val = ListValueComponent.merge( flag_vals) if flag_vals else None elif is_dict_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to {}. flag_val = DictValueComponent.merge( flag_vals) if flag_vals else None elif len(flag_vals) > 1: raise ParseError( "Multiple cmd line flags specified for option {} in {}".format( dest, self._scope_str())) elif len(flag_vals) == 1: flag_val = flag_vals[0] else: flag_val = None # Rank all available values. # Note that some of these values may already be of the value type, but type conversion # is idempotent, so this is OK. values_to_rank = [ to_value_type(x) for x in [ flag_val, env_val_or_str, config_val_or_str, config_default_val_or_str, kwargs.get("default"), None, ] ] # Note that ranked_vals will always have at least one element, and all elements will be # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list( reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) def record_option(value, rank, option_details=None): deprecation_version = kwargs.get("removal_version") self._option_tracker.record_option( scope=self._scope, option=dest, value=value, rank=rank, deprecation_version=deprecation_version, details=option_details, ) # Record info about the derivation of each of the contributing values. detail_history = [] for ranked_val in ranked_vals: if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT): details = config_details elif ranked_val.rank == RankedValue.ENVIRONMENT: details = env_details else: details = None if details: detail_history.append(details) record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details) # Helper function to check various validity constraints on final option values. def check(val): if val is None: return choices = kwargs.get("choices") type_arg = kwargs.get("type") if choices is None and "type" in kwargs: if inspect.isclass(type_arg) and issubclass(type_arg, Enum): choices = list(type_arg) # TODO: convert this into an enum() pattern match! if choices is not None and val not in choices: raise ParseError( "`{}` is not an allowed value for option {} in {}. " "Must be one of: {}".format(val, dest, self._scope_str(), choices)) if type_arg == file_option: check_file_exists(val) if type_arg == dir_option: check_dir_exists(val) def check_file_exists(val) -> None: error_prefix = f"File value `{val}` for option `{dest}` in `{self._scope_str()}`" try: path = Path(val) path_with_buildroot = Path(get_buildroot(), val) except TypeError: raise ParseError( f"{error_prefix} cannot be parsed as a file path.") if not path.is_file() and not path_with_buildroot.is_file(): raise ParseError(f"{error_prefix} does not exist.") def check_dir_exists(val) -> None: error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`" try: path = Path(val) path_with_buildroot = Path(get_buildroot(), val) except TypeError: raise ParseError( f"{error_prefix} cannot be parsed as a directory path.") if not path.is_dir() and not path_with_buildroot.is_dir(): raise ParseError(f"{error_prefix} does not exist.") # Generate the final value from all available values, and check that it (or its members, # if a list) are in the set of allowed choices. if is_list_option(kwargs): merged_rank = ranked_vals[-1].rank merged_val = ListValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val # TODO: run `check()` for all elements of a list option too!!! merged_val = [ self._convert_member_type(kwargs.get("member_type", str), x) for x in merged_val ] if kwargs.get("member_type") == shell_str: merged_val = flatten_shlexed_list(merged_val) for val in merged_val: check(val) ret = RankedValue(merged_rank, merged_val) elif is_dict_option(kwargs): # TODO: convert `member_type` for dict values too! merged_rank = ranked_vals[-1].rank merged_val = DictValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val for val in merged_val: check(val) ret = RankedValue(merged_rank, merged_val) else: ret = ranked_vals[-1] check(ret.value) # Record info about the derivation of the final value. merged_details = ", ".join(detail_history) if detail_history else None record_option(value=ret.value, rank=ret.rank, option_details=merged_details) # All done! return ret
def execute(self): requested_compiler = JvmPlatform.global_instance().get_options( ).compiler if requested_compiler != self.compiler_name: return deprecated_conditional( lambda: requested_compiler == self.Compiler.ZINC, removal_version='1.20.0.dev0', entity_description='Requested a deprecated compiler: [{}].'.format( requested_compiler), hint_message='Compiler will be defaulted to [{}].'.format( self.compiler_name)) if requested_compiler == self.Compiler.ZINC and self.compiler_name == self.Compiler.RSC: # Issue a deprecation warning (above) and rewrite zinc to rsc, as zinc is being deprecated. JvmPlatform.global_instance().get_options().compiler = RankedValue( 0, self.compiler_name) elif requested_compiler != self.compiler_name: # If the requested compiler is not the one supported by this task, log and abort self.context.log.debug( 'Requested an unsupported compiler [{}], aborting'.format( requested_compiler)) return # In case we have no relevant targets and return early, create the requested product maps. self.create_empty_extra_products() relevant_targets = list(self.context.targets(predicate=self.select)) if not relevant_targets: return # Clone the compile_classpath to the runtime_classpath. classpath_product = self.create_runtime_classpath() fingerprint_strategy = DependencyContext.global_instance( ).create_fingerprint_strategy(classpath_product) # Note, JVM targets are validated (`vts.update()`) as they succeed. As a result, # we begin writing artifacts out to the cache immediately instead of waiting for # all targets to finish. with self.invalidated(relevant_targets, invalidate_dependents=True, fingerprint_strategy=fingerprint_strategy, topological_order=True) as invalidation_check: compile_contexts = { vt.target: self.create_compile_context(vt.target, vt.results_dir) for vt in invalidation_check.all_vts } self.do_compile( invalidation_check, compile_contexts, classpath_product, ) if not self.get_options().use_classpath_jars: # Once compilation has completed, replace the classpath entry for each target with # its jar'd representation. for ccs in compile_contexts.values(): cc = self.select_runtime_context(ccs) for conf in self._confs: classpath_product.remove_for_target( cc.target, [(conf, cc.classes_dir)]) classpath_product.add_for_target( cc.target, [(conf, cc.jar_file)])
def _test_outlining_dep_for_scala_java_and_test_targets(self, youtline): if youtline: workflow = RscCompile.JvmCompileWorkflowType.outline_and_zinc key_str = "outline" else: workflow = RscCompile.JvmCompileWorkflowType.rsc_and_zinc key_str = "rsc" self.set_options(workflow=RankedValue( value=workflow, rank=RankedValue.CONFIG, )) self.init_dependencies_for_scala_libraries() scala_dep = self.make_target( 'scala/classpath:scala_dep', target_type=ScalaLibrary, sources=['com/example/Bar.scala'] ) java_target = self.make_target( 'java/classpath:java_lib', target_type=JavaLibrary, sources=['com/example/Foo.java'], dependencies=[scala_dep], tags={'use-compiler:zinc-only'} ) scala_target = self.make_target( 'scala/classpath:scala_lib', target_type=ScalaLibrary, sources=['com/example/Foo.scala'], dependencies=[scala_dep] ) test_target = self.make_target( 'scala/classpath:scala_test', target_type=JUnitTests, sources=['com/example/Test.scala'], dependencies=[scala_target], tags={'use-compiler:zinc-only'} ) with temporary_dir(root_dir=self.build_root) as tmp_dir: invalid_targets = [java_target, scala_target, scala_dep, test_target] task = self.create_task_with_target_roots( target_roots=[java_target, scala_target, test_target] ) jobs = task._create_compile_jobs( compile_contexts=self.create_compile_contexts(invalid_targets, task, tmp_dir), invalid_targets=invalid_targets, invalid_vts=self.wrap_in_vts(invalid_targets), classpath_product=None) dependee_graph = self.construct_dependee_graph_str(jobs, task) self.maxDiff = None # Double curly braces {{}} because f-string self.assertEqual(dedent(f""" double_check_cache(java/classpath:java_lib) <- {{ zinc[zinc-java](java/classpath:java_lib) }} zinc[zinc-java](java/classpath:java_lib) <- {{ write_to_cache(java/classpath:java_lib) }} write_to_cache(java/classpath:java_lib) <- {{}} double_check_cache(scala/classpath:scala_lib) <- {{ {key_str}(scala/classpath:scala_lib), zinc[{key_str}-and-zinc](scala/classpath:scala_lib) }} {key_str}(scala/classpath:scala_lib) <- {{ write_to_cache(scala/classpath:scala_lib), double_check_cache(scala/classpath:scala_test), zinc[zinc-only](scala/classpath:scala_test) }} zinc[{key_str}-and-zinc](scala/classpath:scala_lib) <- {{ write_to_cache(scala/classpath:scala_lib) }} write_to_cache(scala/classpath:scala_lib) <- {{}} double_check_cache(scala/classpath:scala_dep) <- {{ {key_str}(scala/classpath:scala_dep), zinc[{key_str}-and-zinc](scala/classpath:scala_dep) }} {key_str}(scala/classpath:scala_dep) <- {{ double_check_cache(scala/classpath:scala_lib), {key_str}(scala/classpath:scala_lib), zinc[{key_str}-and-zinc](scala/classpath:scala_lib), write_to_cache(scala/classpath:scala_dep), double_check_cache(scala/classpath:scala_test), zinc[zinc-only](scala/classpath:scala_test) }} zinc[{key_str}-and-zinc](scala/classpath:scala_dep) <- {{ double_check_cache(java/classpath:java_lib), zinc[zinc-java](java/classpath:java_lib), write_to_cache(scala/classpath:scala_dep) }} write_to_cache(scala/classpath:scala_dep) <- {{}} double_check_cache(scala/classpath:scala_test) <- {{ zinc[zinc-only](scala/classpath:scala_test) }} zinc[zinc-only](scala/classpath:scala_test) <- {{ write_to_cache(scala/classpath:scala_test) }} write_to_cache(scala/classpath:scala_test) <- {{}} """).strip(), dependee_graph)
def _compute_value(self, dest, kwargs, flag_val_strs): """Compute the value to use for an option. The source of the default value is chosen according to the ranking in RankedValue. """ # Helper function to convert a string to a value of the option's type. def to_value_type(val_str): if val_str is None: return None elif kwargs.get('type') == bool: return self._ensure_bool(val_str) else: type_arg = kwargs.get('type', str) try: return self._wrap_type(type_arg)(val_str) except TypeError as e: raise ParseError( "Error applying type '{}' to option value '{}', for option '--{}' in {}: {}" .format(type_arg.__name__, val_str, dest, self._scope_str(), e)) # Helper function to expand a fromfile=True value string, if needed. # May return a string or a dict/list decoded from a json/yaml file. def expand(val_or_str): if (kwargs.get('fromfile', True) and val_or_str and isinstance(val_or_str, str) and val_or_str.startswith('@')): if val_or_str.startswith( '@@' ): # Support a literal @ for fromfile values via @@. return val_or_str[1:] else: fromfile = val_or_str[1:] try: with open(fromfile, 'r') as fp: s = fp.read().strip() if fromfile.endswith('.json'): return json.loads(s) elif fromfile.endswith( '.yml') or fromfile.endswith('.yaml'): return yaml.safe_load(s) else: return s except (IOError, ValueError, yaml.YAMLError) as e: raise self.FromfileError( 'Failed to read {} in {} from file {}: {}'.format( dest, self._scope_str(), fromfile, e)) else: return val_or_str # Get value from config files, and capture details about its derivation. config_details = None config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope config_default_val_or_str = expand( self._config.get(Config.DEFAULT_SECTION, dest, default=None)) config_val_or_str = expand( self._config.get(config_section, dest, default=None)) config_source_file = (self._config.get_source_for_option( config_section, dest) or self._config.get_source_for_option( Config.DEFAULT_SECTION, dest)) if config_source_file is not None: config_source_file = os.path.relpath(config_source_file) config_details = 'in {}'.format(config_source_file) # Get value from environment, and capture details about its derivation. udest = dest.upper() if self._scope == GLOBAL_SCOPE: # For convenience, we allow three forms of env var for global scope options. # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO # for all the other scopes. However we also allow simply PANTS_FOO. And if the option name # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO. env_vars = [ 'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest) ] if udest.startswith('PANTS_'): env_vars.append(udest) else: sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub( '_', self._scope.upper()) env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)] env_val_or_str = None env_details = None if self._env: for env_var in env_vars: if env_var in self._env: env_val_or_str = expand(self._env.get(env_var)) env_details = 'from env var {}'.format(env_var) break # Get value from cmd-line flags. flag_vals = [to_value_type(expand(x)) for x in flag_val_strs] if is_list_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to []. flag_val = ListValueComponent.merge( flag_vals) if flag_vals else None elif is_dict_option(kwargs): # Note: It's important to set flag_val to None if no flags were specified, so we can # distinguish between no flags set vs. explicit setting of the value to {}. flag_val = DictValueComponent.merge( flag_vals) if flag_vals else None elif len(flag_vals) > 1: raise ParseError( 'Multiple cmd line flags specified for option {} in {}'.format( dest, self._scope_str())) elif len(flag_vals) == 1: flag_val = flag_vals[0] else: flag_val = None # Rank all available values. # Note that some of these values may already be of the value type, but type conversion # is idempotent, so this is OK. values_to_rank = [ to_value_type(x) for x in [ flag_val, env_val_or_str, config_val_or_str, config_default_val_or_str, kwargs.get('default'), None ] ] # Note that ranked_vals will always have at least one element, and all elements will be # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list( reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) def record_option(value, rank, option_details=None): deprecation_version = kwargs.get('removal_version') self._option_tracker.record_option( scope=self._scope, option=dest, value=value, rank=rank, deprecation_version=deprecation_version, details=option_details) # Record info about the derivation of each of the contributing values. detail_history = [] for ranked_val in ranked_vals: if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT): details = config_details elif ranked_val.rank == RankedValue.ENVIRONMENT: details = env_details else: details = None if details: detail_history.append(details) record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details) # Helper function to check various validity constraints on final option values. def check(val): if val is not None: choices = kwargs.get('choices') # If the `type` argument has an `all_variants` attribute, use that as `choices` if not # already set. Using an attribute instead of checking a subclass allows `type` arguments # which are functions to have an implicit fallback `choices` set as well. if choices is None and 'type' in kwargs: type_arg = kwargs.get('type') if hasattr(type_arg, 'all_variants'): choices = list(type_arg.all_variants) # TODO: convert this into an enum() pattern match! if choices is not None and val not in choices: raise ParseError( '`{}` is not an allowed value for option {} in {}. ' 'Must be one of: {}'.format(val, dest, self._scope_str(), choices)) elif kwargs.get( 'type') == dir_option and not os.path.isdir(val): raise ParseError( 'Directory value `{}` for option {} in {} does not exist.' .format(val, dest, self._scope_str())) elif kwargs.get( 'type') == file_option and not os.path.isfile(val): raise ParseError( 'File value `{}` for option {} in {} does not exist.'. format(val, dest, self._scope_str())) # Generate the final value from all available values, and check that it (or its members, # if a list) are in the set of allowed choices. if is_list_option(kwargs): merged_rank = ranked_vals[-1].rank merged_val = ListValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val # TODO: run `check()` for all elements of a list option too!!! merged_val = [ self._convert_member_type(kwargs.get('member_type', str), x) for x in merged_val ] for val in merged_val: check(val) ret = RankedValue(merged_rank, merged_val) elif is_dict_option(kwargs): # TODO: convert `member_type` for dict values too! merged_rank = ranked_vals[-1].rank merged_val = DictValueComponent.merge( [rv.value for rv in ranked_vals if rv.value is not None]).val for val in merged_val: check(val) ret = RankedValue(merged_rank, merged_val) else: ret = ranked_vals[-1] check(ret.value) # Record info about the derivation of the final value. merged_details = ', '.join(detail_history) if detail_history else None record_option(value=ret.value, rank=ret.rank, option_details=merged_details) # All done! return ret
def do_test(expected_default, **kwargs): kwargs["default"] = RankedValue(Rank.HARDCODED, kwargs["default"]) assert expected_default == HelpInfoExtracter.compute_default(**kwargs)
def setUp(self): super().setUp() # We're tied tightly to pex implementation details here faking out a python binary that outputs # only one value no matter what arguments, environment or input stream it has attached. That # value is the interpreter identity which is a JSON dict with exactly the following keys: # binary, python_tag, abi_tag, platform_tag, version, supported_tags, env_markers. def fake_interpreter(python_tag: str, abi_tag: str, version: Tuple[int, int, int]): interpreter_dir = safe_mkdtemp() binary = os.path.join(interpreter_dir, "python") values = dict( binary=binary, python_tag=python_tag, abi_tag=abi_tag, platform_tag="", version=version, supported_tags=[], env_markers={}, ) id_str = json.dumps(values) with open(binary, "w") as fp: fp.write( dedent(f""" #!{PythonInterpreter.get().binary} from __future__ import print_function print({id_str!r}) """).strip()) chmod_plus_x(binary) return PythonInterpreter.from_binary(binary) # impl, abi, impl_version, major, minor, patch self.fake_interpreters = [ fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 77, 777)), fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 88, 888)), fake_interpreter(python_tag="ip", abi_tag="ip2", version=(2, 99, 999)), ] self.set_options_for_scope( PythonSetup.options_scope, interpreter_constraints=RankedValue(Rank.CONFIG, ["IronPython>=2.55"]), interpreter_search_paths=[ interpreter.binary for interpreter in self.fake_interpreters ], ) self.reqtgt = self.make_target( spec="req", target_type=PythonRequirementLibrary, requirements=[], ) self.tgt1 = self._fake_target("tgt1") self.tgt2 = self._fake_target("tgt2", compatibility=["IronPython>2.77.777"]) self.tgt3 = self._fake_target("tgt3", compatibility=["IronPython>2.88.888"]) self.tgt4 = self._fake_target("tgt4", compatibility=["IronPython<2.99.999"]) self.tgt20 = self._fake_target("tgt20", dependencies=[self.tgt2]) self.tgt30 = self._fake_target("tgt30", dependencies=[self.tgt3]) self.tgt40 = self._fake_target("tgt40", dependencies=[self.tgt4])
def test_is_python2(constraints, compatibilities): Subsystem.reset() init_subsystem(PythonSetup, {PythonSetup.options_scope: { 'interpreter_constraints': RankedValue(RankedValue.CONFIG, constraints)}}) assert is_python2(compatibilities, PythonSetup.global_instance())