def compute_default(kwargs) -> str:
        """Compute the default val for help display for an option registered with these kwargs."""
        ranked_default = kwargs.get("default")
        typ = kwargs.get("type", str)

        default = ranked_default.value if ranked_default else None
        if default is None:
            return "None"

        if is_list_option(kwargs):
            member_typ = kwargs.get("member_type", str)

            def member_str(val):
                return f"'{val}'" if member_typ == str else f"{val}"

            default_str = (
                f"\"[{', '.join([member_str(val) for val in default])}]\""
                if default else "[]")
        elif is_dict_option(kwargs):
            if default:
                default_str = "{{ {} }}".format(", ".join(
                    ["'{}': {}".format(k, v) for k, v in default.items()]))
            else:
                default_str = "{}"
        elif typ == str:
            default_str = default.replace("\n", " ")
        elif inspect.isclass(typ) and issubclass(typ, Enum):
            default_str = default.value
        else:
            default_str = str(default)
        return default_str
Esempio n. 2
0
    def get_fingerprintable_for_scope(self, scope):
        """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

    Fingerprintable options are options registered via a "fingerprint=True" kwarg.

    :API: public
    """
        pairs = []
        # Note that we iterate over options registered at `scope` and at all enclosing scopes, since
        # option-using code can read those values indirectly via its own OptionValueContainer, so
        # they can affect that code's output.
        registration_scope = scope
        while registration_scope is not None:
            parser = self._parser_hierarchy.get_parser_by_scope(
                registration_scope)
            # Sort the arguments, so that the fingerprint is consistent.
            for (_, kwargs) in sorted(parser.option_registrations_iter()):
                if kwargs.get(
                        'recursive') and not kwargs.get('recursive_root'):
                    continue  # We only need to fprint recursive options once.
                if kwargs.get('fingerprint') is not True:
                    continue
                # Note that we read the value from scope, even if the registration was on an enclosing
                # scope, to get the right value for recursive options (and because this mirrors what
                # option-using code does).
                val = self.for_scope(scope)[kwargs['dest']]
                # If we have a list then we delegate to the fingerprinting implementation of the members.
                if is_list_option(kwargs):
                    val_type = kwargs.get('member_type', str)
                else:
                    val_type = kwargs.get('type', str)
                pairs.append((val_type, val))
            registration_scope = (None if registration_scope == '' else
                                  enclosing_scope(registration_scope))
        return pairs
Esempio n. 3
0
    def get_fingerprintable_for_scope(
        self,
        scope: str,
        daemon_only: bool = False,
    ):
        """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

        Options are fingerprintable by default, but may be registered with "fingerprint=False".

        This method also searches enclosing options scopes of `bottom_scope` to determine the set of
        fingerprintable pairs.

        :param scope: The scope to gather fingerprintable options for.
        :param daemon_only: If true, only look at daemon=True options.
        """

        pairs = []
        parser = self.get_parser(scope)
        # Sort the arguments, so that the fingerprint is consistent.
        for (_, kwargs) in sorted(parser.option_registrations_iter()):
            if not kwargs.get("fingerprint", True):
                continue
            if daemon_only and not kwargs.get("daemon", False):
                continue
            val = self.for_scope(scope)[kwargs["dest"]]
            # If we have a list then we delegate to the fingerprinting implementation of the members.
            if is_list_option(kwargs):
                val_type = kwargs.get("member_type", str)
            else:
                val_type = kwargs.get("type", str)
            pairs.append((val_type, val))
        return pairs
Esempio n. 4
0
  def get_fingerprintable_for_scope(self, scope):
    """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

    Fingerprintable options are options registered via a "fingerprint=True" kwarg.

    :API: public
    """
    pairs = []
    # Note that we iterate over options registered at `scope` and at all enclosing scopes, since
    # option-using code can read those values indirectly via its own OptionValueContainer, so
    # they can affect that code's output.
    registration_scope = scope
    while registration_scope is not None:
      parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
      # Sort the arguments, so that the fingerprint is consistent.
      for (_, kwargs) in sorted(parser.option_registrations_iter()):
        if kwargs.get('recursive') and not kwargs.get('recursive_root'):
          continue  # We only need to fprint recursive options once.
        if kwargs.get('fingerprint') is not True:
          continue
        # Note that we read the value from scope, even if the registration was on an enclosing
        # scope, to get the right value for recursive options (and because this mirrors what
        # option-using code does).
        val = self.for_scope(scope)[kwargs['dest']]
        # If we have a list then we delegate to the fingerprinting implementation of the members.
        if is_list_option(kwargs):
          val_type = kwargs.get('member_type', str)
        else:
          val_type = kwargs.get('type', str)
        pairs.append((val_type, val))
      registration_scope = (None if registration_scope == ''
                            else enclosing_scope(registration_scope))
    return pairs
Esempio n. 5
0
    def get_fingerprintable_for_scope(
        self,
        bottom_scope: str,
        include_passthru: Optional[bool] = None,
        fingerprint_key: str = "fingerprint",
        invert: bool = False,
    ):
        """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

        Fingerprintable options are options registered via a "fingerprint=True" kwarg. This flag
        can be parameterized with `fingerprint_key` for special cases.

        This method also searches enclosing options scopes of `bottom_scope` to determine the set of
        fingerprintable pairs.

        :param bottom_scope: The scope to gather fingerprintable options for.
        :param include_passthru: Whether to include passthru args captured by `bottom_scope` in the
                                 fingerprintable options.
        :param fingerprint_key: The option kwarg to match against (defaults to 'fingerprint').
        :param invert: Whether or not to invert the boolean check for the fingerprint_key value.

        :API: public
        """

        deprecated_conditional(
            predicate=lambda: include_passthru is not None,
            removal_version="1.31.0.dev0",
            entity_description="get_fingerprintable_for_scope `include_passthru` arg",
            hint_message=(
                "passthru arguments are fingerprinted if their associated option value is."
            ),
        )

        fingerprint_default = bool(invert)
        pairs = []

        # Note that we iterate over options registered at `bottom_scope` and at all
        # enclosing scopes, since option-using code can read those values indirectly
        # via its own OptionValueContainer, so they can affect that code's output.
        for registration_scope in all_enclosing_scopes(bottom_scope):
            parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
            # Sort the arguments, so that the fingerprint is consistent.
            for (_, kwargs) in sorted(parser.option_registrations_iter()):
                if kwargs.get("recursive", False) and not kwargs.get("recursive_root", False):
                    continue  # We only need to fprint recursive options once.
                if not kwargs.get(fingerprint_key, fingerprint_default):
                    continue
                # Note that we read the value from scope, even if the registration was on an enclosing
                # scope, to get the right value for recursive options (and because this mirrors what
                # option-using code does).
                val = self.for_scope(bottom_scope)[kwargs["dest"]]
                # If we have a list then we delegate to the fingerprinting implementation of the members.
                if is_list_option(kwargs):
                    val_type = kwargs.get("member_type", str)
                else:
                    val_type = kwargs.get("type", str)
                pairs.append((val_type, val))
        return pairs
Esempio n. 6
0
 def merge_in_rank(vals):
     if not vals:
         return None
     expanded_vals = [to_value_type(expand(x)) for x in vals]
     if is_list_option(kwargs):
         return ListValueComponent.merge(expanded_vals)
     if is_dict_option(kwargs):
         return DictValueComponent.merge(expanded_vals)
     return expanded_vals[-1]  # Last value wins.
Esempio n. 7
0
    def get_fingerprintable_for_scope(self,
                                      scope,
                                      include_passthru=False,
                                      fingerprint_key=None,
                                      invert=False):
        """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

    Fingerprintable options are options registered via a "fingerprint=True" kwarg. This flag
    can be parameterized with `fingerprint_key` for special cases.

    :param str scope: The scope to gather fingerprintable options for.
    :param bool include_passthru: Whether to include passthru args captured by `scope` in the
                                  fingerprintable options.
    :param string fingerprint_key: The option kwarg to match against (defaults to 'fingerprint').
    :param bool invert: Whether or not to invert the boolean check for the fingerprint_key value.

    :API: public
    """
        fingerprint_key = fingerprint_key or 'fingerprint'
        fingerprint_default = False if invert else None
        pairs = []

        if include_passthru:
            # Passthru args can only be sent to outermost scopes so we gather them once here up-front.
            passthru_args = self.passthru_args_for_scope(scope)
            # NB: We can't sort passthru args, the underlying consumer may be order-sensitive.
            pairs.extend((str, passthru_arg) for passthru_arg in passthru_args)

        # Note that we iterate over options registered at `scope` and at all enclosing scopes, since
        # option-using code can read those values indirectly via its own OptionValueContainer, so
        # they can affect that code's output.
        registration_scope = scope
        while registration_scope is not None:
            parser = self._parser_hierarchy.get_parser_by_scope(
                registration_scope)
            # Sort the arguments, so that the fingerprint is consistent.
            for (_, kwargs) in sorted(parser.option_registrations_iter()):
                if kwargs.get(
                        'recursive') and not kwargs.get('recursive_root'):
                    continue  # We only need to fprint recursive options once.
                if kwargs.get(fingerprint_key,
                              fingerprint_default) is not (False if invert else
                                                           True):
                    continue
                # Note that we read the value from scope, even if the registration was on an enclosing
                # scope, to get the right value for recursive options (and because this mirrors what
                # option-using code does).
                val = self.for_scope(scope)[kwargs['dest']]
                # If we have a list then we delegate to the fingerprinting implementation of the members.
                if is_list_option(kwargs):
                    val_type = kwargs.get('member_type', str)
                else:
                    val_type = kwargs.get('type', str)
                pairs.append((val_type, val))
            registration_scope = (None if registration_scope == '' else
                                  enclosing_scope(registration_scope))
        return pairs
Esempio n. 8
0
    def compute_metavar(kwargs):
        """Compute the metavar to display in help for an option registered with these kwargs."""

        stringify = lambda t: HelpInfoExtracter.stringify_type(t)

        metavar = kwargs.get("metavar")
        if not metavar:
            if is_list_option(kwargs):
                member_typ = kwargs.get("member_type", str)
                metavar = stringify(member_typ)
                # In a cmd-line list literal, string members must be quoted.
                if member_typ == str:
                    metavar = f"'{metavar}'"
            elif is_dict_option(kwargs):
                metavar = f'"{stringify(dict)}"'
            else:
                metavar = stringify(kwargs.get("type", str))
        if is_list_option(kwargs):
            # For lists, the metavar (either explicit or deduced) is the representation
            # of a single list member, so we turn the help string into a list of those here.
            return f'"[{metavar}, {metavar}, ...]"'
        return metavar
Esempio n. 9
0
    def compute_default(**kwargs) -> Any:
        """Compute the default val for help display for an option registered with these kwargs.

        Returns a pair (default, stringified default suitable for display).
        """
        ranked_default = kwargs.get("default")
        fallback: Any = None
        if is_list_option(kwargs):
            fallback = []
        elif is_dict_option(kwargs):
            fallback = {}
        default = (ranked_default.value if ranked_default
                   and ranked_default.value is not None else fallback)
        return default
Esempio n. 10
0
  def get_fingerprintable_for_scope(self, bottom_scope, include_passthru=False,
                                    fingerprint_key=None, invert=False):
    """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

    Fingerprintable options are options registered via a "fingerprint=True" kwarg. This flag
    can be parameterized with `fingerprint_key` for special cases.

    This method also searches enclosing options scopes of `bottom_scope` to determine the set of
    fingerprintable pairs.

    :param str bottom_scope: The scope to gather fingerprintable options for.
    :param bool include_passthru: Whether to include passthru args captured by `bottom_scope` in the
                                  fingerprintable options.
    :param string fingerprint_key: The option kwarg to match against (defaults to 'fingerprint').
    :param bool invert: Whether or not to invert the boolean check for the fingerprint_key value.

    :API: public
    """
    fingerprint_key = fingerprint_key or 'fingerprint'
    fingerprint_default = bool(invert)
    pairs = []

    if include_passthru:
      # Passthru args can only be sent to outermost scopes so we gather them once here up-front.
      passthru_args = self.passthru_args_for_scope(bottom_scope)
      # NB: We can't sort passthru args, the underlying consumer may be order-sensitive.
      pairs.extend((str, pass_arg) for pass_arg in passthru_args)

    # Note that we iterate over options registered at `bottom_scope` and at all
    # enclosing scopes, since option-using code can read those values indirectly
    # via its own OptionValueContainer, so they can affect that code's output.
    for registration_scope in all_enclosing_scopes(bottom_scope):
      parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
      # Sort the arguments, so that the fingerprint is consistent.
      for (_, kwargs) in sorted(parser.option_registrations_iter()):
        if kwargs.get('recursive', False) and not kwargs.get('recursive_root', False):
          continue  # We only need to fprint recursive options once.
        if kwargs.get(fingerprint_key, fingerprint_default) is not True:
          continue
        # Note that we read the value from scope, even if the registration was on an enclosing
        # scope, to get the right value for recursive options (and because this mirrors what
        # option-using code does).
        val = self.for_scope(bottom_scope)[kwargs['dest']]
        # If we have a list then we delegate to the fingerprinting implementation of the members.
        if is_list_option(kwargs):
          val_type = kwargs.get('member_type', str)
        else:
          val_type = kwargs.get('type', str)
        pairs.append((val_type, val))
    return pairs
    def compute_metavar(kwargs):
        """Compute the metavar to display in help for an option registered with these kwargs."""
        def stringify(t: Type) -> str:
            if t == dict:
                return "{'key1': val1, 'key2': val2, ...}"
            return f"<{t.__name__}>"

        metavar = kwargs.get("metavar")
        if not metavar:
            if is_list_option(kwargs):
                member_typ = kwargs.get("member_type", str)
                metavar = stringify(member_typ)
                # In a cmd-line list literal, string members must be quoted.
                if member_typ == str:
                    metavar = f"'{metavar}'"
            elif is_dict_option(kwargs):
                metavar = f'"{stringify(dict)}"'
            else:
                metavar = stringify(kwargs.get("type", str))
        if is_list_option(kwargs):
            # For lists, the metavar (either explicit or deduced) is the representation
            # of a single list member, so we turn the help string into a list of those here.
            return f'"[{metavar}, {metavar}, ...]"'
        return metavar
Esempio n. 12
0
  def register(*args, **kwargs):
    option_name = Parser.parse_dest(*args, **kwargs)

    default = kwargs.get(b'default')
    if default is None:
      if kwargs.get(b'type') == bool:
        default = False
      if kwargs.get(b'type') == list:
        default = []
    defaults[option_name] = RankedValue(RankedValue.HARDCODED, default)

    fingerprint = kwargs.get(b'fingerprint', False)
    if fingerprint:
      if is_list_option(kwargs):
        val_type = kwargs.get(b'member_type', str)
      else:
        val_type = kwargs.get(b'type', str)
      fingerprintables[option_name] = val_type
Esempio n. 13
0
    def register(*args, **kwargs):
        option_name = Parser.parse_dest(*args, **kwargs)

        default = kwargs.get(b'default')
        if default is None:
            if kwargs.get(b'type') == bool:
                default = False
            if kwargs.get(b'type') == list:
                default = []
        defaults[option_name] = RankedValue(RankedValue.HARDCODED, default)

        fingerprint = kwargs.get(b'fingerprint', False)
        if fingerprint:
            if is_list_option(kwargs):
                val_type = kwargs.get(b'member_type', str)
            else:
                val_type = kwargs.get(b'type', str)
            fingerprintables[option_name] = val_type
Esempio n. 14
0
    def register(*args, **kwargs):
        _, option_dest = Parser.parse_name_and_dest(*args, **kwargs)

        default = kwargs.get("default")
        if default is None:
            if kwargs.get("type") == bool:
                default = False
            if kwargs.get("type") == list:
                default = []
        defaults[option_dest] = RankedValue(Rank.HARDCODED, default)

        fingerprint = kwargs.get("fingerprint", False)
        if fingerprint:
            if is_list_option(kwargs):
                val_type = kwargs.get("member_type", str)
            else:
                val_type = kwargs.get("type", str)
            fingerprintables[option_dest] = val_type
Esempio n. 15
0
  def get_fingerprintable_for_scope(self, scope, include_passthru=False):
    """Returns a list of fingerprintable (option type, option value) pairs for the given scope.

    Fingerprintable options are options registered via a "fingerprint=True" kwarg.

    :param str scope: The scope to gather fingerprintable options for.
    :param bool include_passthru: Whether to include passthru args captured by `scope` in the
                                  fingerprintable options.

    :API: public
    """
    pairs = []

    if include_passthru:
      # Passthru args can only be sent to outermost scopes so we gather them once here up-front.
      passthru_args = self.passthru_args_for_scope(scope)
      # NB: We can't sort passthru args, the underlying consumer may be order-sensitive.
      pairs.extend((str, passthru_arg) for passthru_arg in passthru_args)

    # Note that we iterate over options registered at `scope` and at all enclosing scopes, since
    # option-using code can read those values indirectly via its own OptionValueContainer, so
    # they can affect that code's output.
    registration_scope = scope
    while registration_scope is not None:
      parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
      # Sort the arguments, so that the fingerprint is consistent.
      for (_, kwargs) in sorted(parser.option_registrations_iter()):
        if kwargs.get('recursive') and not kwargs.get('recursive_root'):
          continue  # We only need to fprint recursive options once.
        if kwargs.get('fingerprint') is not True:
          continue
        # Note that we read the value from scope, even if the registration was on an enclosing
        # scope, to get the right value for recursive options (and because this mirrors what
        # option-using code does).
        val = self.for_scope(scope)[kwargs['dest']]
        # If we have a list then we delegate to the fingerprinting implementation of the members.
        if is_list_option(kwargs):
          val_type = kwargs.get('member_type', str)
        else:
          val_type = kwargs.get('type', str)
        pairs.append((val_type, val))
      registration_scope = (None if registration_scope == ''
                            else enclosing_scope(registration_scope))
    return pairs
Esempio n. 16
0
    def compute_default(**kwargs) -> Any:
        """Compute the default val for help display for an option registered with these kwargs."""
        # If the kwargs already determine a string representation of the default for use in help
        # messages, use that.
        default_help_repr = kwargs.get("default_help_repr")
        if default_help_repr is not None:
            return str(
                default_help_repr
            )  # Should already be a string, but might as well be safe.

        ranked_default = kwargs.get("default")
        fallback: Any = None
        if is_list_option(kwargs):
            fallback = []
        elif is_dict_option(kwargs):
            fallback = {}
        default = (ranked_default.value if ranked_default
                   and ranked_default.value is not None else fallback)
        return default
Esempio n. 17
0
    def _compute_value(self, dest, kwargs, flag_val_strs):
        """Compute the value to use for an option.

    The source of the default value is chosen according to the ranking in RankedValue.
    """

        # Helper function to convert a string to a value of the option's type.
        def to_value_type(val_str):
            if val_str is None:
                return None
            elif kwargs.get('type') == bool:
                return self._ensure_bool(val_str)
            else:
                type_arg = kwargs.get('type', str)
                try:
                    return self._wrap_type(type_arg)(val_str)
                except TypeError as e:
                    raise ParseError(
                        "Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
                        .format(type_arg.__name__, val_str, dest,
                                self._scope_str(), e))

        # Helper function to expand a fromfile=True value string, if needed.
        # May return a string or a dict/list decoded from a json/yaml file.
        def expand(val_or_str):
            if (kwargs.get('fromfile', True) and val_or_str
                    and isinstance(val_or_str, str)
                    and val_or_str.startswith('@')):
                if val_or_str.startswith(
                        '@@'
                ):  # Support a literal @ for fromfile values via @@.
                    return val_or_str[1:]
                else:
                    fromfile = val_or_str[1:]
                    try:
                        with open(fromfile, 'r') as fp:
                            s = fp.read().strip()
                            if fromfile.endswith('.json'):
                                return json.loads(s)
                            elif fromfile.endswith(
                                    '.yml') or fromfile.endswith('.yaml'):
                                return yaml.safe_load(s)
                            else:
                                return s
                    except (IOError, ValueError, yaml.YAMLError) as e:
                        raise self.FromfileError(
                            'Failed to read {} in {} from file {}: {}'.format(
                                dest, self._scope_str(), fromfile, e))
            else:
                return val_or_str

        # Get value from config files, and capture details about its derivation.
        config_details = None
        config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
        config_default_val_or_str = expand(
            self._config.get(Config.DEFAULT_SECTION, dest, default=None))
        config_val_or_str = expand(
            self._config.get(config_section, dest, default=None))
        config_source_file = (self._config.get_source_for_option(
            config_section, dest) or self._config.get_source_for_option(
                Config.DEFAULT_SECTION, dest))
        if config_source_file is not None:
            config_source_file = os.path.relpath(config_source_file)
            config_details = 'in {}'.format(config_source_file)

        # Get value from environment, and capture details about its derivation.
        udest = dest.upper()
        if self._scope == GLOBAL_SCOPE:
            # For convenience, we allow three forms of env var for global scope options.
            # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
            # for all the other scopes.  However we also allow simply PANTS_FOO. And if the option name
            # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
            # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
            # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
            env_vars = [
                'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)
            ]
            if udest.startswith('PANTS_'):
                env_vars.append(udest)
        else:
            sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
                '_', self._scope.upper())
            env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]

        env_val_or_str = None
        env_details = None
        if self._env:
            for env_var in env_vars:
                if env_var in self._env:
                    env_val_or_str = expand(self._env.get(env_var))
                    env_details = 'from env var {}'.format(env_var)
                    break

        # Get value from cmd-line flags.
        flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
        if is_list_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to [].
            flag_val = ListValueComponent.merge(
                flag_vals) if flag_vals else None
        elif is_dict_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to {}.
            flag_val = DictValueComponent.merge(
                flag_vals) if flag_vals else None
        elif len(flag_vals) > 1:
            raise ParseError(
                'Multiple cmd line flags specified for option {} in {}'.format(
                    dest, self._scope_str()))
        elif len(flag_vals) == 1:
            flag_val = flag_vals[0]
        else:
            flag_val = None

        # Rank all available values.
        # Note that some of these values may already be of the value type, but type conversion
        # is idempotent, so this is OK.

        values_to_rank = [
            to_value_type(x) for x in [
                flag_val, env_val_or_str, config_val_or_str,
                config_default_val_or_str,
                kwargs.get('default'), None
            ]
        ]
        # Note that ranked_vals will always have at least one element, and all elements will be
        # instances of RankedValue (so none will be None, although they may wrap a None value).
        ranked_vals = list(
            reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

        def record_option(value, rank, option_details=None):
            deprecation_version = kwargs.get('removal_version')
            self._option_tracker.record_option(
                scope=self._scope,
                option=dest,
                value=value,
                rank=rank,
                deprecation_version=deprecation_version,
                details=option_details)

        # Record info about the derivation of each of the contributing values.
        detail_history = []
        for ranked_val in ranked_vals:
            if ranked_val.rank in (RankedValue.CONFIG,
                                   RankedValue.CONFIG_DEFAULT):
                details = config_details
            elif ranked_val.rank == RankedValue.ENVIRONMENT:
                details = env_details
            else:
                details = None
            if details:
                detail_history.append(details)
            record_option(value=ranked_val.value,
                          rank=ranked_val.rank,
                          option_details=details)

        # Helper function to check various validity constraints on final option values.
        def check(val):
            if val is not None:
                choices = kwargs.get('choices')
                # If the `type` argument has an `all_variants` attribute, use that as `choices` if not
                # already set. Using an attribute instead of checking a subclass allows `type` arguments
                # which are functions to have an implicit fallback `choices` set as well.
                if choices is None and 'type' in kwargs:
                    type_arg = kwargs.get('type')
                    if hasattr(type_arg, 'all_variants'):
                        choices = list(type_arg.all_variants)
                # TODO: convert this into an enum() pattern match!
                if choices is not None and val not in choices:
                    raise ParseError(
                        '`{}` is not an allowed value for option {} in {}. '
                        'Must be one of: {}'.format(val, dest,
                                                    self._scope_str(),
                                                    choices))
                elif kwargs.get(
                        'type') == dir_option and not os.path.isdir(val):
                    raise ParseError(
                        'Directory value `{}` for option {} in {} does not exist.'
                        .format(val, dest, self._scope_str()))
                elif kwargs.get(
                        'type') == file_option and not os.path.isfile(val):
                    raise ParseError(
                        'File value `{}` for option {} in {} does not exist.'.
                        format(val, dest, self._scope_str()))

        # Generate the final value from all available values, and check that it (or its members,
        # if a list) are in the set of allowed choices.
        if is_list_option(kwargs):
            merged_rank = ranked_vals[-1].rank
            merged_val = ListValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            # TODO: run `check()` for all elements of a list option too!!!
            merged_val = [
                self._convert_member_type(kwargs.get('member_type', str), x)
                for x in merged_val
            ]
            for val in merged_val:
                check(val)
            ret = RankedValue(merged_rank, merged_val)
        elif is_dict_option(kwargs):
            # TODO: convert `member_type` for dict values too!
            merged_rank = ranked_vals[-1].rank
            merged_val = DictValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            for val in merged_val:
                check(val)
            ret = RankedValue(merged_rank, merged_val)
        else:
            ret = ranked_vals[-1]
            check(ret.value)

        # Record info about the derivation of the final value.
        merged_details = ', '.join(detail_history) if detail_history else None
        record_option(value=ret.value,
                      rank=ret.rank,
                      option_details=merged_details)

        # All done!
        return ret
Esempio n. 18
0
    def get_option_help_info(self, args, kwargs):
        """Returns an OptionHelpInfo for the option registered with the given (args, kwargs)."""
        display_args = []
        scoped_cmd_line_args = []
        unscoped_cmd_line_args = []

        for arg in args:
            is_short_arg = len(arg) == 2
            unscoped_cmd_line_args.append(arg)
            if self._scope_prefix:
                scoped_arg = '--{}-{}'.format(self._scope_prefix,
                                              arg.lstrip('-'))
            else:
                scoped_arg = arg
            scoped_cmd_line_args.append(scoped_arg)

            if kwargs.get('type') == bool:
                if is_short_arg:
                    display_args.append(scoped_arg)
                else:
                    unscoped_cmd_line_args.append('--no-{}'.format(arg[2:]))
                    scoped_cmd_line_args.append('--no-{}'.format(
                        scoped_arg[2:]))
                    display_args.append('--[no-]{}'.format(scoped_arg[2:]))
            else:
                metavar = self.compute_metavar(kwargs)
                display_arg = '{}={}'.format(scoped_arg, metavar)
                if is_list_option(kwargs):
                    # Show the multi-arg append form.
                    display_args.append('{arg_str} ({arg_str}) ...'.format(
                        arg_str=display_arg))
                    # Also show the list literal form, both with and without the append operator.
                    if metavar.startswith('"') and metavar.endswith('"'):
                        # We quote the entire list literal, so we shouldn't quote the individual members.
                        metavar = metavar[1:-1]
                    display_args.append(
                        '{arg}="[{metavar}, {metavar}, ...]"'.format(
                            arg=scoped_arg, metavar=metavar))
                    display_args.append(
                        '{arg}="+[{metavar}, {metavar}, ...]"'.format(
                            arg=scoped_arg, metavar=metavar))
                else:
                    display_args.append(display_arg)

        typ = kwargs.get('type', str)
        default = self.compute_default(kwargs)
        help_msg = kwargs.get('help', 'No help available.')
        removal_version = kwargs.get('removal_version')
        deprecated_message = None
        if removal_version:
            deprecated_tense = deprecated.get_deprecated_tense(removal_version)
            deprecated_message = 'DEPRECATED. {} removed in version: {}'.format(
                deprecated_tense, removal_version)
        removal_hint = kwargs.get('removal_hint')
        choices = ', '.join(
            str(choice) for choice in kwargs.get('choices', [])) or None

        ret = OptionHelpInfo(registering_class=kwargs.get(
            'registering_class', type(None)),
                             display_args=display_args,
                             scoped_cmd_line_args=scoped_cmd_line_args,
                             unscoped_cmd_line_args=unscoped_cmd_line_args,
                             typ=typ,
                             default=default,
                             help=help_msg,
                             deprecated_message=deprecated_message,
                             removal_version=removal_version,
                             removal_hint=removal_hint,
                             choices=choices)
        return ret
Esempio n. 19
0
    def _compute_value(self, dest, kwargs, flag_val_strs):
        """Compute the value to use for an option.

    The source of the default value is chosen according to the ranking in RankedValue.
    """

        # Helper function to convert a string to a value of the option's type.
        def to_value_type(val_str):
            if val_str is None:
                return None
            elif kwargs.get('type') == bool:
                return self._ensure_bool(val_str)
            else:
                return self._wrap_type(kwargs.get('type', str))(val_str)

        # Helper function to expand a fromfile=True value string, if needed.
        def expand(val_str):
            if kwargs.get('fromfile',
                          False) and val_str and val_str.startswith('@'):
                if val_str.startswith(
                        '@@'
                ):  # Support a literal @ for fromfile values via @@.
                    return val_str[1:]
                else:
                    fromfile = val_str[1:]
                    try:
                        with open(fromfile) as fp:
                            return fp.read().strip()
                    except IOError as e:
                        raise self.FromfileError(
                            'Failed to read {} in {} from file {}: {}'.format(
                                dest, self._scope_str(), fromfile, e))
            else:
                return val_str

        # Get value from config files, and capture details about its derivation.
        config_details = None
        config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
        config_val_str = expand(
            self._config.get(config_section, dest, default=None))
        config_source_file = self._config.get_source_for_option(
            config_section, dest)
        if config_source_file is not None:
            config_source_file = os.path.relpath(config_source_file)
            config_details = 'in {}'.format(config_source_file)

        # Get value from environment, and capture details about its derivation.
        udest = dest.upper()
        if self._scope == GLOBAL_SCOPE:
            # For convenience, we allow three forms of env var for global scope options.
            # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
            # for all the other scopes.  However we also allow simply PANTS_FOO. And if the option name
            # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
            # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
            # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
            env_vars = [
                'PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)
            ]
            if udest.startswith('PANTS_'):
                env_vars.append(udest)
        else:
            sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
                '_', self._scope.upper())
            env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]

        env_val_str = None
        env_details = None
        if self._env:
            for env_var in env_vars:
                if env_var in self._env:
                    env_val_str = expand(self._env.get(env_var))
                    env_details = 'from env var {}'.format(env_var)
                    break

        # Get value from cmd-line flags.
        flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
        if is_list_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to [].
            flag_val = ListValueComponent.merge(
                flag_vals) if flag_vals else None
        elif is_dict_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to {}.
            flag_val = DictValueComponent.merge(
                flag_vals) if flag_vals else None
        elif len(flag_vals) > 1:
            raise ParseError(
                'Multiple cmd line flags specified for option {} in {}'.format(
                    dest, self._scope_str()))
        elif len(flag_vals) == 1:
            flag_val = flag_vals[0]
        else:
            flag_val = None

        # Rank all available values.
        # Note that some of these values may already be of the value type, but type conversion
        # is idempotent, so this is OK.

        values_to_rank = [
            to_value_type(x) for x in [
                flag_val, env_val_str, config_val_str,
                kwargs.get('default'), None
            ]
        ]
        # Note that ranked_vals will always have at least one element, and all elements will be
        # instances of RankedValue (so none will be None, although they may wrap a None value).
        ranked_vals = list(
            reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

        # Record info about the derivation of each of the values.
        for ranked_val in ranked_vals:
            if ranked_val.rank == RankedValue.CONFIG:
                details = config_details
            elif ranked_val.rank == RankedValue.ENVIRONMENT:
                details = env_details
            else:
                details = None
            self._option_tracker.record_option(
                scope=self._scope,
                option=dest,
                value=ranked_val.value,
                rank=ranked_val.rank,
                deprecation_version=kwargs.get('removal_version'),
                details=details)

        # Helper function to check various validity constraints on final option values.
        def check(val):
            if val is not None:
                choices = kwargs.get('choices')
                if choices is not None and val not in choices:
                    raise ParseError(
                        '`{}` is not an allowed value for option {} in {}. '
                        'Must be one of: {}'.format(val, dest,
                                                    self._scope_str(),
                                                    choices))
                elif kwargs.get(
                        'type') == dir_option and not os.path.isdir(val):
                    raise ParseError(
                        'Directory value `{}` for option {} in {} does not exist.'
                        .format(val, dest, self._scope_str()))
                elif kwargs.get(
                        'type') == file_option and not os.path.isfile(val):
                    raise ParseError(
                        'File value `{}` for option {} in {} does not exist.'.
                        format(val, dest, self._scope_str()))

        # Generate the final value from all available values, and check that it (or its members,
        # if a list) are in the set of allowed choices.
        if is_list_option(kwargs):
            merged_rank = ranked_vals[-1].rank
            merged_val = ListValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            merged_val = [
                self._convert_member_type(kwargs.get('member_type', str), x)
                for x in merged_val
            ]
            map(check, merged_val)
            ret = RankedValue(merged_rank, merged_val)
        elif is_dict_option(kwargs):
            merged_rank = ranked_vals[-1].rank
            merged_val = DictValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            map(check, merged_val)
            ret = RankedValue(merged_rank, merged_val)
        else:
            ret = ranked_vals[-1]
            check(ret.value)

        # All done!
        return ret
Esempio n. 20
0
  def _compute_value(self, dest, kwargs, flag_val_strs):
    """Compute the value to use for an option.

    The source of the default value is chosen according to the ranking in RankedValue.
    """
    # Helper function to convert a string to a value of the option's type.
    def to_value_type(val_str):
      if val_str is None:
        return None
      elif kwargs.get('type') == bool:
        return self._ensure_bool(val_str)
      else:
        return self._wrap_type(kwargs.get('type', str))(val_str)

    # Helper function to expand a fromfile=True value string, if needed.
    def expand(val_str):
      if kwargs.get('fromfile', False) and val_str and val_str.startswith('@'):
        if val_str.startswith('@@'):   # Support a literal @ for fromfile values via @@.
          return val_str[1:]
        else:
          fromfile = val_str[1:]
          try:
            with open(fromfile) as fp:
              return fp.read().strip()
          except IOError as e:
            raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
                dest, self._scope_str(), fromfile, e))
      else:
        return val_str

    # Get value from config files, and capture details about its derivation.
    config_details = None
    config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
    config_val_str = expand(self._config.get(config_section, dest, default=None))
    config_source_file = self._config.get_source_for_option(config_section, dest)
    if config_source_file is not None:
      config_source_file = os.path.relpath(config_source_file)
      config_details = 'in {}'.format(config_source_file)

    # Get value from environment, and capture details about its derivation.
    udest = dest.upper()
    if self._scope == GLOBAL_SCOPE:
      # For convenience, we allow three forms of env var for global scope options.
      # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
      # for all the other scopes.  However we also allow simply PANTS_FOO. And if the option name
      # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
      # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
      # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
      env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
      if udest.startswith('PANTS_'):
        env_vars.append(udest)
    else:
      sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
      env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]

    env_val_str = None
    env_details = None
    if self._env:
      for env_var in env_vars:
        if env_var in self._env:
          env_val_str = expand(self._env.get(env_var))
          env_details = 'from env var {}'.format(env_var)
          break

    # Get value from cmd-line flags.
    flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
    if is_list_option(kwargs):
      # Note: It's important to set flag_val to None if no flags were specified, so we can
      # distinguish between no flags set vs. explicit setting of the value to [].
      flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
    elif len(flag_vals) > 1:
      raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
          dest, self._scope_str()))
    elif len(flag_vals) == 1:
      flag_val = flag_vals[0]
    else:
      flag_val = None

    # Rank all available values.
    # Note that some of these values may already be of the value type, but type conversion
    # is idempotent, so this is OK.

    values_to_rank = [to_value_type(x) for x in
                      [flag_val, env_val_str, config_val_str, kwargs.get('default'), None]]
    # Note that ranked_vals will always have at least one element, and all elements will be
    # instances of RankedValue (so none will be None, although they may wrap a None value).
    ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

    # Record info about the derivation of each of the values.
    for ranked_val in ranked_vals:
      if ranked_val.rank == RankedValue.CONFIG:
        details = config_details
      elif ranked_val.rank == RankedValue.ENVIRONMENT:
        details = env_details
      else:
        details = None
      self._option_tracker.record_option(scope=self._scope,
                                         option=dest,
                                         value=ranked_val.value,
                                         rank=ranked_val.rank,
                                         deprecation_version=kwargs.get('removal_version'),
                                         details=details)

    # Helper function to check various validity constraints on final option values.
    def check(val):
      if val is not None:
        choices = kwargs.get('choices')
        if choices is not None and val not in choices:
          raise ParseError('{} is not an allowed value for option {} in {}. '
                           'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
        elif kwargs.get('type') == file_option and not os.path.isfile(val):
          raise ParseError('File value {} for option {} in {} does not exist.'.format(
              val, dest, self._scope_str()))

    # Generate the final value from all available values, and check that it (or its members,
    # if a list) are in the set of allowed choices.
    if is_list_option(kwargs):
      merged_rank = ranked_vals[-1].rank
      merged_val = ListValueComponent.merge(
          [rv.value for rv in ranked_vals if rv.value is not None]).val
      merged_val = [self._wrap_type(kwargs.get('member_type', str))(x) for x in merged_val]
      map(check, merged_val)
      ret = RankedValue(merged_rank, merged_val)
    else:
      ret = ranked_vals[-1]
      check(ret.value)

    # All done!
    return ret
  def get_option_help_info(self, args, kwargs):
    """Returns an OptionHelpInfo for the option registered with the given (args, kwargs).

    :API: public
    """
    display_args = []
    scoped_cmd_line_args = []
    unscoped_cmd_line_args = []

    for arg in args:
      is_short_arg = len(arg) == 2
      unscoped_cmd_line_args.append(arg)
      if self._scope_prefix:
        scoped_arg = '--{}-{}'.format(self._scope_prefix, arg.lstrip('-'))
      else:
        scoped_arg = arg
      scoped_cmd_line_args.append(scoped_arg)

      if is_boolean_option(kwargs):
        if is_short_arg:
          display_args.append(scoped_arg)
        else:
          unscoped_cmd_line_args.append('--no-{}'.format(arg[2:]))
          scoped_cmd_line_args.append('--no-{}'.format(scoped_arg[2:]))
          display_args.append('--[no-]{}'.format(scoped_arg[2:]))
      else:
        metavar = self.compute_metavar(kwargs)
        display_arg = '{}={}'.format(scoped_arg, metavar)
        if is_list_option(kwargs):
          # Show the multi-arg append form.
          display_args.append('{arg_str} ({arg_str}) ...'.format(arg_str=display_arg))
          # Also show the list literal form, both with and without the append operator.
          if metavar.startswith('"') and metavar.endswith('"'):
            # We quote the entire list literal, so we shouldn't quote the individual members.
            metavar = metavar[1:-1]
          display_args.append('{arg}="[{metavar}, {metavar}, ...]"'.format(arg=scoped_arg,
                                                                           metavar=metavar))
          display_args.append('{arg}="+[{metavar}, {metavar}, ...]"'.format(arg=scoped_arg,
                                                                            metavar=metavar))
        else:
          display_args.append(display_arg)

    if is_boolean_option(kwargs):
      typ = bool
    else:
      typ = kwargs.get('type', str)
    default = self.compute_default(kwargs)
    help_msg = kwargs.get('help', 'No help available.')
    deprecated_version = kwargs.get('deprecated_version')
    deprecated_message = None
    if deprecated_version:
      deprecated_tense = self._get_deprecated_tense(deprecated_version)
      deprecated_message = 'DEPRECATED. {} removed in version: {}'.format(deprecated_tense,
                                                                          deprecated_version)
    deprecated_hint = kwargs.get('deprecated_hint')
    choices = ', '.join(kwargs.get('choices')) if kwargs.get('choices') else None

    ret = OptionHelpInfo(registering_class=kwargs.get('registering_class', type(None)),
                         display_args=display_args,
                         scoped_cmd_line_args=scoped_cmd_line_args,
                         unscoped_cmd_line_args=unscoped_cmd_line_args,
                         typ=typ,
                         fromfile=kwargs.get('fromfile', False),
                         default=default,
                         help=help_msg,
                         deprecated_version=deprecated_version,
                         deprecated_message=deprecated_message,
                         deprecated_hint=deprecated_hint,
                         choices=choices)
    return ret
Esempio n. 22
0
    def _compute_value(self, dest, kwargs, flag_val_strs):
        """Compute the value to use for an option.

        The source of the default value is chosen according to the ranking in RankedValue.
        """

        # Helper function to convert a string to a value of the option's type.
        def to_value_type(val_str):
            if val_str is None:
                return None
            if kwargs.get("type") == bool:
                return self._ensure_bool(val_str)
            type_arg = kwargs.get("type", str)
            try:
                return self._wrap_type(type_arg)(val_str)
            except (TypeError, ValueError) as e:
                raise ParseError(
                    f"Error applying type '{type_arg.__name__}' to option value '{val_str}', for option "
                    f"'--{dest}' in {self._scope_str()}: {e}")

        # Helper function to expand a fromfile=True value string, if needed.
        # May return a string or a dict/list decoded from a json/yaml file.
        def expand(val_or_str):
            if (kwargs.get("fromfile", True) and isinstance(val_or_str, str)
                    and val_or_str.startswith("@")):
                if val_or_str.startswith(
                        "@@"
                ):  # Support a literal @ for fromfile values via @@.
                    return val_or_str[1:]
                else:
                    fromfile = val_or_str[1:]
                    try:
                        with open(fromfile, "r") as fp:
                            s = fp.read().strip()
                            if fromfile.endswith(".json"):
                                return json.loads(s)
                            elif fromfile.endswith(
                                    ".yml") or fromfile.endswith(".yaml"):
                                return yaml.safe_load(s)
                            else:
                                return s
                    except (IOError, ValueError, yaml.YAMLError) as e:
                        raise FromfileError(
                            f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}"
                        )
            else:
                return val_or_str

        # Get value from config files, and capture details about its derivation.
        config_details = None
        config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
        config_default_val_or_str = expand(
            self._config.get(Config.DEFAULT_SECTION, dest, default=None))
        config_val_or_str = expand(
            self._config.get(config_section, dest, default=None))
        config_source_file = self._config.get_source_for_option(
            config_section, dest) or self._config.get_source_for_option(
                Config.DEFAULT_SECTION, dest)
        if config_source_file is not None:
            config_source_file = os.path.relpath(config_source_file)
            config_details = f"in {config_source_file}"

        # Get value from environment, and capture details about its derivation.
        udest = dest.upper()
        if self._scope == GLOBAL_SCOPE:
            # For convenience, we allow three forms of env var for global scope options.
            # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
            # for all the other scopes.  However we also allow simply PANTS_FOO. And if the option name
            # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
            # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
            # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
            env_vars = [f"PANTS_GLOBAL_{udest}", f"PANTS_{udest}"]
            if udest.startswith("PANTS_"):
                env_vars.append(udest)
        else:
            sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub(
                "_", self._scope.upper())
            env_vars = [f"PANTS_{sanitized_env_var_scope}_{udest}"]

        env_val_or_str = None
        env_details = None
        if self._env:
            for env_var in env_vars:
                if env_var in self._env:
                    env_val_or_str = expand(self._env.get(env_var))
                    env_details = f"from env var {env_var}"
                    break

        # Get value from cmd-line flags.
        flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
        if is_list_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to [].
            flag_val = ListValueComponent.merge(
                flag_vals) if flag_vals else None
        elif is_dict_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to {}.
            flag_val = DictValueComponent.merge(
                flag_vals) if flag_vals else None
        elif len(flag_vals) > 1:
            raise ParseError(
                "Multiple cmd line flags specified for option {} in {}".format(
                    dest, self._scope_str()))
        elif len(flag_vals) == 1:
            flag_val = flag_vals[0]
        else:
            flag_val = None

        # Rank all available values.
        # Note that some of these values may already be of the value type, but type conversion
        # is idempotent, so this is OK.

        values_to_rank = [
            to_value_type(x) for x in [
                flag_val,
                env_val_or_str,
                config_val_or_str,
                config_default_val_or_str,
                kwargs.get("default"),
                None,
            ]
        ]
        # Note that ranked_vals will always have at least one element, and all elements will be
        # instances of RankedValue (so none will be None, although they may wrap a None value).
        ranked_vals = list(
            reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

        def record_option(value, rank, option_details=None):
            deprecation_version = kwargs.get("removal_version")
            self._option_tracker.record_option(
                scope=self._scope,
                option=dest,
                value=value,
                rank=rank,
                deprecation_version=deprecation_version,
                details=option_details,
            )

        # Record info about the derivation of each of the contributing values.
        detail_history = []
        for ranked_val in ranked_vals:
            if ranked_val.rank in (RankedValue.CONFIG,
                                   RankedValue.CONFIG_DEFAULT):
                details = config_details
            elif ranked_val.rank == RankedValue.ENVIRONMENT:
                details = env_details
            else:
                details = None
            if details:
                detail_history.append(details)
            record_option(value=ranked_val.value,
                          rank=ranked_val.rank,
                          option_details=details)

        # Helper function to check various validity constraints on final option values.
        def check(val):
            if val is None:
                return
            choices = kwargs.get("choices")
            type_arg = kwargs.get("type")
            if choices is None and "type" in kwargs:
                if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
                    choices = list(type_arg)
            # TODO: convert this into an enum() pattern match!
            if choices is not None and val not in choices:
                raise ParseError(
                    "`{}` is not an allowed value for option {} in {}. "
                    "Must be one of: {}".format(val, dest, self._scope_str(),
                                                choices))

            if type_arg == file_option:
                check_file_exists(val)
            if type_arg == dir_option:
                check_dir_exists(val)

        def check_file_exists(val) -> None:
            error_prefix = f"File value `{val}` for option `{dest}` in `{self._scope_str()}`"
            try:
                path = Path(val)
                path_with_buildroot = Path(get_buildroot(), val)
            except TypeError:
                raise ParseError(
                    f"{error_prefix} cannot be parsed as a file path.")
            if not path.is_file() and not path_with_buildroot.is_file():
                raise ParseError(f"{error_prefix} does not exist.")

        def check_dir_exists(val) -> None:
            error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`"
            try:
                path = Path(val)
                path_with_buildroot = Path(get_buildroot(), val)
            except TypeError:
                raise ParseError(
                    f"{error_prefix} cannot be parsed as a directory path.")
            if not path.is_dir() and not path_with_buildroot.is_dir():
                raise ParseError(f"{error_prefix} does not exist.")

        # Generate the final value from all available values, and check that it (or its members,
        # if a list) are in the set of allowed choices.
        if is_list_option(kwargs):
            merged_rank = ranked_vals[-1].rank
            merged_val = ListValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            # TODO: run `check()` for all elements of a list option too!!!
            merged_val = [
                self._convert_member_type(kwargs.get("member_type", str), x)
                for x in merged_val
            ]
            if kwargs.get("member_type") == shell_str:
                merged_val = flatten_shlexed_list(merged_val)
            for val in merged_val:
                check(val)
            ret = RankedValue(merged_rank, merged_val)
        elif is_dict_option(kwargs):
            # TODO: convert `member_type` for dict values too!
            merged_rank = ranked_vals[-1].rank
            merged_val = DictValueComponent.merge(
                [rv.value for rv in ranked_vals if rv.value is not None]).val
            for val in merged_val:
                check(val)
            ret = RankedValue(merged_rank, merged_val)
        else:
            ret = ranked_vals[-1]
            check(ret.value)

        # Record info about the derivation of the final value.
        merged_details = ", ".join(detail_history) if detail_history else None
        record_option(value=ret.value,
                      rank=ret.rank,
                      option_details=merged_details)

        # All done!
        return ret
Esempio n. 23
0
  def _compute_value(self, dest, kwargs, flag_val_strs):
    """Compute the value to use for an option.

    The source of the default value is chosen according to the ranking in RankedValue.
    """
    # Helper function to convert a string to a value of the option's type.
    def to_value_type(val_str):
      if val_str is None:
        return None
      elif kwargs.get('type') == bool:
        return self._ensure_bool(val_str)
      else:
        type_arg = kwargs.get('type', str)
        try:
          return self._wrap_type(type_arg)(val_str)
        except TypeError as e:
          raise ParseError(
            "Error applying type '{}' to option value '{}', for option '--{}' in {}: {}"
            .format(type_arg.__name__, val_str, dest, self._scope_str(), e))

    # Helper function to expand a fromfile=True value string, if needed.
    # May return a string or a dict/list decoded from a json/yaml file.
    def expand(val_or_str):
      if (kwargs.get('fromfile', True) and val_or_str and
          isinstance(val_or_str, str) and val_or_str.startswith('@')):
        if val_or_str.startswith('@@'):   # Support a literal @ for fromfile values via @@.
          return val_or_str[1:]
        else:
          fromfile = val_or_str[1:]
          try:
            with open(fromfile, 'r') as fp:
              s = fp.read().strip()
              if fromfile.endswith('.json'):
                return json.loads(s)
              elif fromfile.endswith('.yml') or fromfile.endswith('.yaml'):
                return yaml.safe_load(s)
              else:
                return s
          except (IOError, ValueError, yaml.YAMLError) as e:
            raise self.FromfileError('Failed to read {} in {} from file {}: {}'.format(
                dest, self._scope_str(), fromfile, e))
      else:
        return val_or_str

    # Get value from config files, and capture details about its derivation.
    config_details = None
    config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
    config_default_val_or_str = expand(self._config.get(Config.DEFAULT_SECTION, dest, default=None))
    config_val_or_str = expand(self._config.get(config_section, dest, default=None))
    config_source_file = (self._config.get_source_for_option(config_section, dest) or
        self._config.get_source_for_option(Config.DEFAULT_SECTION, dest))
    if config_source_file is not None:
      config_source_file = os.path.relpath(config_source_file)
      config_details = 'in {}'.format(config_source_file)

    # Get value from environment, and capture details about its derivation.
    udest = dest.upper()
    if self._scope == GLOBAL_SCOPE:
      # For convenience, we allow three forms of env var for global scope options.
      # The fully-specified env var is PANTS_GLOBAL_FOO, which is uniform with PANTS_<SCOPE>_FOO
      # for all the other scopes.  However we also allow simply PANTS_FOO. And if the option name
      # itself starts with 'pants-' then we also allow simply FOO. E.g., PANTS_WORKDIR instead of
      # PANTS_PANTS_WORKDIR or PANTS_GLOBAL_PANTS_WORKDIR. We take the first specified value we
      # find, in this order: PANTS_GLOBAL_FOO, PANTS_FOO, FOO.
      env_vars = ['PANTS_GLOBAL_{0}'.format(udest), 'PANTS_{0}'.format(udest)]
      if udest.startswith('PANTS_'):
        env_vars.append(udest)
    else:
      sanitized_env_var_scope = self._ENV_SANITIZER_RE.sub('_', self._scope.upper())
      env_vars = ['PANTS_{0}_{1}'.format(sanitized_env_var_scope, udest)]

    env_val_or_str = None
    env_details = None
    if self._env:
      for env_var in env_vars:
        if env_var in self._env:
          env_val_or_str = expand(self._env.get(env_var))
          env_details = 'from env var {}'.format(env_var)
          break

    # Get value from cmd-line flags.
    flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
    if is_list_option(kwargs):
      # Note: It's important to set flag_val to None if no flags were specified, so we can
      # distinguish between no flags set vs. explicit setting of the value to [].
      flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
    elif is_dict_option(kwargs):
      # Note: It's important to set flag_val to None if no flags were specified, so we can
      # distinguish between no flags set vs. explicit setting of the value to {}.
      flag_val = DictValueComponent.merge(flag_vals) if flag_vals else None
    elif len(flag_vals) > 1:
      raise ParseError('Multiple cmd line flags specified for option {} in {}'.format(
          dest, self._scope_str()))
    elif len(flag_vals) == 1:
      flag_val = flag_vals[0]
    else:
      flag_val = None

    # Rank all available values.
    # Note that some of these values may already be of the value type, but type conversion
    # is idempotent, so this is OK.

    values_to_rank = [to_value_type(x) for x in
                      [flag_val, env_val_or_str, config_val_or_str,
                       config_default_val_or_str, kwargs.get('default'), None]]
    # Note that ranked_vals will always have at least one element, and all elements will be
    # instances of RankedValue (so none will be None, although they may wrap a None value).
    ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

    def record_option(value, rank, option_details=None):
      deprecation_version = kwargs.get('removal_version')
      self._option_tracker.record_option(scope=self._scope,
                                         option=dest,
                                         value=value,
                                         rank=rank,
                                         deprecation_version=deprecation_version,
                                         details=option_details)

    # Record info about the derivation of each of the contributing values.
    detail_history = []
    for ranked_val in ranked_vals:
      if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT):
        details = config_details
      elif ranked_val.rank == RankedValue.ENVIRONMENT:
        details = env_details
      else:
        details = None
      if details:
        detail_history.append(details)
      record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details)

    # Helper function to check various validity constraints on final option values.
    def check(val):
      if val is not None:
        choices = kwargs.get('choices')
        # If the `type` argument has an `all_variants` attribute, use that as `choices` if not
        # already set. Using an attribute instead of checking a subclass allows `type` arguments
        # which are functions to have an implicit fallback `choices` set as well.
        if choices is None and 'type' in kwargs:
          type_arg = kwargs.get('type')
          if hasattr(type_arg, 'all_variants'):
            choices = list(type_arg.all_variants)
        # TODO: convert this into an enum() pattern match!
        if choices is not None and val not in choices:
          raise ParseError('`{}` is not an allowed value for option {} in {}. '
                           'Must be one of: {}'.format(val, dest, self._scope_str(), choices))
        elif kwargs.get('type') == dir_option and not os.path.isdir(val):
          raise ParseError('Directory value `{}` for option {} in {} does not exist.'.format(
              val, dest, self._scope_str()))
        elif kwargs.get('type') == file_option and not os.path.isfile(val):
          raise ParseError('File value `{}` for option {} in {} does not exist.'.format(
              val, dest, self._scope_str()))

    # Generate the final value from all available values, and check that it (or its members,
    # if a list) are in the set of allowed choices.
    if is_list_option(kwargs):
      merged_rank = ranked_vals[-1].rank
      merged_val = ListValueComponent.merge(
          [rv.value for rv in ranked_vals if rv.value is not None]).val
      # TODO: run `check()` for all elements of a list option too!!!
      merged_val = [self._convert_member_type(kwargs.get('member_type', str), x)
                    for x in merged_val]
      for val in merged_val:
        check(val)
      ret = RankedValue(merged_rank, merged_val)
    elif is_dict_option(kwargs):
      # TODO: convert `member_type` for dict values too!
      merged_rank = ranked_vals[-1].rank
      merged_val = DictValueComponent.merge(
          [rv.value for rv in ranked_vals if rv.value is not None]).val
      for val in merged_val:
        check(val)
      ret = RankedValue(merged_rank, merged_val)
    else:
      ret = ranked_vals[-1]
      check(ret.value)

    # Record info about the derivation of the final value.
    merged_details = ', '.join(detail_history) if detail_history else None
    record_option(value=ret.value, rank=ret.rank, option_details=merged_details)

    # All done!
    return ret
Esempio n. 24
0
    def _compute_value(self, dest, kwargs, flag_val_strs, passthru_arg_strs):
        """Compute the value to use for an option.

        The source of the value is chosen according to the ranking in Rank.
        """
        type_arg = kwargs.get("type", str)
        member_type = kwargs.get("member_type", str)

        def to_value_type(val_str):
            return self.to_value_type(val_str, type_arg, member_type, dest)

        # Helper function to expand a fromfile=True value string, if needed.
        # May return a string or a dict/list decoded from a json/yaml file.
        def expand(val_or_str):
            if (
                kwargs.get("fromfile", True)
                and isinstance(val_or_str, str)
                and val_or_str.startswith("@")
            ):
                if val_or_str.startswith("@@"):  # Support a literal @ for fromfile values via @@.
                    return val_or_str[1:]
                else:
                    fromfile = val_or_str[1:]
                    try:
                        with open(fromfile, "r") as fp:
                            s = fp.read().strip()
                            if fromfile.endswith(".json"):
                                return json.loads(s)
                            elif fromfile.endswith(".yml") or fromfile.endswith(".yaml"):
                                return yaml.safe_load(s)
                            else:
                                return s
                    except (IOError, ValueError, yaml.YAMLError) as e:
                        raise FromfileError(
                            f"Failed to read {dest} in {self._scope_str()} from file {fromfile}: {e!r}"
                        )
            else:
                return val_or_str

        # Get value from config files, and capture details about its derivation.
        config_details = None
        config_section = GLOBAL_SCOPE_CONFIG_SECTION if self._scope == GLOBAL_SCOPE else self._scope
        config_default_val_or_str = expand(
            self._config.get(Config.DEFAULT_SECTION, dest, default=None)
        )
        config_val_or_str = expand(self._config.get(config_section, dest, default=None))
        config_source_file = self._config.get_source_for_option(
            config_section, dest
        ) or self._config.get_source_for_option(Config.DEFAULT_SECTION, dest)
        if config_source_file is not None:
            config_source_file = os.path.relpath(config_source_file)
            config_details = f"from {config_source_file}"

        # Get value from environment, and capture details about its derivation.
        env_vars = self.get_env_var_names(self._scope, dest)
        env_val_or_str = None
        env_details = None
        if self._env:
            for env_var in env_vars:
                if env_var in self._env:
                    env_val_or_str = expand(self._env.get(env_var))
                    env_details = f"from env var {env_var}"
                    break

        # Get value from cmd-line flags.
        flag_vals = [to_value_type(expand(x)) for x in flag_val_strs]
        if kwargs.get("passthrough"):
            # NB: Passthrough arguments are either of type `str` or `shell_str`
            # (see self._validate): the former never need interpretation, and the latter do not
            # need interpretation when they have been provided directly via `sys.argv` as the
            # passthrough args have been.
            flag_vals.append(
                ListValueComponent(ListValueComponent.MODIFY, [*passthru_arg_strs], [])
            )

        if is_list_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to [].
            flag_val = ListValueComponent.merge(flag_vals) if flag_vals else None
        elif is_dict_option(kwargs):
            # Note: It's important to set flag_val to None if no flags were specified, so we can
            # distinguish between no flags set vs. explicit setting of the value to {}.
            flag_val = DictValueComponent.merge(flag_vals) if flag_vals else None
        elif len(flag_vals) > 1:
            raise ParseError(
                f"Multiple cmd line flags specified for option {dest} in {self._scope_str()}"
            )
        elif len(flag_vals) == 1:
            flag_val = flag_vals[0]
        else:
            flag_val = None
        flag_details = None if flag_val is None else "from command-line flag"

        # Rank all available values.
        # Note that some of these values may already be of the value type, but type conversion
        # is idempotent, so this is OK.

        values_to_rank = [
            (to_value_type(x), detail)
            for (x, detail) in [
                (flag_val, flag_details),
                (env_val_or_str, env_details),
                (config_val_or_str, config_details),
                (config_default_val_or_str, config_details),
                (kwargs.get("default"), None),
                (None, None),
            ]
        ]
        # Note that ranked_vals will always have at least one element, and all elements will be
        # instances of RankedValue (so none will be None, although they may wrap a None value).
        ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank))))

        def group(value_component_type, process_val_func) -> List[RankedValue]:
            # We group any values that are merged together, so that the history can reflect
            # merges vs. replacements in a useful way. E.g., if we merge [a, b] and [c],
            # and then replace it with [d, e], the history will contain:
            #   - [d, e] (from command-line flag)
            #   - [a, b, c] (from env var, from config)
            # And similarly for dicts.
            grouped: List[List[RankedValue]] = [[]]
            for ranked_val in ranked_vals:
                if ranked_val.value and ranked_val.value.action == value_component_type.REPLACE:
                    grouped.append([])
                grouped[-1].append(ranked_val)
            return [
                RankedValue(
                    grp[-1].rank,
                    process_val_func(
                        value_component_type.merge(
                            rv.value for rv in grp if rv.value is not None
                        ).val
                    ),
                    ", ".join(rv.details for rv in grp if rv.details),
                )
                for grp in grouped
                if grp
            ]

        if is_list_option(kwargs):

            def process_list(lst):
                lst = [self._convert_member_type(member_type, val) for val in lst]
                if member_type == shell_str:
                    lst = flatten_shlexed_list(lst)
                return lst

            historic_ranked_vals = group(ListValueComponent, process_list)
        elif is_dict_option(kwargs):
            historic_ranked_vals = group(DictValueComponent, lambda x: x)
        else:
            historic_ranked_vals = ranked_vals

        value_history = OptionValueHistory(tuple(historic_ranked_vals))

        # Helper function to check various validity constraints on final option values.
        def check_scalar_value(val):
            if val is None:
                return
            choices = kwargs.get("choices")
            if choices is None and "type" in kwargs:
                if inspect.isclass(type_arg) and issubclass(type_arg, Enum):
                    choices = list(type_arg)
            if choices is not None and val not in choices:
                raise ParseError(
                    f"`{val}` is not an allowed value for option {dest} in {self._scope_str()}. "
                    f"Must be one of: {choices}"
                )
            elif type_arg == file_option:
                check_file_exists(val)
            elif type_arg == dir_option:
                check_dir_exists(val)

        def check_file_exists(val) -> None:
            error_prefix = f"File value `{val}` for option `{dest}` in `{self._scope_str()}`"
            try:
                path = Path(val)
                path_with_buildroot = Path(get_buildroot(), val)
            except TypeError:
                raise ParseError(f"{error_prefix} cannot be parsed as a file path.")
            if not path.is_file() and not path_with_buildroot.is_file():
                raise ParseError(f"{error_prefix} does not exist.")

        def check_dir_exists(val) -> None:
            error_prefix = f"Directory value `{val}` for option `{dest}` in `{self._scope_str()}`"
            try:
                path = Path(val)
                path_with_buildroot = Path(get_buildroot(), val)
            except TypeError:
                raise ParseError(f"{error_prefix} cannot be parsed as a directory path.")
            if not path.is_dir() and not path_with_buildroot.is_dir():
                raise ParseError(f"{error_prefix} does not exist.")

        # Validate the final value.
        final_val = value_history.final_value
        if isinstance(final_val.value, list):
            for component in final_val.value:
                check_scalar_value(component)
            if inspect.isclass(member_type) and issubclass(member_type, Enum):
                if len(final_val.value) != len(set(final_val.value)):
                    raise ParseError(f"Duplicate enum values specified in list: {final_val.value}")
        elif isinstance(final_val.value, dict):
            for component in final_val.value.values():
                check_scalar_value(component)
        else:
            check_scalar_value(final_val.value)

        return value_history