Beispiel #1
0
    def _iter_fields(
        self, data: typing.Mapping, files: typing.Mapping
    ) -> typing.Iterator[typing.Union["FileField", "DataField"]]:
        for name, value in data.items():
            if isinstance(value, list):
                for item in value:
                    yield self.DataField(name=name, value=item)
            else:
                yield self.DataField(name=name, value=value)

        for name, value in files.items():
            yield self.FileField(name=name, value=value)
Beispiel #2
0
def dict2items(
    value: typing.Mapping,
    key_name: str = 'key',
    value_name: str = 'value',
) -> typing.List[typing.Dict[str, typing.Any]]:
    """Convert a mapping to a list of its items.

    For example, converts

    .. code-block:: yaml

        milk: 1
        eggs: 10
        bread: 2

    into

    .. code-block:: yaml

        - key: milk
          value: 1
        - key: eggs
          value: 10
        - key: bread
          value: 2

    .. versionadded:: 1.1

    :param value: Any mapping.
    :param key_name: Key name for input keys.
    :param value_name: Key name for input values.
    :returns: A list of dicts.
    """
    return [{key_name: key, value_name: value} for key, value in value.items()]
Beispiel #3
0
    def _build_values(self, settings: typing.Mapping) -> None:
        """Build up self._values from the values in the given dict."""
        errors = []
        for name, yaml_values in settings.items():
            if not isinstance(yaml_values, dict):
                errors.append(configexc.ConfigErrorDesc(
                    "While parsing {!r}".format(name), "value is not a dict"))
                continue

            values = configutils.Values(configdata.DATA[name])
            if 'global' in yaml_values:
                values.add(yaml_values.pop('global'))

            for pattern, value in yaml_values.items():
                if not isinstance(pattern, str):
                    errors.append(configexc.ConfigErrorDesc(
                        "While parsing {!r}".format(name),
                        "pattern is not of type string"))
                    continue
                try:
                    urlpattern = urlmatch.UrlPattern(pattern)
                except urlmatch.ParseError as e:
                    errors.append(configexc.ConfigErrorDesc(
                        "While parsing pattern {!r} for {!r}"
                        .format(pattern, name), e))
                    continue
                values.add(value, urlpattern)

            self._values[name] = values

        if errors:
            raise configexc.ConfigFileErrors('autoconfig.yml', errors)
Beispiel #4
0
def _sub_chars(string: str, probability: float,
               mapping: typing.Mapping) -> str:
    """Replace substrings with a given probability.

    Given a mapping, search string one by one for keys and replace with
    the appropriate value, with some probability. If your keys are not mutually
    exclusive (e.g. some part of them overlaps), the order in which they appear
    in the mapping becomes important.

    Args:
        string: text
        probability: probability of replacing a group of characters
        mapping: map of substring -> replacement

    Returns:
        enriched text
    """
    for pattern, sub in mapping.items():
        index = 0
        while 0 <= index < len(string):
            index = string.lower().find(pattern, index)
            if index < 0:
                break
            elif random.binomial(1, probability):
                string = string[:index] + sub + string[index + len(pattern):]
                index += len(sub)
            else:
                index += len(pattern)
    return string
Beispiel #5
0
def parse_params(prefix: str, params: typing.Mapping) -> typing.Mapping:
    result = {}
    for key, value in params.items():
        if key.startswith(prefix):
            name = key.strip(prefix)[1:-1]
            result[name] = value
    return result
Beispiel #6
0
    def signal(self, value: t.Mapping) -> None:
        """Set the signal mapping

        Args:
            value (t.Mapping): a mapping with symbols and a 'carrier'

        Raises:
            LayerMisconfigured: If incompatible value is provided
        """
        try:
            assert isinstance(value, t.Mapping), "must be a mapping"
            assert "carrier" in value, "must have a 'carrier' key"
            mapping = dict(filter(lambda x: isinstance(x[0], int), value.items()))
            _0, *_1 = mapping.values()
            assert all(len(_) == len(_0) for _ in _1), "must have same-length mappings"

            self._mapping = mapping
            self._carrier = value["carrier"]
            self._signal = value
            self._subsymbol_count = len(_0)

            if self._subsymbol_count < 2:
                raise LayerMisconfigured(
                    "generic line mapping works only on signals with at " "least 2 subsymbols"
                )

        except AssertionError as e:
            raise LayerMisconfigured("signal {}, got {}".format(*e.args, value))
Beispiel #7
0
def map_effects(mapping: t.Mapping, aw: t.Awaitable):
    mapping = {_map_kind(k): v for (k, v) in mapping.items()}

    def mapping_handler(eff: Effect):
        handle = mapping.get(eff.kind, None)

        if handle is None:
            return handle

        is_callable = isinstance(handle, t.Callable)
        is_mapping = isinstance(handle, t.Mapping)

        if is_mapping and is_callable:
            raise ValueError(
                "Expected either mapping or callable but "
                "got value implementing both", handle)

        if is_mapping:
            return lambda *a, **kw: map_effects(handle,
                                                eff.default_handle(*a, **kw))

        if is_callable:
            return handle

        raise ValueError(f"Handler mapping should contain either "
                         f"another mapping or callable, got {handle}")

    return resolve_effects(mapping_handler, aw)
Beispiel #8
0
    def saturate_mapping(self, value: t.Mapping) -> None:
        try:
            assert isinstance(value, t.Mapping), "must be a mapping"
            assert value, "must not be empty"
            assert dict_depth(value) == 1, "must be a mapping of depth 1"
            assert all(isinstance(_, int) for _ in value.values()), "must have int values"
        except AssertionError as e:
            raise LayerMisconfigured("saturate_mapping {}, got {}".format(*e.args, value))

        if any(isinstance(_, str) for _ in value.keys()):
            _ = {}
            for k, v in value.items():
                _[int(k)] = v
            value = _

        self._saturate_mapping = dict(sorted(value.items(), key=lambda i: i[0]))
Beispiel #9
0
    def _build_values(self, settings: typing.Mapping) -> None:
        """Build up self._values from the values in the given dict."""
        errors = []
        for name, yaml_values in settings.items():
            if not isinstance(yaml_values, dict):
                errors.append(configexc.ConfigErrorDesc(
                    "While parsing {!r}".format(name), "value is not a dict"))
                continue

            values = configutils.Values(configdata.DATA[name])
            if 'global' in yaml_values:
                values.add(yaml_values.pop('global'))

            for pattern, value in yaml_values.items():
                if not isinstance(pattern, str):
                    errors.append(configexc.ConfigErrorDesc(
                        "While parsing {!r}".format(name),
                        "pattern is not of type string"))
                    continue
                try:
                    urlpattern = urlmatch.UrlPattern(pattern)
                except urlmatch.ParseError as e:
                    errors.append(configexc.ConfigErrorDesc(
                        "While parsing pattern {!r} for {!r}"
                        .format(pattern, name), e))
                    continue
                values.add(value, urlpattern)

            self._values[name] = values

        if errors:
            raise configexc.ConfigFileErrors('autoconfig.yml', errors)
Beispiel #10
0
    def __init__(
        self,
        context: PyTorchSAClientContext,
        layers_config: typing.List[typing.Mapping],
        optimizer_config: types.SimpleNamespace,
        loss_config: typing.Mapping,
    ):
        super().__init__()
        self.save_hyperparameters()
        self.context = context

        # model
        layers = []
        for layer_config in layers_config:
            layer_name = layer_config["layer"]
            layer_kwargs = {k: v for k, v in layer_config.items() if k != "layer"}
            layers.append(get_layer_fn(layer_name, layer_kwargs))
        self.model = nn.Sequential(*layers)

        # loss
        loss_name = loss_config["loss"]
        loss_kwargs = {k: v for k, v in loss_config.items() if k != "loss"}
        self.loss_fn, self.expected_label_type = get_loss_fn(loss_name, loss_kwargs)

        # optimizer
        self._optimizer_name = optimizer_config.optimizer
        self._optimizer_kwargs = optimizer_config.kwargs

        self._num_data_consumed = 0
        self._all_consumed_data_aggregated = True

        self._should_early_stop = False
Beispiel #11
0
def compare_all_but(dict_a: t.Mapping,
                    dict_b: t.Mapping,
                    keys_to_ignore: t.Optional[t.Iterable] = None) -> bool:
    """
    Compare two dictionaries, with the possibility to ignore some fields.

    :arg dict_a: First dictionary to compare
    :arg dict_b: Second dictionary to compare
    :kwarg keys_to_ignore: An iterable of keys whose values in the dictionaries will not be
        compared.
    :returns: True if the dictionaries have matching values for all of the keys which were not
        ignored.  False otherwise.
    """
    if keys_to_ignore is None:
        return dict_a == dict_b

    if not isinstance(keys_to_ignore, Set):
        keys_to_ignore = frozenset(keys_to_ignore)

    length_a = len(frozenset(dict_a.keys()) - keys_to_ignore)
    length_b = len(frozenset(dict_b.keys()) - keys_to_ignore)

    if length_a != length_b:
        return False

    sentinel = object()

    for key, value in ((k, v) for k, v in dict_a.items()
                       if k not in keys_to_ignore):
        if value != dict_b.get(key, sentinel):
            return False

    return True
Beispiel #12
0
def create_contexts(args: t.Optional[argparse.Namespace] = None,
                    cfg: t.Mapping = ImmutableDict(),
                    use_extra: bool = True) -> ContextReturn:
    """
    Create new contexts appropriate for setting the app and lib context.

    This function takes values from the application arguments and configuration and sets them on
    the context.  It validates, normalizes, and sets defaults for the contexts based on what is
    available in the arguments and configuration.

    :kwarg args: An :python:obj:`argparse.Namespace` holding the program's command line
        arguments.  See the warning below about working with :python:mod:`argpase`.
    :kwarg cfg: A dictionary holding the program's configuration.
    :kwarg use_extra: When True, the default, all extra arguments and config values will be set as
        fields in ``app_ctx.extra``.  When False, the extra arguments and config values will be
        returned as part of the ContextReturn.
    :returns: A ContextReturn NamedTuple.

    .. warning::
        We cannot tell whether a user set a value via the command line if :python:mod:`argparse`
        sets the field to a default value.  That means when you specify the field in the
        :obj:`AppContext` or :obj:`LibContext` models, you must tell :python:mod:`argparse` not to
        set the field to a default like this::

            parser.add_argument('--breadcrumbs', default=argparse.SUPPRESS)

        If the field is only used via the :attr:`AppContext.extra` mechanism (not explictly set),
        then you should ignore this section and use :python:mod:`argparse`'s default mechanism.
    """
    lib_values = _extract_lib_context_values(args, cfg)
    app_values = _extract_app_context_values(args, cfg)

    #
    # Save the unused values
    #
    known_fields = _FIELDS_IN_APP_CTX.union(_FIELDS_IN_LIB_CTX)

    unused_cfg = {}
    if cfg:
        unused_cfg = {k: v for k, v in cfg.items() if k not in known_fields}

    unused_args = {}
    if args:
        unused_args = {k: v for k, v in vars(args).items() if k not in known_fields}

    # Unused values are saved in app_ctx.extra when use_extra is set
    if use_extra:
        unused_cfg.update(unused_args)
        app_values['extra'] = unused_cfg
        unused_cfg = {}
        unused_args = {}

    unused_args = argparse.Namespace(**unused_args)

    # create new app and lib ctxt from the application's arguments and config.
    app_ctx = AppContext(**app_values)
    lib_ctx = LibContext(**lib_values)

    return ContextReturn(app_ctx=app_ctx, lib_ctx=lib_ctx, args=unused_args, cfg=unused_cfg)
Beispiel #13
0
    def update(self, other: _Mapping, **kwargs):
        """It is important to pass all values through formatters
        """
        for k, v in other.items():
            self[k] = v

        for k, v in kwargs.items():
            self[k] = v
Beispiel #14
0
def create_contexts(args: t.Optional[argparse.Namespace] = None,
                    cfg: t.Mapping = ImmutableDict(),
                    use_extra: bool = True) -> ContextReturn:
    """
    Create new contexts appropriate for setting the app and lib context.

    This function takes values from the application arguments and configuration and sets them on
    the context.  It validates, normalizes, and sets defaults for the contexts based on what is
    available in the arguments and configuration.

    :kwarg args: An :python:obj:`argparse.Namespace` holding the program's command line arguments.
        Note argparse's ability to add default values should not be used with fields which are fully
        expressed in the :obj:`AppContext` or :obj:`LibContext` models.  Instead, set a default in
        the context model.  You can use argpase defaults with fields that get set in
        :attr:`AppContext.extra`.
    :kwarg cfg: A dictionary holding the program's configuration.
    :kwarg use_extra: When True, the default, all extra arguments and config values will be set as
        fields in ``app_ctx.extra``.  When False, the extra arguments and config values will be
        returned as part of the ContextReturn.
    :returns: A ContextReturn NamedTuple.
    """
    lib_values = _extract_lib_context_values(args, cfg)
    app_values = _extract_app_context_values(args, cfg)

    #
    # Save the unused values
    #
    known_fields = _FIELDS_IN_APP_CTX.union(_FIELDS_IN_LIB_CTX)

    unused_cfg = {}
    if cfg:
        unused_cfg = {k: v for k, v in cfg.items() if k not in known_fields}

    unused_args = {}
    if args:
        unused_args = {
            k: v
            for k, v in vars(args).items() if k not in known_fields
        }

    # Unused values are saved in app_ctx.extra when use_extra is set
    if use_extra:
        unused_cfg.update(unused_args)
        app_values['extra'] = unused_cfg
        unused_cfg = {}
        unused_args = {}

    unused_args = argparse.Namespace(**unused_args)

    # create new app and lib ctxt from the application's arguments and config.
    app_ctx = AppContext(**app_values)
    lib_ctx = LibContext(**lib_values)

    return ContextReturn(app_ctx=app_ctx,
                         lib_ctx=lib_ctx,
                         args=unused_args,
                         cfg=unused_cfg)
Beispiel #15
0
def override(target_dict: typing.MutableMapping,
             override_dict: typing.Mapping):
    """Apply the updates in override_dict to the dict target_dict. This is like
  dict.update, but recursive. i.e. if the existing element is a dict, then
  override elements of the sub-dict rather than wholesale replacing.

  One special case is added. If a key within override dict starts with '!' then
  it is interpretted as follows:
     - if the associated value is "REMOVE", the key is removed from the parent
       dict
     - use !! for keys that actually start with ! and shouldn't be removed.

  e.g.
  override(
    {
      'outer': { 'inner': { 'key': 'oldValue', 'existingKey': True } }
    },
    {
      'outer': { 'inner': { 'key': 'newValue' } },
      'newKey': { 'newDict': True },
    }
  )
  yields:
    {
      'outer': {
        'inner': {
           'key': 'newValue',
           'existingKey': True
        }
      },
      'newKey': { newDict: True }
    }
  """

    for key, value in override_dict.items():
        #
        # Handle special ! syntax:
        #   "!keyname" : "REMOVE",   --> remove the key 'keyname' from target_dict
        #
        if key[0:1] == "!" and key[1:2] != "!":
            key = key[1:]
            if value == "REMOVE":
                target_dict.pop(key, None)
                continue

        current_value = target_dict.get(key)
        if not isinstance(current_value, Mapping):
            # Thing or Mapping overrides Thing or None
            target_dict[key] = value
        elif isinstance(value, Mapping):
            # Mapping overrides mapping, recurse
            target_dict[key] = override(current_value, value)
        else:
            # Thing overrides Mapping
            target_dict[key] = value

    return target_dict
Beispiel #16
0
    def set_many(self, items: t.Mapping, ttl: t.Optional[T_TTL] = None) -> None:
        """
        Set multiple cache keys at once.

        Args:
            items: Mapping of cache key/values to set.
            ttl: TTL value. Defaults to ``None`` which uses :attr:`ttl`.
        """
        for key, value in items.items():
            self.set(key, value, ttl=ttl)
Beispiel #17
0
def recursive_mapping_iterator(
    nested_mapping: typing.Mapping,
) -> typing.Generator[typing.Tuple[typing.Text, typing.Any], None,
                      None]:  # noqa: E501
    for key, value in nested_mapping.items():
        if isinstance(value, collections.abc.Mapping):
            for inner_key, inner_value in recursive_mapping_iterator(value):
                yield key + "." + inner_key, inner_value
        else:
            yield key, value
Beispiel #18
0
 def recursive_worker(d: MutableMappingType,
                      u: MappingType) -> MutableMappingType:
     for k, v in u.items():
         dv = d.get(k, {})
         if not isinstance(dv, MutableMapping):
             d[k] = v
         elif isinstance(v, Mapping):
             d[k] = recursive_worker(dv, v)
         else:
             d[k] = v
     return d
Beispiel #19
0
def format_args(args: typing.Sequence = None,
                kwargs: typing.Mapping = None) -> str:
    """Format a list of arguments/kwargs to a function-call like string."""
    if args is not None:
        arglist = [utils.compact_text(repr(arg), 200) for arg in args]
    else:
        arglist = []
    if kwargs is not None:
        for k, v in kwargs.items():
            arglist.append('{}={}'.format(k, utils.compact_text(repr(v), 200)))
    return ', '.join(arglist)
Beispiel #20
0
    def update_from_dict(
        self,
        options: t.Mapping,
        *,
        exist_only: bool = True,
        prefix: str = "",
        processor: t.Callable = identity,
        annotations: t.Dict = None,
    ):
        """Update the configuration from given dictionary."""
        annotations = annotations or {}
        prefix = prefix.upper()
        prefix_length = len(prefix)
        for name, value in options.items():
            vtype = annotations.get(name)
            name = name.upper()
            if prefix:
                if not name.startswith(prefix):
                    continue

                name = name[prefix_length:]

            if not name or name.startswith("_"):
                continue

            if exist_only and name not in self.__storage:
                continue

            if name in self.__storage:
                vtype = self.__annotations[name]

            elif exist_only:
                continue

            else:
                if not vtype:
                    vtype = type(value)

                vtype = (vtype if vtype in types_supported
                         and getattr(vtype, "_inst", True) else identity)
                self.__annotations[name] = vtype

            try:
                value = processor(value)
                if value is not None:
                    value = vtype(value)
                self.__storage[name] = value
            except (ValueError, TypeError):
                logger.warning(
                    "Ignored invalid configuration value given for %s: %s",
                    name, value)
                continue
Beispiel #21
0
def recursive_update(d: Dict, u: _Mapping) -> Union[Dict, _Mapping]:
    if d is None:
        return u

    if u is None:
        return d

    for k, v in u.items():
        if isinstance(v, Mapping):
            d[k] = recursive_update(d.get(k, {}), v)
        else:
            d[k] = v

    return d
Beispiel #22
0
    def update_relation_data(
        self,
        relation_id: int,
        app_or_unit: str,
        key_values: typing.Mapping,
    ) -> None:
        """Update the relation data for a given unit or application in a given relation.

        This also triggers the `relation_changed` event for this relation_id.

        Args:
            relation_id: The integer relation_id representing this relation.
            app_or_unit: The unit or application name that is being updated.
                This can be the local or remote application.
            key_values: Each key/value will be updated in the relation data.
        """
        relation_name = self._backend._relation_names[relation_id]
        relation = self._model.get_relation(relation_name, relation_id)
        if '/' in app_or_unit:
            entity = self._model.get_unit(app_or_unit)
        else:
            entity = self._model.get_app(app_or_unit)
        rel_data = relation.data.get(entity, None)
        if rel_data is not None:
            # rel_data may have cached now-stale data, so _invalidate() it.
            # Note, this won't cause the data to be loaded if it wasn't already.
            rel_data._invalidate()

        new_values = self._backend._relation_data[relation_id][
            app_or_unit].copy()
        for k, v in key_values.items():
            if v == '':
                new_values.pop(k, None)
            else:
                new_values[k] = v
        self._backend._relation_data[relation_id][app_or_unit] = new_values

        if app_or_unit == self._model.unit.name:
            # No events for our own unit
            return
        if app_or_unit == self._model.app.name:
            # updating our own app only generates an event if it is a peer relation and we
            # aren't the leader
            is_peer = self._meta.relations[relation_name].role.is_peer()
            if not is_peer:
                return
            if self._model.unit.is_leader():
                return
        self._emit_relation_changed(relation_id, app_or_unit)
Beispiel #23
0
def recursive_update(d: Dict, u: _Mapping,
                     stop_keys: List[AnyStr] = [],
                     allow_recursion: bool = True) -> Union[Dict, _Mapping]:
    if d is None:
        return u

    if u is None:
        return d

    for k, v in u.items():
        if isinstance(v, Mapping) and allow_recursion:
            d[k] = recursive_update(d.get(k, {}), v, stop_keys=stop_keys, allow_recursion=k not in stop_keys)
        else:
            d[k] = v

    return d
Beispiel #24
0
def _dispatch(
    func: typing.Callable,
    matches: typing.Mapping,
    bound_args: typing.Tuple,
    bound_kwargs: Kwargs,
) -> typing.Any:
    for key, value in matches.items():
        if key in bound_kwargs:
            raise TypeError
        bound_kwargs[key] = value
    function_sig = inspect.signature(func)
    function_args = function_sig.bind(**bound_kwargs)
    for parameter in _varargs(function_sig):
        function_args.arguments[parameter.name] = bound_args
    function_args.apply_defaults()
    return func(*function_args.args, **function_args.kwargs)
Beispiel #25
0
def associate_data(root_map: typing.Mapping,
                   *args: typing.Mapping) -> typing.List[typing.List]:
    """
    Convert a number of maps key->value to a list of lists
    [[key, map1[key], map2[key] map3[key] ...] ...]

    The list will be sorted in key order
    Returned inner lists will be in the same order as they are passed as arguments.

    The first map passed is considered the reference point for the list of keys,
    :param root_map: The first map to associate
    :param args: Additional maps to associate to the first one
    :return:
    """
    if len(args) <= 0:
        # Nothing to associate, flatten the root map and return
        return sorted([k, v] for k, v in root_map.items())
    root_keys = set(root_map.keys())
    all_same = True
    # First, check if all the maps have the same list of keys
    for other_map in args:
        if set(other_map.keys()) != root_keys:
            all_same = False
            break
    if all_same:
        # All the maps have the same set of keys, just flatten them
        return sorted([key, root_map[key]] +
                      [other_map[key] for other_map in args]
                      for key in root_keys)
    else:
        # We need to associate the maps, the timestamps are a little out
        rekeyed_maps = []
        for other_map in args:
            matches = ass.associate(root_map,
                                    other_map,
                                    offset=0,
                                    max_difference=3)
            rekeyed_map = {
                root_key: other_map[other_key]
                for root_key, other_key in matches
            }
            root_keys &= set(rekeyed_map.keys())
            rekeyed_maps.append(rekeyed_map)
        return sorted([key, root_map[key]] +
                      [rekeyed_map[key] for rekeyed_map in rekeyed_maps]
                      for key in root_keys)
Beispiel #26
0
def _dict_included(lhs: typing.Mapping, rhs: typing.Mapping) -> bool:
    """
    :return: True if lhs is included in rhs

    >>> dic = dict(a=1, b=dict(c=2))
    >>> ref = dict(a=1, b=dict(c=2), d=dict(e="E"))
    >>> _dict_included(ref, dic)
    False
    >>> _dict_included(dic, {})
    False
    >>> _dict_included(dic, ref)
    True
    """
    for key, val in lhs.items():
        if key not in rhs or rhs[key] != val:
            return False

    return True
Beispiel #27
0
def update(dest: dict, src: typing.Mapping) -> typing.Mapping:
    """
    Perform deep updates for data collections.

    Works for dictionary, list, and set elements of the dict.

    Args:
        dest (dict): The dict to update.
        src (dict): The dict to update with.

    Returns:
        dict: The completely updated dict

    """
    dest.update({
        key: (
            update(dest.get(key, {}), value)
            if isinstance(value, collections.abc.Mapping)

            else set(dest.get(key, set())) | set(value)
            if isinstance(value, collections.abc.Set)

            else list(dest.get(key, [])) + list(value)

            if not isinstance(value, (str, bytes, bytearray)) and all(
                isinstance(value, type_)
                for type_ in {
                    collections.abc.Sized,
                    collections.abc.Iterable,
                    collections.abc.Container,
                    collections.abc.Sequence
                }
            )

            else value
        )

        for key, value in src.items()
    })

    return dest
Beispiel #28
0
    def put_hash(self,
                 pool: str,
                 key: str,
                 value: _Mapping,
                 ttl: int = None) -> _Mapping:
        """Put a hash
        """
        # Redis does not store empty hashes, so we need to mark empty hashes from our side
        if not value:
            value = {'__pytsite_empty_hash_marker': True}

        key = self._fqkn(pool, key)

        self._client.hmset(key,
                           {k: _pickle.dumps(v)
                            for k, v in value.items()})

        if ttl:
            self._client.expire(key, ttl)

        return value
Beispiel #29
0
def _sub_words(string: str, probability: float,
               mapping: typing.Mapping) -> str:
    """Replace words with a given probability.

    Split a string into words (the naïve way, on whitespace). Then, search
    word list for each key in the mapping, and replace it with its value with
    some probability. Then join them back together with a single whitespace.

    Args:
        string: text
        probability: probability of replacing a word
        mapping: map of substring -> replacement

    Returns:
        enriched text
    """
    words = string.split()
    for pattern, sub in mapping.items():
        for index, word in enumerate(words):
            if (word.lower() == pattern) and random.binomial(1, probability):
                words[index] = sub
    return " ".join(word for word in words if word)
Beispiel #30
0
    def from_config(cls, config: t.Mapping):
        """Initializes a CLIApp instance from a mapping object.

        :param config:
        :return:
        """
        self_config = dict(config.items())

        app_meta = AppMeta(**self_config['meta'])
        app_spec_data = self_config.pop('app_spec')

        app_spec_set = []
        for proc_num, app_spec_config in enumerate(app_spec_data):

            proc_id = app_spec_config.get('proc_id', None)
            proc_id = proc_num if proc_id is None else proc_id

            app_spec_config = dict(app_spec_config)
            app_spec_config['proc_id'] = proc_id
            app_spec = AppSpec.from_config(app_spec_config)
            app_spec_set.append(app_spec)

        return cls(meta=app_meta, app_spec=app_spec_set)
Beispiel #31
0
def _sorted_items(mapping: typing.Mapping) -> typing.Generator:
    """Given a mapping where values are iterables, yield items whose values contained references are not used as
    keys first:

    Example:
        >>> dct = {'two': ('two', 'one', 'foo'), 'one': ('hi', 'six', 'net'), 'six': ('three', 'four'), 'foo': ['bar']}
        >>> for k, v in _sorted_items(dct):
        ...     print(k, v)
        ...
        six ('three', 'four')
        foo ['bar']
        one ('hi', 'six', 'net')
        two ('two', 'one', 'foo')
    """
    to_yield = set(mapping)
    while to_yield:
        for key, values in mapping.items():
            if key not in to_yield or (
                    to_yield - {key}
                    & set(values)):  # other keys left to yield before this one
                continue
            yield key, values
            to_yield.remove(key)
Beispiel #32
0
def combine_dicts(
    first: typing.Mapping,
    second: typing.Mapping,
    recursive: bool = False,
    list_merge: str = 'replace',
) -> typing.Dict:
    result = dict(first)

    for key, value in second.items():
        try:
            existing = result[key]
        except KeyError:
            pass
        else:
            if (recursive and isinstance(existing, abcoll.MutableMapping)
                    and isinstance(value, abcoll.Mapping)):
                value = combine_dicts(existing, value)
            elif (isinstance(existing, abcoll.Sequence)
                  and isinstance(value, abcoll.Sequence)):
                value = combine_lists(existing, value, list_merge)

        result[key] = value

    return result
Beispiel #33
0
 def fromdict(cls, mapping: t.Mapping, ncols=1):
     iterable = ["{}: {}".format(*i) for i in mapping.items()]
     return cls.new(iterable, ncols)
def attributes_with_annotations(namespace: typing.Mapping):
    for name, value in namespace.items():
        yield from _if_annotation(name, value)