def test_from_params(self):
# Save a vocab to check we can load it from_params.
vocab_dir = os.path.join(self.TEST_DIR, 'vocab_save')
vocab = Vocabulary(non_padded_namespaces=["a", "c"])
vocab.add_token_to_namespace("a0", namespace="a") # non-padded, should start at 0
vocab.add_token_to_namespace("a1", namespace="a")
vocab.add_token_to_namespace("a2", namespace="a")
vocab.add_token_to_namespace("b2", namespace="b") # padded, should start at 2
vocab.add_token_to_namespace("b3", namespace="b")
vocab.save_to_files(vocab_dir)
params = Params({"directory_path": vocab_dir})
vocab2 = Vocabulary.from_params(params)
assert vocab.get_index_to_token_vocabulary("a") == vocab2.get_index_to_token_vocabulary("a")
assert vocab.get_index_to_token_vocabulary("b") == vocab2.get_index_to_token_vocabulary("b")
# Test case where we build a vocab from a dataset.
vocab2 = Vocabulary.from_params(Params({}), self.dataset)
assert vocab2.get_index_to_token_vocabulary("tokens") == {0: '@@PADDING@@',
1: '@@UNKNOWN@@',
2: 'a', 3: 'c', 4: 'b'}
# Test from_params raises when we have neither a dataset and a vocab_directory.
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(Params({}))
# Test from_params raises when there are any other dict keys
# present apart from 'vocabulary_directory' and we aren't calling from_dataset.
with pytest.raises(ConfigurationError):
_ = Vocabulary.from_params(Params({"directory_path": vocab_dir, "min_count": {'tokens': 2}}))
def from_params(cls, params: Params) -> 'B':
params.add_file_to_archive("filename")
filename = params.pop("filename")
c_params = params.pop("c")
c = C.from_params(c_params)
return cls(filename, c)
def from_params(cls, params: Params) -> 'ELMoTokenCharactersIndexer':
"""
Parameters
----------
namespace : ``str``, optional (default=``elmo_characters``)
"""
namespace = params.pop('namespace', 'elmo_characters')
params.assert_empty(cls.__name__)
return cls(namespace=namespace)
def from_params(cls,
params: Params,
instances: Iterable['adi.Instance'] = None):
"""
There are two possible ways to build a vocabulary; from a
collection of instances, using :func:`Vocabulary.from_instances`, or
from a pre-saved vocabulary, using :func:`Vocabulary.from_files`.
This method wraps both of these options, allowing their specification
from a ``Params`` object, generated from a JSON configuration file.
Parameters
----------
params: Params, required.
dataset: Dataset, optional.
If ``params`` doesn't contain a ``vocabulary_directory`` key,
the ``Vocabulary`` can be built directly from a ``Dataset``.
Returns
-------
A ``Vocabulary``.
"""
vocabulary_directory = params.pop("directory_path", None)
if not vocabulary_directory and not instances:
raise ConfigurationError(
"You must provide either a Params object containing a "
"vocab_directory key or a Dataset to build a vocabulary from.")
if vocabulary_directory and instances:
logger.info("Loading Vocab from files instead of dataset.")
if vocabulary_directory:
params.assert_empty("Vocabulary - from files")
return Vocabulary.from_files(vocabulary_directory)
min_count = params.pop("min_count", None)
max_vocab_size = params.pop_int("max_vocab_size", None)
non_padded_namespaces = params.pop("non_padded_namespaces",
DEFAULT_NON_PADDED_NAMESPACES)
pretrained_files = params.pop("pretrained_files", {})
only_include_pretrained_words = params.pop_bool(
"only_include_pretrained_words", False)
params.assert_empty("Vocabulary - from dataset")
return Vocabulary.from_instances(
instances=instances,
min_count=min_count,
max_vocab_size=max_vocab_size,
non_padded_namespaces=non_padded_namespaces,
pretrained_files=pretrained_files,
only_include_pretrained_words=only_include_pretrained_words)
def test_load_from_file(self):
filename = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'fixtures/bidaf/experiment.json')
params = Params.from_file(filename)
assert "dataset_reader" in params
assert "trainer" in params
model_params = params.pop("model")
assert model_params.pop("type") == "bidaf"
def from_params(cls, params: Params) -> 'Initializer': # type: ignore
# pylint: disable=arguments-differ
# Just a string - corresponds to the name of an initializer.
if isinstance(params, str):
return cls.by_name(params)()
else:
choice = params.pop_choice("type", cls.list_available())
return cls.by_name(choice).from_params(params)
def from_params(cls, params: Params) -> 'TokenCharactersIndexer':
"""
Parameters
----------
namespace : ``str``, optional (default=``token_characters``)
We will use this namespace in the :class:`Vocabulary` to map the characters in each token
to indices.
character_tokenizer : ``Params``, optional (default=``Params({})``)
We use a :class:`CharacterTokenizer` to handle splitting tokens into characters, as it has
options for byte encoding and other things. These parameters get passed to the character
tokenizer. The default is to use unicode characters and to retain casing.
"""
namespace = params.pop('namespace', 'token_characters')
character_tokenizer_params = params.pop('character_tokenizer', {})
character_tokenizer = CharacterTokenizer.from_params(
character_tokenizer_params)
params.assert_empty(cls.__name__)
return cls(namespace=namespace,
character_tokenizer=character_tokenizer)
def test_as_flat_dict(self):
params = Params({
'a': 10,
'b': {
'c': 20,
'd': 'stuff'
}
}).as_flat_dict()
assert params == {'a': 10, 'b.c': 20, 'b.d': 'stuff'}
def test_overrides(self):
filename = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'fixtures/bidaf/experiment.json')
overrides = '{ "train_data_path": "FOO", "model": { "type": "BAR" },'\
'model.text_field_embedder.tokens.type: "BAZ" }'
params = Params.from_file(filename, overrides)
assert "dataset_reader" in params
assert "trainer" in params
assert params["train_data_path"] == "FOO"
model_params = params.pop("model")
assert model_params.pop("type") == "BAR"
assert model_params["text_field_embedder.tokens.type"] == "BAZ"
def from_params(cls, params: Params) -> 'C':
params.add_file_to_archive("c_file")
c_file = params.pop("c_file")
return cls(c_file)
def from_params(cls, params: Params) -> 'A':
b_params = params.pop("b")
return cls(B.from_params(b_params))
def test_add_file_to_archive(self):
# Creates actual files since add_file_to_archive will throw an exception
# if the file does not exist.
tempdir = tempfile.mkdtemp()
my_file = os.path.join(tempdir, "my_file.txt")
my_other_file = os.path.join(tempdir, "my_other_file.txt")
open(my_file, 'w').close()
open(my_other_file, 'w').close()
# Some nested classes just to exercise the ``from_params``
# and ``add_file_to_archive`` methods.
class A:
def __init__(self, b: 'B') -> None:
self.b = b
@classmethod
def from_params(cls, params: Params) -> 'A':
b_params = params.pop("b")
return cls(B.from_params(b_params))
class B:
def __init__(self, filename: str, c: 'C') -> None:
self.filename = filename
self.c_dict = {"here": c}
@classmethod
def from_params(cls, params: Params) -> 'B':
params.add_file_to_archive("filename")
filename = params.pop("filename")
c_params = params.pop("c")
c = C.from_params(c_params)
return cls(filename, c)
class C:
def __init__(self, c_file: str) -> None:
self.c_file = c_file
@classmethod
def from_params(cls, params: Params) -> 'C':
params.add_file_to_archive("c_file")
c_file = params.pop("c_file")
return cls(c_file)
params = Params({
"a": {
"b": {
"filename": my_file,
"c": {
"c_file": my_other_file
}
}
}
})
# Construct ``A`` from params but then just throw it away.
A.from_params(params.pop("a"))
assert params.files_to_archive == {
"a.b.filename": my_file,
"a.b.c.c_file": my_other_file
}
def from_params(cls, params: Params): # type: ignore
return cls(**params.as_dict())
def create_kwargs(cls: Type[T], params: Params, **extras) -> Dict[str, Any]:
"""
Given some class, a `Params` object, and potentially other keyword arguments,
create a dict of keyword args suitable for passing to the class's constructor.
The function does this by finding the class's constructor, matching the constructor
arguments to entries in the `params` object, and instantiating values for the parameters
using the type annotation and possibly a from_params method.
Any values that are provided in the `extras` will just be used as is.
For instance, you might provide an existing `Vocabulary` this way.
"""
# Get the signature of the constructor.
signature = inspect.signature(cls.__init__)
kwargs: Dict[str, Any] = {}
# Iterate over all the constructor parameters and their annotations.
for name, param in signature.parameters.items():
# Skip "self". You're not *required* to call the first parameter "self",
# so in theory this logic is fragile, but if you don't call the self parameter
# "self" you kind of deserve what happens.
if name == "self":
continue
# If the annotation is a compound type like typing.Dict[str, int],
# it will have an __origin__ field indicating `typing.Dict`
# and an __args__ field indicating `(str, int)`. We capture both.
annotation = remove_optional(param.annotation)
origin = getattr(annotation, '__origin__', None)
args = getattr(annotation, '__args__', [])
# The parameter is optional if its default value is not the "no default" sentinel.
default = param.default
optional = default != _NO_DEFAULT
# Some constructors expect extra non-parameter items, e.g. vocab: Vocabulary.
# We check the provided `extras` for these and just use them if they exist.
if name in extras:
kwargs[name] = extras[name]
# The next case is when the parameter type is itself constructible from_params.
elif hasattr(annotation, 'from_params'):
if name in params:
# Our params have an entry for this, so we use that.
subparams = params.pop(name)
if takes_arg(annotation.from_params, 'extras'):
# If annotation.params accepts **extras, we need to pass them all along.
# For example, `BasicTextFieldEmbedder.from_params` requires a Vocabulary
# object, but `TextFieldEmbedder.from_params` does not.
subextras = extras
else:
# Otherwise, only supply the ones that are actual args; any additional ones
# will cause a TypeError.
subextras = {
k: v
for k, v in extras.items()
if takes_arg(annotation.from_params, k)
}
# In some cases we allow a string instead of a param dict, so
# we need to handle that case separately.
if isinstance(subparams, str):
kwargs[name] = annotation.by_name(subparams)()
else:
kwargs[name] = annotation.from_params(params=subparams,
**subextras)
elif not optional:
# Not optional and not supplied, that's an error!
raise ConfigurationError(
f"expected key {name} for {cls.__name__}")
else:
kwargs[name] = default
# If the parameter type is a Python primitive, just pop it off
# using the correct casting pop_xyz operation.
elif annotation == str:
kwargs[name] = (params.pop(name, default)
if optional else params.pop(name))
elif annotation == int:
kwargs[name] = (params.pop_int(name, default)
if optional else params.pop_int(name))
elif annotation == bool:
kwargs[name] = (params.pop_bool(name, default)
if optional else params.pop_bool(name))
elif annotation == float:
kwargs[name] = (params.pop_float(name, default)
if optional else params.pop_float(name))
# This is special logic for handling types like Dict[str, TokenIndexer], which it creates by
# instantiating each value from_params and returning the resulting dict.
elif origin in (Dict, dict) and len(args) == 2 and hasattr(
args[-1], 'from_params'):
value_cls = annotation.__args__[-1]
value_dict = {}
for key, value_params in params.pop(name, Params({})).items():
value_dict[key] = value_cls.from_params(params=value_params,
**extras)
kwargs[name] = value_dict
else:
# Pass it on as is and hope for the best. ¯\_(ツ)_/¯
if optional:
kwargs[name] = params.pop(name, default)
else:
kwargs[name] = params.pop(name)
params.assert_empty(cls.__name__)
return kwargs
def from_params(cls: Type[T], params: Params, **extras) -> T:
"""
This is the automatic implementation of `from_params`. Any class that subclasses `FromParams`
(or `Registrable`, which itself subclasses `FromParams`) gets this implementation for free.
If you want your class to be instantiated from params in the "obvious" way -- pop off parameters
and hand them to your constructor with the same names -- this provides that functionality.
If you need more complex logic in your from `from_params` method, you'll have to implement
your own method that overrides this one.
"""
# pylint: disable=protected-access
from elmo.common.registrable import Registrable # import here to avoid circular imports
info(
f"instantiating class {cls} from params {getattr(params, 'params', params)} "
f"and extras {extras}")
if params is None:
return None
registered_subclasses = Registrable._registry.get(cls)
if registered_subclasses is not None:
# We know ``cls`` inherits from Registrable, so we'll use a cast to make mypy happy.
# We have to use a disable to make pylint happy.
# pylint: disable=no-member
as_registrable = cast(Type[Registrable], cls)
default_to_first_choice = as_registrable.default_implementation is not None
choice = params.pop_choice(
"type",
choices=as_registrable.list_available(),
default_to_first_choice=default_to_first_choice)
subclass = registered_subclasses[choice]
# We want to call subclass.from_params. It's possible that it's just the "free"
# implementation here, in which case it accepts `**extras` and we are not able
# to make any assumptions about what extra parameters it needs.
#
# It's also possible that it has a custom `from_params` method. In that case it
# won't accept any **extra parameters and we'll need to filter them out.
if not takes_arg(subclass.from_params, 'extras'):
# Necessarily subclass.from_params is a custom implementation, so we need to
# pass it only the args it's expecting.
extras = {
k: v
for k, v in extras.items()
if takes_arg(subclass.from_params, k)
}
return subclass.from_params(params=params, **extras)
else:
# This is not a base class, so convert our params and extras into a dict of kwargs.
if cls.__init__ == object.__init__:
# This class does not have an explicit constructor, so don't give it any kwargs.
# Without this logic, create_kwargs will look at object.__init__ and see that
# it takes *args and **kwargs and look for those.
kwargs: Dict[str, Any] = {}
else:
# This class has a constructor, so create kwargs for it.
kwargs = create_kwargs(cls, params, **extras)
return cls(**kwargs) # type: ignore