def __init__(self, vocab: Dict[str, int], hparams=None):
self._hparams = HParams(hparams, self.default_hparams())
# Initialize embeddings
init_fn_kwargs = self._hparams.init_fn.kwargs.todict()
if "shape" in init_fn_kwargs or "size" in init_fn_kwargs:
raise ValueError("Argument 'shape' or 'size' must not be "
"specified. They are inferred automatically.")
init_fn: Callable[..., np.ndarray]
init_fn = utils.get_function(
self._hparams.init_fn.type,
["numpy.random", "numpy", "texar.torch.custom"])
try:
self._word_vecs = init_fn( # type: ignore
size=[len(vocab), self._hparams.dim],
**init_fn_kwargs)
except TypeError:
self._word_vecs = init_fn( # type: ignore
shape=[len(vocab), self._hparams.dim],
**init_fn_kwargs)
# Optionally read embeddings from file
if self._hparams.file is not None and self._hparams.file != "":
read_fn: Callable[[str, Dict[str, int], np.ndarray], np.ndarray]
read_fn = utils.get_function( # type: ignore
self._hparams.read_fn, [
"texar.torch.data.embedding", "texar.torch.data",
"texar.torch.custom"
])
self._word_vecs = read_fn(
self._hparams.file, # type: ignore
vocab,
self._word_vecs)
def get_grad_clip_fn(hparams: Optional[Union[HParams,
Dict[str, Any]]] = None) -> \
Optional[Callable[[torch.Tensor], Optional[torch.Tensor]]]:
r"""Create a gradient clipping function.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.torch.core.default_optimization_hparams` for
all hyperparameters and default values.
Returns:
A gradient clipping function.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_grad_clip = hparams["gradient_clip"]
grad_clip_type = hparams_grad_clip["type"]
if grad_clip_type == "" or grad_clip_type is None:
grad_clip_fn = None
else:
grad_clip_modules = ['torch.nn.utils', 'texar.torch.custom']
grad_clip_fn = utils.get_function(grad_clip_type, grad_clip_modules)
grad_clip_fn_kwargs = hparams_grad_clip["kwargs"].todict()
grad_clip_fn = functools.partial(grad_clip_fn, **grad_clip_fn_kwargs)
return grad_clip_fn
def get_activation_fn(fn_name: Optional[Union[str,
Callable[[torch.Tensor],
torch.Tensor]]] = None,
kwargs: Union[HParams, Dict, None] = None) \
-> Optional[Callable[[torch.Tensor], torch.Tensor]]:
r"""Returns an activation function `fn` with the signature
`output = fn(input)`.
If the function specified by :attr:`fn_name` has more than one arguments
without default values, then all these arguments except the input feature
argument must be specified in :attr:`kwargs`. Arguments with default values
can also be specified in :attr:`kwargs` to take values other than the
defaults. In this case a partial function is returned with the above
signature.
Args:
fn_name (str or callable): An activation function, or its name or
module path. The function can be:
- Built-in function defined in
:torch_docs:`torch.nn.functional<nn.html#torch-nn-functional>`
- User-defined activation functions in module
:mod:`texar.torch.custom`.
- External activation functions. Must provide the full module path,
e.g., ``"my_module.my_activation_fn"``.
kwargs (optional): A `dict` or instance of :class:`~texar.torch.HParams`
containing the keyword arguments of the activation function.
Returns:
An activation function. `None` if :attr:`fn_name` is `None`.
"""
if fn_name is None:
return None
fn_modules = [
'torch', 'torch.nn.functional', 'texar.torch.custom',
'texar.torch.core.layers'
]
activation_fn_ = utils.get_function(fn_name, fn_modules)
activation_fn = activation_fn_
# Make a partial function if necessary
if kwargs is not None:
if isinstance(kwargs, HParams):
kwargs = kwargs.todict()
def _partial_fn(features):
return activation_fn_(features, **kwargs)
activation_fn = _partial_fn
return activation_fn
def get_initializer(hparams=None) \
-> Optional[Callable[[torch.Tensor], torch.Tensor]]:
r"""Returns an initializer instance.
Args:
hparams (dict or HParams, optional): Hyperparameters with the structure
.. code-block:: python
{
"type": "initializer_class_or_function",
"kwargs": {
# ...
}
}
The `"type"` field can be a function name or module path. If name is
provided, it be must be from one the following modules:
:torch_docs:`torch.nn.init <nn.html#torch-nn-init>` and
:mod:`texar.torch.custom`.
Besides, the `"type"` field can also be an initialization function
called with :python:`initialization_fn(**kwargs)`. In this case
`"type"` can be the function, or its name or module path. If no
keyword argument is required, `"kwargs"` can be omitted.
Returns:
An initializer instance. `None` if :attr:`hparams` is `None`.
"""
if hparams is None:
return None
kwargs = hparams.get('kwargs', {})
if isinstance(kwargs, HParams):
kwargs = kwargs.todict()
modules = ['torch.nn.init', 'torch', 'texar.torch.custom']
initializer_fn = utils.get_function(hparams['type'], modules)
initializer = functools.partial(initializer_fn, **kwargs)
return initializer
def __init__(self,
input_size: int,
encoder_output_size: int,
vocab_size: int,
token_embedder: Optional[TokenEmbedder] = None,
token_pos_embedder: Optional[TokenPosEmbedder] = None,
cell: Optional[RNNCellBase] = None,
output_layer: Optional[Union[nn.Module, torch.Tensor]] = None,
cell_input_fn: Optional[Callable[[torch.Tensor, torch.Tensor],
torch.Tensor]] = None,
hparams=None):
super().__init__(input_size,
vocab_size,
token_embedder,
token_pos_embedder,
cell=cell,
output_layer=output_layer,
hparams=hparams)
attn_hparams = self._hparams['attention']
attn_kwargs = attn_hparams['kwargs'].todict()
# Compute the correct input_size internally.
if cell is None:
if cell_input_fn is None:
if attn_hparams["attention_layer_size"] is None:
input_size += encoder_output_size
else:
input_size += attn_hparams["attention_layer_size"]
else:
if attn_hparams["attention_layer_size"] is None:
input_size = cell_input_fn(
torch.empty(input_size),
torch.empty(encoder_output_size)).shape[-1]
else:
input_size = cell_input_fn(
torch.empty(input_size),
torch.empty(
attn_hparams["attention_layer_size"])).shape[-1]
self._cell = layers.get_rnn_cell(input_size,
self._hparams.rnn_cell)
# Parse the `probability_fn` argument
if 'probability_fn' in attn_kwargs:
prob_fn = attn_kwargs['probability_fn']
if prob_fn is not None and not callable(prob_fn):
prob_fn = get_function(
prob_fn, ['torch.nn.functional', 'texar.torch.core'])
attn_kwargs['probability_fn'] = prob_fn
# Parse `encoder_output_size` and `decoder_output_size` arguments
if attn_hparams['type'] in [
'BahdanauAttention', 'BahdanauMonotonicAttention'
]:
attn_kwargs.update({"decoder_output_size": self._cell.hidden_size})
attn_kwargs.update({"encoder_output_size": encoder_output_size})
attn_modules = ['texar.torch.core']
# TODO: Support multiple attention mechanisms.
self.attention_mechanism: AttentionMechanism
self.attention_mechanism = check_or_get_instance(
attn_hparams["type"],
attn_kwargs,
attn_modules,
classtype=AttentionMechanism)
self._attn_cell_kwargs = {
"attention_layer_size": attn_hparams["attention_layer_size"],
"alignment_history": attn_hparams["alignment_history"],
"output_attention": attn_hparams["output_attention"],
}
self._cell_input_fn = cell_input_fn
if attn_hparams["output_attention"] and vocab_size is not None and \
self.attention_mechanism is not None:
if attn_hparams["attention_layer_size"] is None:
self._output_layer = nn.Linear(encoder_output_size, vocab_size)
else:
self._output_layer = nn.Linear(
sum(attn_hparams["attention_layer_size"]) if isinstance(
attn_hparams["attention_layer_size"], list) else
attn_hparams["attention_layer_size"], vocab_size)
attn_cell = AttentionWrapper(self._cell,
self.attention_mechanism,
cell_input_fn=self._cell_input_fn,
**self._attn_cell_kwargs)
self._cell: AttentionWrapper = attn_cell
self.memory: Optional[torch.Tensor] = None
self.memory_sequence_length: Optional[torch.LongTensor] = None
