def __init__(self, vocab: Dict[str, int], hparams=None): self._hparams = HParams(hparams, self.default_hparams()) # Initialize embeddings init_fn_kwargs = self._hparams.init_fn.kwargs.todict() if "shape" in init_fn_kwargs or "size" in init_fn_kwargs: raise ValueError("Argument 'shape' or 'size' must not be " "specified. They are inferred automatically.") init_fn: Callable[..., np.ndarray] init_fn = utils.get_function( self._hparams.init_fn.type, ["numpy.random", "numpy", "texar.torch.custom"]) try: self._word_vecs = init_fn( # type: ignore size=[len(vocab), self._hparams.dim], **init_fn_kwargs) except TypeError: self._word_vecs = init_fn( # type: ignore shape=[len(vocab), self._hparams.dim], **init_fn_kwargs) # Optionally read embeddings from file if self._hparams.file is not None and self._hparams.file != "": read_fn: Callable[[str, Dict[str, int], np.ndarray], np.ndarray] read_fn = utils.get_function( # type: ignore self._hparams.read_fn, [ "texar.torch.data.embedding", "texar.torch.data", "texar.torch.custom" ]) self._word_vecs = read_fn( self._hparams.file, # type: ignore vocab, self._word_vecs)
def get_grad_clip_fn(hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \ Optional[Callable[[torch.Tensor], Optional[torch.Tensor]]]: r"""Create a gradient clipping function. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. See :func:`~texar.torch.core.default_optimization_hparams` for all hyperparameters and default values. Returns: A gradient clipping function. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()) hparams_grad_clip = hparams["gradient_clip"] grad_clip_type = hparams_grad_clip["type"] if grad_clip_type == "" or grad_clip_type is None: grad_clip_fn = None else: grad_clip_modules = ['torch.nn.utils', 'texar.torch.custom'] grad_clip_fn = utils.get_function(grad_clip_type, grad_clip_modules) grad_clip_fn_kwargs = hparams_grad_clip["kwargs"].todict() grad_clip_fn = functools.partial(grad_clip_fn, **grad_clip_fn_kwargs) return grad_clip_fn
def get_activation_fn(fn_name: Optional[Union[str, Callable[[torch.Tensor], torch.Tensor]]] = None, kwargs: Union[HParams, Dict, None] = None) \ -> Optional[Callable[[torch.Tensor], torch.Tensor]]: r"""Returns an activation function `fn` with the signature `output = fn(input)`. If the function specified by :attr:`fn_name` has more than one arguments without default values, then all these arguments except the input feature argument must be specified in :attr:`kwargs`. Arguments with default values can also be specified in :attr:`kwargs` to take values other than the defaults. In this case a partial function is returned with the above signature. Args: fn_name (str or callable): An activation function, or its name or module path. The function can be: - Built-in function defined in :torch_docs:`torch.nn.functional<nn.html#torch-nn-functional>` - User-defined activation functions in module :mod:`texar.torch.custom`. - External activation functions. Must provide the full module path, e.g., ``"my_module.my_activation_fn"``. kwargs (optional): A `dict` or instance of :class:`~texar.torch.HParams` containing the keyword arguments of the activation function. Returns: An activation function. `None` if :attr:`fn_name` is `None`. """ if fn_name is None: return None fn_modules = [ 'torch', 'torch.nn.functional', 'texar.torch.custom', 'texar.torch.core.layers' ] activation_fn_ = utils.get_function(fn_name, fn_modules) activation_fn = activation_fn_ # Make a partial function if necessary if kwargs is not None: if isinstance(kwargs, HParams): kwargs = kwargs.todict() def _partial_fn(features): return activation_fn_(features, **kwargs) activation_fn = _partial_fn return activation_fn
def get_initializer(hparams=None) \ -> Optional[Callable[[torch.Tensor], torch.Tensor]]: r"""Returns an initializer instance. Args: hparams (dict or HParams, optional): Hyperparameters with the structure .. code-block:: python { "type": "initializer_class_or_function", "kwargs": { # ... } } The `"type"` field can be a function name or module path. If name is provided, it be must be from one the following modules: :torch_docs:`torch.nn.init <nn.html#torch-nn-init>` and :mod:`texar.torch.custom`. Besides, the `"type"` field can also be an initialization function called with :python:`initialization_fn(**kwargs)`. In this case `"type"` can be the function, or its name or module path. If no keyword argument is required, `"kwargs"` can be omitted. Returns: An initializer instance. `None` if :attr:`hparams` is `None`. """ if hparams is None: return None kwargs = hparams.get('kwargs', {}) if isinstance(kwargs, HParams): kwargs = kwargs.todict() modules = ['torch.nn.init', 'torch', 'texar.torch.custom'] initializer_fn = utils.get_function(hparams['type'], modules) initializer = functools.partial(initializer_fn, **kwargs) return initializer
def __init__(self, input_size: int, encoder_output_size: int, vocab_size: int, token_embedder: Optional[TokenEmbedder] = None, token_pos_embedder: Optional[TokenPosEmbedder] = None, cell: Optional[RNNCellBase] = None, output_layer: Optional[Union[nn.Module, torch.Tensor]] = None, cell_input_fn: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None, hparams=None): super().__init__(input_size, vocab_size, token_embedder, token_pos_embedder, cell=cell, output_layer=output_layer, hparams=hparams) attn_hparams = self._hparams['attention'] attn_kwargs = attn_hparams['kwargs'].todict() # Compute the correct input_size internally. if cell is None: if cell_input_fn is None: if attn_hparams["attention_layer_size"] is None: input_size += encoder_output_size else: input_size += attn_hparams["attention_layer_size"] else: if attn_hparams["attention_layer_size"] is None: input_size = cell_input_fn( torch.empty(input_size), torch.empty(encoder_output_size)).shape[-1] else: input_size = cell_input_fn( torch.empty(input_size), torch.empty( attn_hparams["attention_layer_size"])).shape[-1] self._cell = layers.get_rnn_cell(input_size, self._hparams.rnn_cell) # Parse the `probability_fn` argument if 'probability_fn' in attn_kwargs: prob_fn = attn_kwargs['probability_fn'] if prob_fn is not None and not callable(prob_fn): prob_fn = get_function( prob_fn, ['torch.nn.functional', 'texar.torch.core']) attn_kwargs['probability_fn'] = prob_fn # Parse `encoder_output_size` and `decoder_output_size` arguments if attn_hparams['type'] in [ 'BahdanauAttention', 'BahdanauMonotonicAttention' ]: attn_kwargs.update({"decoder_output_size": self._cell.hidden_size}) attn_kwargs.update({"encoder_output_size": encoder_output_size}) attn_modules = ['texar.torch.core'] # TODO: Support multiple attention mechanisms. self.attention_mechanism: AttentionMechanism self.attention_mechanism = check_or_get_instance( attn_hparams["type"], attn_kwargs, attn_modules, classtype=AttentionMechanism) self._attn_cell_kwargs = { "attention_layer_size": attn_hparams["attention_layer_size"], "alignment_history": attn_hparams["alignment_history"], "output_attention": attn_hparams["output_attention"], } self._cell_input_fn = cell_input_fn if attn_hparams["output_attention"] and vocab_size is not None and \ self.attention_mechanism is not None: if attn_hparams["attention_layer_size"] is None: self._output_layer = nn.Linear(encoder_output_size, vocab_size) else: self._output_layer = nn.Linear( sum(attn_hparams["attention_layer_size"]) if isinstance( attn_hparams["attention_layer_size"], list) else attn_hparams["attention_layer_size"], vocab_size) attn_cell = AttentionWrapper(self._cell, self.attention_mechanism, cell_input_fn=self._cell_input_fn, **self._attn_cell_kwargs) self._cell: AttentionWrapper = attn_cell self.memory: Optional[torch.Tensor] = None self.memory_sequence_length: Optional[torch.LongTensor] = None