def __init__(self, vocab, hparams=None): self._hparams = HParams(hparams, self.default_hparams()) # Initialize embeddings init_fn_kwargs = self._hparams.init_fn.kwargs.todict() if "shape" in init_fn_kwargs or "size" in init_fn_kwargs: raise ValueError("Argument 'shape' or 'size' must not be " "specified. They are inferred automatically.") init_fn = utils.get_function( self._hparams.init_fn.type, ["numpy.random", "numpy", "texar.tf.custom"]) try: self._word_vecs = init_fn(size=[len(vocab), self._hparams.dim], **init_fn_kwargs) except TypeError: self._word_vecs = init_fn(shape=[len(vocab), self._hparams.dim], **init_fn_kwargs) # Optionally read embeddings from file if self._hparams.file is not None and self._hparams.file != "": read_fn = utils.get_function(self._hparams.read_fn, [ "texar.tf.data.embedding", "texar.tf.data", "texar.tf.custom" ]) self._word_vecs = \ read_fn(self._hparams.file, vocab, self._word_vecs)
def __init__( self, memory, memory_sequence_length=None, cell=None, cell_dropout_mode=None, vocab_size=None, output_layer=None, #attention_layer=None, # TODO(zhiting): only valid for tf>=1.0 cell_input_fn=None, hparams=None): RNNDecoderBase.__init__(self, cell, vocab_size, output_layer, cell_dropout_mode, hparams) attn_hparams = self._hparams['attention'] attn_kwargs = attn_hparams['kwargs'].todict() # Parse the 'probability_fn' argument if 'probability_fn' in attn_kwargs: prob_fn = attn_kwargs['probability_fn'] if prob_fn is not None and not callable(prob_fn): prob_fn = utils.get_function(prob_fn, [ 'tensorflow.nn', 'tensorflow.contrib.sparsemax', 'tensorflow.contrib.seq2seq' ]) attn_kwargs['probability_fn'] = prob_fn attn_kwargs.update({ "memory_sequence_length": memory_sequence_length, "memory": memory }) self._attn_kwargs = attn_kwargs attn_modules = ['tensorflow.contrib.seq2seq', 'texar.tf.custom'] # Use variable_scope to ensure all trainable variables created in # the attention mechanism are collected with tf.variable_scope(self.variable_scope): attention_mechanism = utils.check_or_get_instance( attn_hparams["type"], attn_kwargs, attn_modules, classtype=tf.contrib.seq2seq.AttentionMechanism) self._attn_cell_kwargs = { "attention_layer_size": attn_hparams["attention_layer_size"], "alignment_history": attn_hparams["alignment_history"], "output_attention": attn_hparams["output_attention"], } self._cell_input_fn = cell_input_fn # Use variable_scope to ensure all trainable variables created in # AttentionWrapper are collected with tf.variable_scope(self.variable_scope): #if attention_layer is not None: # self._attn_cell_kwargs["attention_layer_size"] = None attn_cell = AttentionWrapper( self._cell, attention_mechanism, cell_input_fn=self._cell_input_fn, #attention_layer=attention_layer, **self._attn_cell_kwargs) self._cell = attn_cell
def _make_bucket_length_fn(self): length_fn = self._hparams.bucket_length_fn if not length_fn: length_fn = lambda x: x[self.length_name] elif not is_callable(length_fn): # pylint: disable=redefined-variable-type length_fn = utils.get_function(length_fn, ["texar.tf.custom"]) return length_fn
def get_gradient_clip_fn(hparams=None): """Creates a gradient clipping function based on the hyperparameters. See the :attr:`gradient_clip` field in :meth:`~texar.tf.core.default_optimization_hparams` for all hyperparameters and default values. The gradient clipping function takes a list of `(gradients, variables)` tuples and returns a list of `(clipped_gradients, variables)` tuples. Typical examples include :tf_main:`tf.clip_by_global_norm <clip_by_global_norm>`, :tf_main:`tf.clip_by_value <clip_by_value>`, :tf_main:`tf.clip_by_norm <clip_by_norm>`, :tf_main:`tf.clip_by_average_norm <clip_by_average_norm>`, etc. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. Returns: function or `None`: If hparams["type"] is specified, returns the respective function. If hparams["type"] is empty, returns `None`. """ if hparams is None or isinstance(hparams, dict): hparams = HParams( hparams, default_optimization_hparams()["gradient_clip"]) fn_type = hparams["type"] if fn_type is None or fn_type == "": return None fn_modules = ["tensorflow", "texar.tf.custom"] clip_fn = utils.get_function(fn_type, fn_modules) clip_fn_args = utils.get_args(clip_fn) fn_kwargs = hparams["kwargs"] if isinstance(fn_kwargs, HParams): fn_kwargs = fn_kwargs.todict() def grad_clip_fn(grads_and_vars): """Gradient clipping function. Args: grads_and_vars (list): A list of `(gradients, variables)` tuples. Returns: list: A list of `(clipped_gradients, variables)` tuples. """ grads, vars_ = zip(*grads_and_vars) if clip_fn == tf.clip_by_global_norm: clipped_grads, _ = clip_fn(t_list=grads, **fn_kwargs) elif 't_list' in clip_fn_args: clipped_grads = clip_fn(t_list=grads, **fn_kwargs) elif 't' in clip_fn_args: # e.g., tf.clip_by_value clipped_grads = [clip_fn(t=grad, **fn_kwargs) for grad in grads] return list(zip(clipped_grads, vars_)) return grad_clip_fn
def get_initializer(hparams=None): """Returns an initializer instance. .. role:: python(code) :language: python Args: hparams (dict or HParams, optional): Hyperparameters with the structure .. code-block:: python { "type": "initializer_class_or_function", "kwargs": { #... } } The "type" field can be a initializer class, its name or module path, or class instance. If class name is provided, the class must be from one the following modules: :tf_main:`tf.initializers <initializers>`, :tf_main:`tf.keras.initializers <keras/initializers>`, :tf_main:`tf < >`, and :mod:`texar.tf.custom`. The class is created by :python:`initializer_class(**kwargs)`. If a class instance is given, "kwargs" is ignored and can be omitted. Besides, the "type" field can also be an initialization function called with :python:`initialization_fn(**kwargs)`. In this case "type" can be the function, or its name or module path. If function name is provided, the function must be from one of the above modules or module `tf.contrib.layers`. If no keyword argument is required, "kwargs" can be omitted. Returns: An initializer instance. `None` if :attr:`hparams` is `None`. """ if hparams is None: return None kwargs = hparams.get("kwargs", {}) if isinstance(kwargs, HParams): kwargs = kwargs.todict() modules = [ "tensorflow.initializers", "tensorflow.keras.initializers", "tensorflow", "texar.tf.custom" ] try: initializer = utils.check_or_get_instance(hparams["type"], kwargs, modules) except (TypeError, ValueError): modules = ['tensorflow.contrib.layers'] + modules initializer_fn = utils.get_function(hparams["type"], modules) initializer = initializer_fn(**kwargs) return initializer
def get_activation_fn(fn_name="identity", kwargs=None): """Returns an activation function `fn` with the signature `output = fn(input)`. If the function specified by :attr:`fn_name` has more than one arguments without default values, then all these arguments except the input feature argument must be specified in :attr:`kwargs`. Arguments with default values can also be specified in :attr:`kwargs` to take values other than the defaults. In this case a partial function is returned with the above signature. Args: fn_name (str or callable): An activation function, or its name or module path. The function can be: - Built-in function defined in :tf_main:`tf < >` or :tf_main:`tf.nn <nn>`, e.g., :tf_main:`tf.identity <identity>`. - User-defined activation functions in module :mod:`texar.tf.custom`. - External activation functions. Must provide the full module path, e.g., "my_module.my_activation_fn". kwargs (optional): A `dict` or instance of :class:`~texar.tf.HParams` containing the keyword arguments of the activation function. Returns: An activation function. `None` if :attr:`fn_name` is `None`. """ if fn_name is None: return None fn_modules = [ 'tensorflow', 'tensorflow.nn', 'texar.tf.custom', 'texar.tf.core.layers' ] activation_fn_ = utils.get_function(fn_name, fn_modules) activation_fn = activation_fn_ # Make a partial function if necessary if kwargs is not None: if isinstance(kwargs, HParams): kwargs = kwargs.todict() def _partial_fn(features): return activation_fn_(features, **kwargs) activation_fn = _partial_fn return activation_fn
def _make_bucket_length_fn(self): length_fn = self._hparams.bucket_length_fn if not length_fn: # Uses the length of the first text data i = -1 for i, hparams_i in enumerate(self._hparams.datasets): if _is_text_data(hparams_i["data_type"]): break if i < 0: raise ValueError("Undefined `length_fn`.") length_fn = lambda x: x[self.length_name(i)] elif not is_callable(length_fn): # pylint: disable=redefined-variable-type length_fn = utils.get_function(length_fn, ["texar.tf.custom"]) return length_fn
def _make_other_transformations(other_trans_hparams, data_spec): """Creates a list of tranformation functions based on the hyperparameters. Args: other_trans_hparams (list): A list of transformation functions, names, or full paths. data_spec: An instance of :class:`texar.tf.data._DataSpec` to be passed to transformation functions. Returns: A list of transformation functions. """ other_trans = [] for tran in other_trans_hparams: if not is_callable(tran): tran = utils.get_function(tran, ["texar.tf.custom"]) other_trans.append(dsutils.make_partial(tran, data_spec)) return other_trans
def get_constraint_fn(fn_name="NonNeg"): """Returns a constraint function. .. role:: python(code) :language: python The function must follow the signature: :python:`w_ = constraint_fn(w)`. Args: fn_name (str or callable): The name or full path to a constraint function, or the function itself. The function can be: - Built-in constraint functions defined in modules \ :tf_main:`tf.keras.constraints <keras/constraints>` \ (e.g., :tf_main:`NonNeg <keras/constraints/NonNeg>`) \ or :tf_main:`tf < >` or :tf_main:`tf.nn <nn>` \ (e.g., activation functions). - User-defined function in :mod:`texar.tf.custom`. - Externally defined function. Must provide the full path, \ e.g., `"my_module.my_constraint_fn"`. If a callable is provided, then it is returned directly. Returns: The constraint function. `None` if :attr:`fn_name` is `None`. """ if fn_name is None: return None fn_modules = [ 'tensorflow.keras.constraints', 'tensorflow', 'tensorflow.nn', 'texar.tf.custom' ] constraint_fn = utils.get_function(fn_name, fn_modules) return constraint_fn
def get_learning_rate_decay_fn(hparams=None): """Creates learning rate decay function based on the hyperparameters. See the :attr:`learning_rate_decay` field in :meth:`~texar.tf.core.default_optimization_hparams` for all hyperparameters and default values. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. Returns: function or None: If hparams["type"] is specified, returns a function that takes `(learning_rate, step, **kwargs)` and returns a decayed learning rate. If hparams["type"] is empty, returns `None`. """ if hparams is None or isinstance(hparams, dict): hparams = HParams( hparams, default_optimization_hparams()["learning_rate_decay"]) fn_type = hparams["type"] if fn_type is None or fn_type == "": return None fn_modules = ["tensorflow.train", "texar.tf.custom"] decay_fn = utils.get_function(fn_type, fn_modules) fn_kwargs = hparams["kwargs"] if fn_kwargs is HParams: fn_kwargs = fn_kwargs.todict() start_step = tf.cast(hparams["start_decay_step"], tf.int32) end_step = tf.cast(hparams["end_decay_step"], tf.int32) def lr_decay_fn(learning_rate, global_step): """Learning rate decay function. Args: learning_rate (float or Tensor): The original learning rate. global_step (int or scalar int Tensor): optimization step counter. Returns: scalar float Tensor: decayed learning rate. """ offset_global_step = tf.maximum( tf.minimum(tf.cast(global_step, tf.int32), end_step) - start_step, 0) if decay_fn == tf.train.piecewise_constant: decayed_lr = decay_fn(x=offset_global_step, **fn_kwargs) else: fn_kwargs_ = { "learning_rate": learning_rate, "global_step": offset_global_step} fn_kwargs_.update(fn_kwargs) decayed_lr = utils.call_function_with_redundant_kwargs( decay_fn, fn_kwargs_) decayed_lr = tf.maximum(decayed_lr, hparams["min_learning_rate"]) return decayed_lr return lr_decay_fn
def _build(self, distribution='MultivariateNormalDiag', distribution_kwargs=None, transform=True, num_samples=None): """Samples from a distribution and optionally performs transformation with an MLP layer. The inputs and outputs are the same as :class:`~texar.tf.modules.ReparameterizedStochasticConnector` except that the distribution does not need to be reparameterizable, and gradient cannot be back-propagate through the samples. Args: distribution: A instance of subclass of :tf_main:`TF Distribution <distributions/Distribution>`, or :tf_hmpg:`tensorflow_probability Distribution <probability>`. Can be a class, its name or module path, or a class instance. distribution_kwargs (dict, optional): Keyword arguments for the distribution constructor. Ignored if `distribution` is a class instance. transform (bool): Whether to perform MLP transformation of the distribution samples. If `False`, the structure/shape of a sample must match :attr:`output_size`. num_samples (optional): An `int` or `int` Tensor. Number of samples to generate. If not given, generate a single sample. Note that if batch size has already been included in `distribution`'s dimensionality, `num_samples` should be left as `None`. Returns: A tuple (output, sample), where - output: A Tensor or a (nested) tuple of Tensors with the same \ structure and size of :attr:`output_size`. The batch dimension \ equals :attr:`num_samples` if specified, or is determined by the \ distribution dimensionality. If :attr:`transform` is `False`, \ :attr:`output` will be equal to :attr:`sample`. - sample: The sample from the distribution, prior to transformation. Raises: ValueError: The output does not match :attr:`output_size`. """ dstr = check_or_get_instance( distribution, distribution_kwargs, ["tensorflow.distributions", "tensorflow_probability.distributions", "texar.tf.custom"]) if num_samples: sample = dstr.sample(num_samples) else: sample = dstr.sample() if dstr.event_shape == []: sample = tf.reshape(sample, sample.shape.concatenate(tf.TensorShape(1))) # Disable gradients through samples sample = tf.stop_gradient(sample) sample = tf.cast(sample, tf.float32) if transform: fn_modules = ['tensorflow', 'tensorflow.nn', 'texar.tf.custom'] activation_fn = get_function(self.hparams.activation_fn, fn_modules) output = _mlp_transform(sample, self._output_size, activation_fn) else: output = sample _assert_same_size(output, self._output_size) if not self._built: self._add_internal_trainable_variables() self._built = True return output, sample