def test_type_kwargs(self): """The the special cases involving "type" and "kwargs" hyperparameters. """ default_hparams = {"type": "type_name", "kwargs": {"arg1": "argv1"}} hparams = {"type": "type_name"} hparams_ = HParams(hparams, default_hparams) self.assertEqual(hparams_.kwargs.todict(), default_hparams["kwargs"]) hparams = {"type": "type_name", "kwargs": {"arg2": "argv2"}} hparams_ = HParams(hparams, default_hparams) full_kwargs = {} full_kwargs.update(default_hparams["kwargs"]) full_kwargs.update(hparams["kwargs"]) self.assertEqual(hparams_.kwargs.todict(), full_kwargs) hparams = {"kwargs": {"arg2": "argv2"}} hparams_ = HParams(hparams, default_hparams) self.assertEqual(hparams_.kwargs.todict(), full_kwargs) hparams = {"type": "type_name2"} hparams_ = HParams(hparams, default_hparams) self.assertEqual(hparams_.kwargs.todict(), {}) hparams = {"type": "type_name2", "kwargs": {"arg3": "argv3"}} hparams_ = HParams(hparams, default_hparams) self.assertEqual(hparams_.kwargs.todict(), hparams["kwargs"])
def setUp(self): self._vocab_size = 10 self._max_time = 16 self._batch_size = 8 self._emb_dim = 20 self._attention_dim = 256 self._inputs = torch.randint(self._vocab_size, size=(self._batch_size, self._max_time)) embedding = torch.rand(self._vocab_size, self._emb_dim, dtype=torch.float) self._embedder = WordEmbedder(init_value=embedding) self._encoder_output = torch.rand(self._batch_size, self._max_time, 64) self._test_hparams = {} # (cell_type, is_multi) -> hparams for cell_type in ["RNNCell", "LSTMCell", "GRUCell"]: hparams = { "rnn_cell": { 'type': cell_type, 'kwargs': { 'num_units': 256, }, }, "attention": { "kwargs": { "num_units": self._attention_dim }, } } self._test_hparams[(cell_type, False)] = HParams( hparams, AttentionRNNDecoder.default_hparams()) hparams = { "rnn_cell": { 'type': 'LSTMCell', 'kwargs': { 'num_units': 256, }, 'num_layers': 3, }, "attention": { "kwargs": { "num_units": self._attention_dim }, } } self._test_hparams[("LSTMCell", True)] = HParams( hparams, AttentionRNNDecoder.default_hparams())
def __init__(self, hparams): self._hparams = HParams(hparams, self.default_hparams()) feature_types = self._hparams.dataset.feature_original_types self._features = _convert_feature_hparams(feature_types) convert_types = self._hparams.dataset.feature_convert_types self._convert_types = { key: get_numpy_dtype(value) for key, value in convert_types.items() } for key, dtype in self._convert_types.items(): self._features[key] = self._features[key]._replace(dtype=dtype) image_options = self._hparams.dataset.image_options if isinstance(image_options, HParams): image_options = [image_options] self._image_transforms = {} for options in image_options: key = options.get('image_feature_name') if key is None or key not in self._features: continue self._image_transforms[key] = _create_image_transform( options.get('resize_height'), options.get('resize_width'), options.get('resize_method') or 'bilinear') self._other_transforms = self._hparams.dataset.other_transformations data_source = PickleDataSource(self._hparams.dataset.files) super().__init__(data_source, hparams)
def __init__(self, vocab: Dict[str, int], hparams=None): self._hparams = HParams(hparams, self.default_hparams()) # Initialize embeddings init_fn_kwargs = self._hparams.init_fn.kwargs.todict() if "shape" in init_fn_kwargs or "size" in init_fn_kwargs: raise ValueError("Argument 'shape' or 'size' must not be " "specified. They are inferred automatically.") init_fn: Callable[..., np.ndarray] init_fn = utils.get_function(self._hparams.init_fn.type, ["numpy.random", "numpy", "texar.custom"]) try: self._word_vecs = init_fn( # type: ignore size=[len(vocab), self._hparams.dim], **init_fn_kwargs) except TypeError: self._word_vecs = init_fn( # type: ignore shape=[len(vocab), self._hparams.dim], **init_fn_kwargs) # Optionally read embeddings from file if self._hparams.file is not None and self._hparams.file != "": read_fn: Callable[[str, Dict[str, int], np.ndarray], np.ndarray] read_fn = utils.get_function( # type: ignore self._hparams.read_fn, ["texar.data.embedding", "texar.data", "texar.custom"]) self._word_vecs = read_fn(self._hparams.file, vocab, self._word_vecs)
def test_switch_dropout(self): """Tests dropout mode. """ emb_dim = 4 num_units = 64 hparams = { "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, }, } mode = tf.placeholder(tf.string) hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_, mode) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) output_train, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN}) self.assertEqual(output_train.shape[0], batch_size) output_test, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL}) self.assertEqual(output_test.shape[0], batch_size)
def get_grad_clip_fn(hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \ Optional[Callable[[torch.Tensor], Optional[torch.Tensor]]]: r"""Create a gradient clipping function. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. See :func:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. Returns: A gradient clipping function. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()) hparams_grad_clip = hparams["gradient_clip"] grad_clip_type = hparams_grad_clip["type"] if grad_clip_type == "" or grad_clip_type is None: grad_clip_fn = None else: grad_clip_modules = ['torch.nn.utils', 'texar.custom'] grad_clip_fn = utils.get_function(grad_clip_type, grad_clip_modules) grad_clip_fn_kwargs = hparams_grad_clip["kwargs"].todict() grad_clip_fn = functools.partial(grad_clip_fn, **grad_clip_fn_kwargs) return grad_clip_fn
def test_get_rnn_cell(self): """Tests :func:`texar.core.layers.get_rnn_cell`. """ emb_dim = 4 num_units = 64 # Given instance hparams = {"type": rnn.LSTMCell(num_units)} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given class hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given string, and complex hyperparameters keep_prob_x = tf.placeholder(name='keep_prob', shape=[], dtype=tf.float32) hparams = { "type": "tensorflow.contrib.rnn.GRUCell", "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, "state_keep_prob": keep_prob_x, "variational_recurrent": True, "input_size": [emb_dim, num_units] }, "residual": True, "highway": True } hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) feed_dict = { keep_prob_x: 1.0, context.global_mode(): tf.estimator.ModeKeys.TRAIN } output_, state_ = sess.run([output, state], feed_dict=feed_dict) self.assertEqual(output_.shape[0], batch_size) if isinstance(state_, (list, tuple)): self.assertEqual(state_[0].shape[0], batch_size) self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units) else: self.assertEqual(state_.shape[0], batch_size) self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def __init__(self, hparams=None): self._hparams = HParams(hparams, self.default_hparams()) self._template = tf.make_template(self._hparams.name, self._build, create_scope_now_=True) self._unique_name = self.variable_scope.name.split("/")[-1] self._trainable_variables = [] self._built = False
def get_gradient_clip_fn(hparams=None): """Creates a gradient clipping function based on the hyperparameters. See the :attr:`gradient_clip` field in :meth:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. The gradient clipping function takes a list of `(gradients, variables)` tuples and returns a list of `(clipped_gradients, variables)` tuples. Typical examples include :tf_main:`tf.clip_by_global_norm <clip_by_global_norm>`, :tf_main:`tf.clip_by_value <clip_by_value>`, :tf_main:`tf.clip_by_norm <clip_by_norm>`, :tf_main:`tf.clip_by_average_norm <clip_by_average_norm>`, etc. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. Returns: function or `None`: If hparams["type"] is specified, returns the respective function. If hparams["type"] is empty, returns `None`. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()["gradient_clip"]) fn_type = hparams["type"] if fn_type is None or fn_type == "": return None fn_modules = ["tensorflow", "texar.custom"] clip_fn = utils.get_function(fn_type, fn_modules) clip_fn_args = utils.get_args(clip_fn) fn_kwargs = hparams["kwargs"] if isinstance(fn_kwargs, HParams): fn_kwargs = fn_kwargs.todict() def grad_clip_fn(grads_and_vars): """Gradient clipping function. Args: grads_and_vars (list): A list of `(gradients, variables)` tuples. Returns: list: A list of `(clipped_gradients, variables)` tuples. """ grads, vars_ = zip(*grads_and_vars) if clip_fn == tf.clip_by_global_norm: clipped_grads, _ = clip_fn(t_list=grads, **fn_kwargs) elif 't_list' in clip_fn_args: clipped_grads = clip_fn(t_list=grads, **fn_kwargs) elif 't' in clip_fn_args: # e.g., tf.clip_by_value clipped_grads = [clip_fn(t=grad, **fn_kwargs) for grad in grads] return list(zip(clipped_grads, vars_)) return grad_clip_fn
def get_train_op(params: Optional[Iterable[Union[torch.Tensor, Dict[str, Any]]]] = None, optimizer: Optional[Optimizer] = None, scheduler: Optional[_LRScheduler] = None, hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \ Callable[[], None]: r"""Creates a training op. Args: params: an iterable of :class:`torch.Tensor` or :class:`dict`. Specifies what Tensors should be optimized. optimizer: A :torch_docs:`torch.optim.Optimizer <optim.html#torch.optim.Optimizer>` instance. scheduler: A :torch_docs:`torch.optim.lr_scheduler._LRScheduler <optim.html#how-to-adjust-learning-rate>` instance. hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. See :func:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. Returns: The callable used for variable optimization. """ hparams = HParams(hparams, default_optimization_hparams()) if params is None and optimizer is None and scheduler is None: raise ValueError("'params', 'optimizer' and 'scheduler' must not be " "None simultaneously.") if scheduler is None: if optimizer is None and params is not None: optimizer = get_optimizer(params, hparams) if optimizer is not None: scheduler = get_scheduler(optimizer, hparams) else: optimizer = scheduler.optimizer # type: ignore grad_clip_fn = get_grad_clip_fn(hparams) # TODO: Support per-parameter options in the future. params_list: List[nn.Parameter] = [] for param_group in optimizer.param_groups: # type: ignore params = param_group["params"] if isinstance(params, torch.Tensor): params_list.append(params) elif isinstance(params, list): params_list += params def _train_op(): if grad_clip_fn is not None: grad_clip_fn(parameters=params_list) optimizer.step() # TODO: Ideally, scheduler should be used in the epoch level. if scheduler is not None: scheduler.step() optimizer.zero_grad() return _train_op
def __init__(self, actions, state_shape, hparams=None): self._hparams = HParams(hparams, self.default_hparams()) self._variable_scope = None self.actions = actions self.state_shape = state_shape self.current_state = None self.timestep = 0
def __init__(self, pretrained_model_name: Optional[str] = None, cache_dir: Optional[str] = None, hparams=None): super().__init__(pretrained_model_name=pretrained_model_name, cache_dir=cache_dir, hparams=hparams) if self.pretrained_model_dir: self._hparams = HParams(self.pretrained_model_hparams, self._hparams.todict()) # Word embedding self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size, hparams=self._hparams.embed) # Segment embedding for each type of tokens self.segment_embedder = WordEmbedder( vocab_size=self._hparams.type_vocab_size, hparams=self._hparams.segment_embed) # Position embedding self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embed) # The BERT encoder (a TransformerEncoder) self.encoder = TransformerEncoder(hparams=self._hparams.encoder) self.pooler = nn.Sequential( nn.Linear(self._hparams.hidden_size, self._hparams.hidden_size), nn.Tanh()) if self.pretrained_model_dir: bert_utils.init_bert_checkpoint(self, self.pretrained_model_dir) elif self._hparams.initializer: initialize = layers.get_initializer(self._hparams.initializer) assert initialize is not None # Do not re-initialize LayerNorm modules. for name, param in self.named_parameters(): if name.split( '.')[-1] == 'weight' and 'layer_norm' not in name: initialize(param)
def get_optimizer_fn(hparams=None): """Returns a function `optimizer_fn` of making optimizer instance, along with the optimizer class. .. role:: python(code) :language: python The function has the signiture :python:`optimizer_fn(learning_rate=None) -> optimizer class instance` See the :attr:`"optimizer"` field of :meth:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. The optimizer class must be a subclass of :tf_main:`tf.train.Optimizer <train/Optimizer>`. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. Returns: - If hparams["type"] is a string or optimizer class, returns\ `(optimizer_fn, optimizer class)`, - If hparams["type"] is an optimizer instance, returns \ `(the optimizer instance, optimizer class)` """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()["optimizer"]) opt = hparams["type"] if isinstance(opt, tf.train.Optimizer): return opt, type(opt) opt_modules = [ 'tensorflow.train', 'tensorflow.contrib.opt', 'texar.core.optimization', 'texar.custom' ] try: opt_class = utils.check_or_get_class(opt, opt_modules, tf.train.Optimizer) except TypeError: raise ValueError( "Unrecognized optimizer. Must be string name of the " "optimizer class, or the class which is a subclass of " "tf.train.Optimizer, or an instance of the subclass of " "Optimizer.") def _get_opt(learning_rate=None): opt_kwargs = hparams["kwargs"].todict() fn_args = set(utils.get_args(opt_class.__init__)) if 'learning_rate' in fn_args and learning_rate is not None: opt_kwargs["learning_rate"] = learning_rate return opt_class(**opt_kwargs) return _get_opt, opt_class
def setUp(self): self._vocab_size = 4 self._max_time = 8 self._batch_size = 16 self._emb_dim = 20 self._inputs = torch.rand( self._batch_size, self._max_time, self._emb_dim, dtype=torch.float) self._embedding = torch.rand( self._vocab_size, self._emb_dim, dtype=torch.float) self._hparams = HParams(None, BasicRNNDecoder.default_hparams())
def _construct(cls, hparams, device: Optional[torch.device] = None, vocab: Optional[Vocab] = None, embedding: Optional[Vocab] = None): mono_text_data = cls.__new__(cls) mono_text_data._hparams = HParams(hparams, mono_text_data.default_hparams()) if mono_text_data._hparams.dataset.variable_utterance: raise NotImplementedError dataset = mono_text_data._hparams.dataset mono_text_data._other_transforms = dataset.other_transformations # Create vocabulary if vocab is not None: mono_text_data._vocab = vocab mono_text_data._bos_token = vocab.bos_token mono_text_data._eos_token = vocab.eos_token else: mono_text_data._bos_token = dataset.bos_token mono_text_data._eos_token = dataset.eos_token bos = utils.default_str(mono_text_data._bos_token, SpecialTokens.BOS) eos = utils.default_str(mono_text_data._eos_token, SpecialTokens.EOS) mono_text_data._vocab = Vocab(dataset.vocab_file, bos_token=bos, eos_token=eos) # Create embedding if embedding is not None: mono_text_data._embedding = embedding else: mono_text_data._embedding = mono_text_data.make_embedding( dataset.embedding_init, mono_text_data._vocab.token_to_id_map_py) mono_text_data._delimiter = dataset.delimiter mono_text_data._max_seq_length = dataset.max_seq_length mono_text_data._length_filter_mode = _LengthFilterMode( mono_text_data._hparams.dataset.length_filter_mode) mono_text_data._pad_length = mono_text_data._max_seq_length if mono_text_data._pad_length is not None: mono_text_data._pad_length += sum( int(x != '') for x in [mono_text_data._bos_token, mono_text_data._eos_token]) data_source: SequenceDataSource[str] = SequenceDataSource([]) super(MonoTextData, mono_text_data).__init__(source=data_source, hparams=hparams, device=device) return mono_text_data
def __init__(self, pretrained_model_name=None, cache_dir=None, hparams=None): EncoderBase.__init__(self, hparams) BertBase.__init__(self, pretrained_model_name, cache_dir, hparams) # put these things to BertBase if self.pretrained_model: self._hparams = HParams(self.pretrained_model_hparams, self._hparams.todict()) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) # Word embedding self.word_embedder = WordEmbedder( vocab_size=self._hparams.vocab_size, hparams=self._hparams.embed) # Segment embedding for each type of tokens self.segment_embedder = WordEmbedder( vocab_size=self._hparams.type_vocab_size, hparams=self._hparams.segment_embed) # Position embedding self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embed) # The BERT encoder (a TransformerEncoder) self.encoder = TransformerEncoder(hparams=self._hparams.encoder) with tf.variable_scope("pooler"): kwargs_i = { "units": self._hparams.hidden_size, "activation": tf.tanh } layer_hparams = {"type": "Dense", "kwargs": kwargs_i} self.pooler = layers.get_layer(hparams=layer_hparams)
def load_pretrained_config(self, pretrained_model_name: Optional[str] = None, cache_dir: Optional[str] = None, hparams=None): r"""Load paths and configurations of the pre-trained model. Args: pretrained_model_name (optional): A str with the name of a pre-trained model to load. If `None`, will use the model name in :attr:`hparams`. cache_dir (optional): The path to a folder in which the pre-trained models will be cached. If `None` (default), a default directory will be used. hparams (dict or HParams, optional): Hyperparameters. Missing hyperparameter will be set to default values. See :meth:`default_hparams` for the hyperparameter structure and default values. """ if not hasattr(self, "_hparams"): self._hparams = HParams(hparams, self.default_hparams()) else: # Probably already parsed by subclasses. We rely on subclass # implementations to get this right. # As a sanity check, we require `hparams` to be `None` in this case. if hparams is not None: raise ValueError( "`self._hparams` is already assigned, but `hparams` " "argument is not None.") self.pretrained_model_dir = None if pretrained_model_name is None: pretrained_model_name = self._hparams.pretrained_model_name if pretrained_model_name is not None: self.pretrained_model_dir = self.download_checkpoint( pretrained_model_name, cache_dir) pretrained_model_hparams = self._transform_config( self.pretrained_model_dir) self._hparams = HParams(pretrained_model_hparams, self._hparams.todict())
def get_optimizer_fn(hparams=None): """Returns a function of making optimizer instance, along with the optimizer class. The function has the signiture: (learning_rate=None) -> instance of the optimizer class, The optimizer class must be a subclass of :tf_main:`~tf.train.Optimizer`. See the :attr:`"optimizer"` field in :meth:`~texar.core.optimization.default_optimization_hparams` for all hyperparameters and default values. If :attr:`hparams["type"]` is an optimier instance, returns the instance directly. Args: hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. Returns: (function that creates optimizer instance, optimizer class), or the optimizer instance. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()["optimizer"]) opt = hparams["type"] if isinstance(opt, tf.train.Optimizer): return opt, type(opt) else: opt_modules = [ 'tensorflow.train', 'tensorflow.contrib.opt', 'texar.custom' ] try: opt_class = utils.check_or_get_class(opt, opt_modules, tf.train.Optimizer) except TypeError: raise ValueError( "Unrecognized optimizer. Must be string name of the " "optimizer class, or the class which is a subclass of " "tf.train.Optimizer, or an instance of the subclass of " "Optimizer.") def _get_opt(learning_rate=None): opt_kwargs = hparams["kwargs"].todict() fn_args = set(inspect.getargspec(opt_class.__init__).args) if 'learning_rate' in fn_args and learning_rate is not None: opt_kwargs["learning_rate"] = learning_rate return opt_class(**opt_kwargs) return _get_opt, opt_class
def __init__(self, hparams: Optional[Union[HParams, Dict[str, Any]]] = None): super().__init__() if not hasattr(self, '_hparams'): self._hparams = HParams(hparams, self.default_hparams()) else: # Probably already parsed by subclasses. We rely on subclass # implementations to get this right. # As a sanity check, we require `hparams` to be `None` in this case. if hparams is not None: raise ValueError( "`self._hparams` is already assigned, but `hparams` " "argument is not None.")
def test_typecheck(self): """Tests type-check functionality. """ def _foo(): pass def _bar(): pass default_hparams = {"fn": _foo, "fn_2": _foo} hparams = {"fn": _foo, "fn_2": _bar} hparams_ = HParams(hparams, default_hparams) self.assertEqual(hparams_.fn, default_hparams["fn"])
def __init__(self, data_hparams, hparams=None): ModelBase.__init__(self, hparams) self._data_hparams = HParams(data_hparams, PairedTextData.default_hparams()) self._src_vocab = None self._tgt_vocab = None self._src_embedder = None self._tgt_embedder = None self._connector = None self._encoder = None self._decoder = None
def __init__(self, pretrained_model_name: Optional[str] = None, cache_dir: Optional[str] = None, hparams=None): super().__init__(pretrained_model_name=pretrained_model_name, cache_dir=cache_dir, hparams=hparams) if self.pretrained_model_dir: self._hparams = HParams(self.pretrained_model_hparams, self._hparams.todict()) # Word embedding self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size, hparams=self._hparams.embed) # Position embedding self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embed) # The GPT2 decoder (a TransformerDecoder) self.decoder = TransformerDecoder( vocab_size=self._hparams.vocab_size, output_layer=self.word_embedder.embedding, hparams=self._hparams.decoder) if self.pretrained_model_dir: gpt2_utils.init_gpt2_checkpoint(self, self.pretrained_model_dir) elif self._hparams.initializer: initialize = layers.get_initializer(self._hparams.initializer) assert initialize is not None # Do not re-initialize LayerNorm modules. for name, param in self.named_parameters(): if name.split( '.')[-1] == 'weight' and 'layer_norm' not in name: initialize(param)
def get_embedding(hparams=None, init_value=None, num_embeds=None, variable_scope='Embedding'): """Creates embedding variable if not exists. Args: hparams (dict or HParams, optional): Embedding hyperparameters. Missing hyperparameters are set to default values. See :func:`~texar.modules.default_embedding_hparams` for all hyperparameters and default values. If :attr:`init_value` is given, :attr:`hparams["initializer"]`, and :attr:`hparams["dim"]` are ignored. init_value (Tensor or numpy array, optional): Initial values of the embedding variable. If not given, embedding is initialized as specified in :attr:`hparams["initializer"]`. num_embeds (int, optional): The number of embedding items (e.g., vocabulary size). Required if :attr:`init_value` is not provided. variable_scope (str or VariableScope, optional): Variable scope of the embedding variable. Returns: Variable or Tensor: A 2D `Variable` or `Tensor` of the same shape with :attr:`init_value` or of the shape :attr:`[num_embeds, hparams["dim"]]`. """ with tf.variable_scope(variable_scope): if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_embedding_hparams()) regularizer = layers.get_regularizer(hparams["regularizer"]) if init_value is None: initializer = layers.get_initializer(hparams["initializer"]) dim = hparams["dim"] if not isinstance(hparams["dim"], (list, tuple)): dim = [dim] embedding = tf.get_variable(name='w', shape=[num_embeds] + dim, initializer=initializer, regularizer=regularizer, trainable=hparams["trainable"]) else: init_value = tf.cast(init_value, tf.float32) embedding = tf.get_variable(name='w', initializer=init_value, regularizer=regularizer, trainable=hparams["trainable"]) return embedding
def __init__(self, hparams): TextDataBase.__init__(self, hparams) # Defaultizes hparams of each dataset datasets_hparams = self._hparams.datasets defaultized_datasets_hparams = [] for ds_hpms in datasets_hparams: data_type = ds_hpms.get("data_type", None) defaultized_ds_hpms = HParams(ds_hpms, _default_dataset_hparams(data_type)) defaultized_datasets_hparams.append(defaultized_ds_hpms) self._hparams.datasets = defaultized_datasets_hparams with tf.name_scope(self.name, self.default_hparams()["name"]): self._make_data()
def _get_static_lr(learning_rate=None, optimizer_class=None, hparams=None): """Return the base static learning_rate. A helper function for creating the optimization function. """ hparams = HParams(hparams, default_optimization_hparams()) opt_hparams = hparams['optimizer'] if learning_rate is None: learning_rate = opt_hparams["kwargs"].get("learning_rate", None) if learning_rate is None: # Try to get learning_rate from the default value of the # optimizer's argument opt_argspec = utils.get_default_arg_values(optimizer_class.__init__) learning_rate = opt_argspec.get("learning_rate", None) return learning_rate
def get_embedding(num_embeds: Optional[int] = None, init_value: Optional[torch.Tensor] = None, hparams=None): r"""Creates embedding variable if not exists. Args: hparams (dict or HParams, optional): Embedding hyperparameters. Missing hyperparameters are set to default values. See :func:`~texar.modules.default_embedding_hparams` for all hyperparameters and default values. If :attr:`init_value` is given, :attr:`hparams["initializer"]`, and :attr:`hparams["dim"]` are ignored. init_value (Tensor or numpy array, optional): Initial values of the embedding variable. If not given, embedding is initialized as specified in :attr:`hparams["initializer"]`. num_embeds (int, optional): The number of embedding items (e.g., vocabulary size). Required if :attr:`init_value` is not provided. Returns: A 2D :tensor:`Tensor` of the same shape with :attr:`init_value` or of the shape ``[num_embeds, hparams["dim"]]``. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_embedding_hparams()) if init_value is None: initializer = layers.get_initializer( getattr(hparams, "initializer", None)) # TODO Shibiao: add regularizer dim = hparams["dim"] if not isinstance(hparams["dim"], (list, tuple)): dim = [dim] embedding = torch.empty(size=[num_embeds] + dim) # initializer should be set by layers.get_initializer if initializer: embedding = initializer(embedding) else: embedding = torch.nn.init.xavier_uniform_(embedding) else: if torch.is_tensor(init_value): embedding = init_value # Do not copy the tensor. else: embedding = torch.tensor(init_value, dtype=torch.float) return embedding
def get_optimizer( params: Union[List[torch.Tensor], List[Dict[str, List[torch.Tensor]]]], hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \ Optimizer: r"""Creates a optimizer instance. Args: params: an iterable of :class:`torch.Tensor` or :class:`dict`. Specifies what Tensors should be optimized. hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. See :func:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. :return: The :torch_docs:`torch.optim.Optimizer <optim.html#torch.optim.Optimizer>` instance specified in :attr:`hparams`. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()) hparams_opt = hparams["optimizer"] optimizer_type = hparams_opt["type"] if isinstance(optimizer_type, Optimizer): optimizer_class = optimizer_type else: optimizer_modules = ['torch.optim', 'texar.custom'] try: optimizer_class = utils.check_or_get_class( # type: ignore optimizer_type, optimizer_modules, Optimizer) except TypeError: raise ValueError( "Unrecognized optimizer. Must be string name of the " "optimizer class, or the class which is a subclass of " "torch.optim.Optimizer, or an instance of the subclass of " "Optimizer.") optimizer_kwargs = hparams_opt["kwargs"].todict() optimizer_kwargs.update({"params": params}) optimizer = optimizer_class(**optimizer_kwargs) # type: ignore return optimizer
def get_scheduler(optimizer: Optimizer, hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \ Optional[_LRScheduler]: r"""Creates a scheduler instance. Args: optimizer: A :torch_docs:`torch.optim.Optimizer <optim.html#torch.optim.Optimizer>` instance. hparams (dict or HParams, optional): hyperparameters. Missing hyperparameters are set to default values automatically. See :func:`~texar.core.default_optimization_hparams` for all hyperparameters and default values. :return: A :torch_docs:`torch.optim.lr_scheduler._LRScheduler <optim.html#how-to-adjust-learning-rate>` instance. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_optimization_hparams()) hparams_scheduler = hparams["learning_rate_decay"] scheduler_type = hparams_scheduler["type"] if scheduler_type == "" or scheduler_type is None: scheduler = None else: if isinstance(scheduler_type, _LRScheduler): scheduler_class = scheduler_type else: scheduler_modules = ['torch.optim.lr_scheduler', 'texar.custom'] try: scheduler_class = utils.check_or_get_class( # type: ignore scheduler_type, scheduler_modules, _LRScheduler) except TypeError: raise ValueError( "Unrecognized lr_scheduler. Must be string name of the " "lr_scheduler class, or the class which is a subclass of " "torch.optim._LRScheduler.") scheduler_kwargs = hparams_scheduler["kwargs"].todict() scheduler_kwargs.update({"optimizer": optimizer}) scheduler = scheduler_class(**scheduler_kwargs) # type: ignore return scheduler
def __init__(self, hparams, device: Optional[torch.device] = None): self._hparams = HParams(hparams, self.default_hparams()) self._other_transforms = self._hparams.dataset.other_transformations data_type = self._hparams.dataset["data_type"] self._typecast_func: Union[Type[int], Type[float]] if data_type == "int": self._typecast_func = int self._to_data_type = np.int32 elif data_type == "float": self._typecast_func = float self._to_data_type = np.float32 else: raise ValueError( "Incorrect 'data_type'. Currently 'int' and " "'float' are supported. Received {}".format(data_type)) data_source = TextLineDataSource( self._hparams.dataset.files, compression_type=self._hparams.dataset.compression_type) super().__init__(data_source, hparams, device=device)
def get_regularizer(hparams=None): r"""Returns a variable regularizer instance. See :func:`~texar.core.default_regularizer_hparams` for all hyperparameters and default values. The "type" field can be a subclass of :class:`~texar.core.regularizers.Regularizer`, its string name or module path, or a class instance. Args: hparams (dict or HParams, optional): Hyperparameters. Missing hyperparameters are set to default values. Returns: A :class:`~texar.core.regularizers.Regularizer` instance. `None` if :attr:`hparams` is `None` or taking the default hyperparameter value. Raises: ValueError: The resulting regularizer is not an instance of :class:`~texar.core.regularizers.Regularizer`. """ if hparams is None: return None if isinstance(hparams, dict): hparams = HParams(hparams, default_regularizer_hparams()) rgl = utils.check_or_get_instance( hparams.type, hparams.kwargs.todict(), ["texar.core.regularizers", "texar.custom"]) if not isinstance(rgl, Regularizer): raise ValueError("The regularizer must be an instance of " "texar.core.regularizers.Regularizer.") if isinstance(rgl, L1L2) and rgl.l1 == 0. and rgl.l2 == 0.: return None return rgl