def test_type_kwargs(self):
"""The the special cases involving "type" and "kwargs"
hyperparameters.
"""
default_hparams = {"type": "type_name", "kwargs": {"arg1": "argv1"}}
hparams = {"type": "type_name"}
hparams_ = HParams(hparams, default_hparams)
self.assertEqual(hparams_.kwargs.todict(), default_hparams["kwargs"])
hparams = {"type": "type_name", "kwargs": {"arg2": "argv2"}}
hparams_ = HParams(hparams, default_hparams)
full_kwargs = {}
full_kwargs.update(default_hparams["kwargs"])
full_kwargs.update(hparams["kwargs"])
self.assertEqual(hparams_.kwargs.todict(), full_kwargs)
hparams = {"kwargs": {"arg2": "argv2"}}
hparams_ = HParams(hparams, default_hparams)
self.assertEqual(hparams_.kwargs.todict(), full_kwargs)
hparams = {"type": "type_name2"}
hparams_ = HParams(hparams, default_hparams)
self.assertEqual(hparams_.kwargs.todict(), {})
hparams = {"type": "type_name2", "kwargs": {"arg3": "argv3"}}
hparams_ = HParams(hparams, default_hparams)
self.assertEqual(hparams_.kwargs.todict(), hparams["kwargs"])
def setUp(self):
self._vocab_size = 10
self._max_time = 16
self._batch_size = 8
self._emb_dim = 20
self._attention_dim = 256
self._inputs = torch.randint(self._vocab_size,
size=(self._batch_size, self._max_time))
embedding = torch.rand(self._vocab_size,
self._emb_dim,
dtype=torch.float)
self._embedder = WordEmbedder(init_value=embedding)
self._encoder_output = torch.rand(self._batch_size, self._max_time, 64)
self._test_hparams = {} # (cell_type, is_multi) -> hparams
for cell_type in ["RNNCell", "LSTMCell", "GRUCell"]:
hparams = {
"rnn_cell": {
'type': cell_type,
'kwargs': {
'num_units': 256,
},
},
"attention": {
"kwargs": {
"num_units": self._attention_dim
},
}
}
self._test_hparams[(cell_type, False)] = HParams(
hparams, AttentionRNNDecoder.default_hparams())
hparams = {
"rnn_cell": {
'type': 'LSTMCell',
'kwargs': {
'num_units': 256,
},
'num_layers': 3,
},
"attention": {
"kwargs": {
"num_units": self._attention_dim
},
}
}
self._test_hparams[("LSTMCell", True)] = HParams(
hparams, AttentionRNNDecoder.default_hparams())
def __init__(self, hparams):
self._hparams = HParams(hparams, self.default_hparams())
feature_types = self._hparams.dataset.feature_original_types
self._features = _convert_feature_hparams(feature_types)
convert_types = self._hparams.dataset.feature_convert_types
self._convert_types = {
key: get_numpy_dtype(value)
for key, value in convert_types.items()
}
for key, dtype in self._convert_types.items():
self._features[key] = self._features[key]._replace(dtype=dtype)
image_options = self._hparams.dataset.image_options
if isinstance(image_options, HParams):
image_options = [image_options]
self._image_transforms = {}
for options in image_options:
key = options.get('image_feature_name')
if key is None or key not in self._features:
continue
self._image_transforms[key] = _create_image_transform(
options.get('resize_height'), options.get('resize_width'),
options.get('resize_method') or 'bilinear')
self._other_transforms = self._hparams.dataset.other_transformations
data_source = PickleDataSource(self._hparams.dataset.files)
super().__init__(data_source, hparams)
def __init__(self, vocab: Dict[str, int], hparams=None):
self._hparams = HParams(hparams, self.default_hparams())
# Initialize embeddings
init_fn_kwargs = self._hparams.init_fn.kwargs.todict()
if "shape" in init_fn_kwargs or "size" in init_fn_kwargs:
raise ValueError("Argument 'shape' or 'size' must not be "
"specified. They are inferred automatically.")
init_fn: Callable[..., np.ndarray]
init_fn = utils.get_function(self._hparams.init_fn.type,
["numpy.random", "numpy", "texar.custom"])
try:
self._word_vecs = init_fn( # type: ignore
size=[len(vocab), self._hparams.dim],
**init_fn_kwargs)
except TypeError:
self._word_vecs = init_fn( # type: ignore
shape=[len(vocab), self._hparams.dim],
**init_fn_kwargs)
# Optionally read embeddings from file
if self._hparams.file is not None and self._hparams.file != "":
read_fn: Callable[[str, Dict[str, int], np.ndarray], np.ndarray]
read_fn = utils.get_function( # type: ignore
self._hparams.read_fn,
["texar.data.embedding", "texar.data", "texar.custom"])
self._word_vecs = read_fn(self._hparams.file, vocab,
self._word_vecs)
def test_switch_dropout(self):
"""Tests dropout mode.
"""
emb_dim = 4
num_units = 64
hparams = {
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
},
}
mode = tf.placeholder(tf.string)
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_, mode)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output_train, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN})
self.assertEqual(output_train.shape[0], batch_size)
output_test, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL})
self.assertEqual(output_test.shape[0], batch_size)
def get_grad_clip_fn(hparams: Optional[Union[HParams,
Dict[str, Any]]] = None) -> \
Optional[Callable[[torch.Tensor], Optional[torch.Tensor]]]:
r"""Create a gradient clipping function.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
Returns:
A gradient clipping function.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_grad_clip = hparams["gradient_clip"]
grad_clip_type = hparams_grad_clip["type"]
if grad_clip_type == "" or grad_clip_type is None:
grad_clip_fn = None
else:
grad_clip_modules = ['torch.nn.utils', 'texar.custom']
grad_clip_fn = utils.get_function(grad_clip_type, grad_clip_modules)
grad_clip_fn_kwargs = hparams_grad_clip["kwargs"].todict()
grad_clip_fn = functools.partial(grad_clip_fn, **grad_clip_fn_kwargs)
return grad_clip_fn
def test_get_rnn_cell(self):
"""Tests :func:`texar.core.layers.get_rnn_cell`.
"""
emb_dim = 4
num_units = 64
# Given instance
hparams = {"type": rnn.LSTMCell(num_units)}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given class
hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given string, and complex hyperparameters
keep_prob_x = tf.placeholder(name='keep_prob',
shape=[],
dtype=tf.float32)
hparams = {
"type": "tensorflow.contrib.rnn.GRUCell",
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
"state_keep_prob": keep_prob_x,
"variational_recurrent": True,
"input_size": [emb_dim, num_units]
},
"residual": True,
"highway": True
}
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {
keep_prob_x: 1.0,
context.global_mode(): tf.estimator.ModeKeys.TRAIN
}
output_, state_ = sess.run([output, state], feed_dict=feed_dict)
self.assertEqual(output_.shape[0], batch_size)
if isinstance(state_, (list, tuple)):
self.assertEqual(state_[0].shape[0], batch_size)
self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units)
else:
self.assertEqual(state_.shape[0], batch_size)
self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def __init__(self, hparams=None):
self._hparams = HParams(hparams, self.default_hparams())
self._template = tf.make_template(self._hparams.name,
self._build,
create_scope_now_=True)
self._unique_name = self.variable_scope.name.split("/")[-1]
self._trainable_variables = []
self._built = False
def get_gradient_clip_fn(hparams=None):
"""Creates a gradient clipping function based on the hyperparameters.
See the :attr:`gradient_clip` field in
:meth:`~texar.core.default_optimization_hparams` for all
hyperparameters and default values.
The gradient clipping function takes a list of `(gradients, variables)`
tuples and returns a list of `(clipped_gradients, variables)` tuples.
Typical examples include
:tf_main:`tf.clip_by_global_norm <clip_by_global_norm>`,
:tf_main:`tf.clip_by_value <clip_by_value>`,
:tf_main:`tf.clip_by_norm <clip_by_norm>`,
:tf_main:`tf.clip_by_average_norm <clip_by_average_norm>`, etc.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically.
Returns:
function or `None`: If hparams["type"] is specified, returns
the respective function. If hparams["type"] is empty,
returns `None`.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams,
default_optimization_hparams()["gradient_clip"])
fn_type = hparams["type"]
if fn_type is None or fn_type == "":
return None
fn_modules = ["tensorflow", "texar.custom"]
clip_fn = utils.get_function(fn_type, fn_modules)
clip_fn_args = utils.get_args(clip_fn)
fn_kwargs = hparams["kwargs"]
if isinstance(fn_kwargs, HParams):
fn_kwargs = fn_kwargs.todict()
def grad_clip_fn(grads_and_vars):
"""Gradient clipping function.
Args:
grads_and_vars (list): A list of `(gradients, variables)` tuples.
Returns:
list: A list of `(clipped_gradients, variables)` tuples.
"""
grads, vars_ = zip(*grads_and_vars)
if clip_fn == tf.clip_by_global_norm:
clipped_grads, _ = clip_fn(t_list=grads, **fn_kwargs)
elif 't_list' in clip_fn_args:
clipped_grads = clip_fn(t_list=grads, **fn_kwargs)
elif 't' in clip_fn_args: # e.g., tf.clip_by_value
clipped_grads = [clip_fn(t=grad, **fn_kwargs) for grad in grads]
return list(zip(clipped_grads, vars_))
return grad_clip_fn
def get_train_op(params: Optional[Iterable[Union[torch.Tensor,
Dict[str, Any]]]] = None,
optimizer: Optional[Optimizer] = None,
scheduler: Optional[_LRScheduler] = None,
hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \
Callable[[], None]:
r"""Creates a training op.
Args:
params: an iterable of :class:`torch.Tensor` or
:class:`dict`. Specifies what Tensors should be optimized.
optimizer: A :torch_docs:`torch.optim.Optimizer
<optim.html#torch.optim.Optimizer>` instance.
scheduler: A :torch_docs:`torch.optim.lr_scheduler._LRScheduler
<optim.html#how-to-adjust-learning-rate>` instance.
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
Returns:
The callable used for variable optimization.
"""
hparams = HParams(hparams, default_optimization_hparams())
if params is None and optimizer is None and scheduler is None:
raise ValueError("'params', 'optimizer' and 'scheduler' must not be "
"None simultaneously.")
if scheduler is None:
if optimizer is None and params is not None:
optimizer = get_optimizer(params, hparams)
if optimizer is not None:
scheduler = get_scheduler(optimizer, hparams)
else:
optimizer = scheduler.optimizer # type: ignore
grad_clip_fn = get_grad_clip_fn(hparams)
# TODO: Support per-parameter options in the future.
params_list: List[nn.Parameter] = []
for param_group in optimizer.param_groups: # type: ignore
params = param_group["params"]
if isinstance(params, torch.Tensor):
params_list.append(params)
elif isinstance(params, list):
params_list += params
def _train_op():
if grad_clip_fn is not None:
grad_clip_fn(parameters=params_list)
optimizer.step()
# TODO: Ideally, scheduler should be used in the epoch level.
if scheduler is not None:
scheduler.step()
optimizer.zero_grad()
return _train_op
def __init__(self, actions, state_shape, hparams=None):
self._hparams = HParams(hparams, self.default_hparams())
self._variable_scope = None
self.actions = actions
self.state_shape = state_shape
self.current_state = None
self.timestep = 0
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(pretrained_model_name=pretrained_model_name,
cache_dir=cache_dir,
hparams=hparams)
if self.pretrained_model_dir:
self._hparams = HParams(self.pretrained_model_hparams,
self._hparams.todict())
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Segment embedding for each type of tokens
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
self.pooler = nn.Sequential(
nn.Linear(self._hparams.hidden_size, self._hparams.hidden_size),
nn.Tanh())
if self.pretrained_model_dir:
bert_utils.init_bert_checkpoint(self, self.pretrained_model_dir)
elif self._hparams.initializer:
initialize = layers.get_initializer(self._hparams.initializer)
assert initialize is not None
# Do not re-initialize LayerNorm modules.
for name, param in self.named_parameters():
if name.split(
'.')[-1] == 'weight' and 'layer_norm' not in name:
initialize(param)
def get_optimizer_fn(hparams=None):
"""Returns a function `optimizer_fn` of making optimizer instance, along
with the optimizer class.
.. role:: python(code)
:language: python
The function has the signiture
:python:`optimizer_fn(learning_rate=None) -> optimizer class instance`
See the :attr:`"optimizer"` field of
:meth:`~texar.core.default_optimization_hparams` for all
hyperparameters and default values.
The optimizer class must be a subclass of
:tf_main:`tf.train.Optimizer <train/Optimizer>`.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically.
Returns:
- If hparams["type"] is a string or optimizer class, returns\
`(optimizer_fn, optimizer class)`,
- If hparams["type"] is an optimizer instance, returns \
`(the optimizer instance, optimizer class)`
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams()["optimizer"])
opt = hparams["type"]
if isinstance(opt, tf.train.Optimizer):
return opt, type(opt)
opt_modules = [
'tensorflow.train', 'tensorflow.contrib.opt',
'texar.core.optimization', 'texar.custom'
]
try:
opt_class = utils.check_or_get_class(opt, opt_modules,
tf.train.Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"tf.train.Optimizer, or an instance of the subclass of "
"Optimizer.")
def _get_opt(learning_rate=None):
opt_kwargs = hparams["kwargs"].todict()
fn_args = set(utils.get_args(opt_class.__init__))
if 'learning_rate' in fn_args and learning_rate is not None:
opt_kwargs["learning_rate"] = learning_rate
return opt_class(**opt_kwargs)
return _get_opt, opt_class
def setUp(self):
self._vocab_size = 4
self._max_time = 8
self._batch_size = 16
self._emb_dim = 20
self._inputs = torch.rand(
self._batch_size, self._max_time, self._emb_dim,
dtype=torch.float)
self._embedding = torch.rand(
self._vocab_size, self._emb_dim, dtype=torch.float)
self._hparams = HParams(None, BasicRNNDecoder.default_hparams())
def _construct(cls,
hparams,
device: Optional[torch.device] = None,
vocab: Optional[Vocab] = None,
embedding: Optional[Vocab] = None):
mono_text_data = cls.__new__(cls)
mono_text_data._hparams = HParams(hparams,
mono_text_data.default_hparams())
if mono_text_data._hparams.dataset.variable_utterance:
raise NotImplementedError
dataset = mono_text_data._hparams.dataset
mono_text_data._other_transforms = dataset.other_transformations
# Create vocabulary
if vocab is not None:
mono_text_data._vocab = vocab
mono_text_data._bos_token = vocab.bos_token
mono_text_data._eos_token = vocab.eos_token
else:
mono_text_data._bos_token = dataset.bos_token
mono_text_data._eos_token = dataset.eos_token
bos = utils.default_str(mono_text_data._bos_token,
SpecialTokens.BOS)
eos = utils.default_str(mono_text_data._eos_token,
SpecialTokens.EOS)
mono_text_data._vocab = Vocab(dataset.vocab_file,
bos_token=bos,
eos_token=eos)
# Create embedding
if embedding is not None:
mono_text_data._embedding = embedding
else:
mono_text_data._embedding = mono_text_data.make_embedding(
dataset.embedding_init,
mono_text_data._vocab.token_to_id_map_py)
mono_text_data._delimiter = dataset.delimiter
mono_text_data._max_seq_length = dataset.max_seq_length
mono_text_data._length_filter_mode = _LengthFilterMode(
mono_text_data._hparams.dataset.length_filter_mode)
mono_text_data._pad_length = mono_text_data._max_seq_length
if mono_text_data._pad_length is not None:
mono_text_data._pad_length += sum(
int(x != '') for x in
[mono_text_data._bos_token, mono_text_data._eos_token])
data_source: SequenceDataSource[str] = SequenceDataSource([])
super(MonoTextData, mono_text_data).__init__(source=data_source,
hparams=hparams,
device=device)
return mono_text_data
def __init__(self,
pretrained_model_name=None,
cache_dir=None,
hparams=None):
EncoderBase.__init__(self, hparams)
BertBase.__init__(self, pretrained_model_name, cache_dir,
hparams) # put these things to BertBase
if self.pretrained_model:
self._hparams = HParams(self.pretrained_model_hparams,
self._hparams.todict())
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
# Word embedding
self.word_embedder = WordEmbedder(
vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Segment embedding for each type of tokens
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
with tf.variable_scope("pooler"):
kwargs_i = {
"units": self._hparams.hidden_size,
"activation": tf.tanh
}
layer_hparams = {"type": "Dense", "kwargs": kwargs_i}
self.pooler = layers.get_layer(hparams=layer_hparams)
def load_pretrained_config(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
r"""Load paths and configurations of the pre-trained model.
Args:
pretrained_model_name (optional): A str with the name
of a pre-trained model to load. If `None`, will use the model
name in :attr:`hparams`.
cache_dir (optional): The path to a folder in which the
pre-trained models will be cached. If `None` (default),
a default directory will be used.
hparams (dict or HParams, optional): Hyperparameters. Missing
hyperparameter will be set to default values. See
:meth:`default_hparams` for the hyperparameter structure
and default values.
"""
if not hasattr(self, "_hparams"):
self._hparams = HParams(hparams, self.default_hparams())
else:
# Probably already parsed by subclasses. We rely on subclass
# implementations to get this right.
# As a sanity check, we require `hparams` to be `None` in this case.
if hparams is not None:
raise ValueError(
"`self._hparams` is already assigned, but `hparams` "
"argument is not None.")
self.pretrained_model_dir = None
if pretrained_model_name is None:
pretrained_model_name = self._hparams.pretrained_model_name
if pretrained_model_name is not None:
self.pretrained_model_dir = self.download_checkpoint(
pretrained_model_name, cache_dir)
pretrained_model_hparams = self._transform_config(
self.pretrained_model_dir)
self._hparams = HParams(pretrained_model_hparams,
self._hparams.todict())
def get_optimizer_fn(hparams=None):
"""Returns a function of making optimizer instance, along with the
optimizer class.
The function has the signiture:
(learning_rate=None) -> instance of the optimizer class,
The optimizer class must be a subclass of :tf_main:`~tf.train.Optimizer`.
See the :attr:`"optimizer"` field in
:meth:`~texar.core.optimization.default_optimization_hparams` for all
hyperparameters and default values.
If :attr:`hparams["type"]` is an optimier instance, returns the instance
directly.
Args:
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically.
Returns:
(function that creates optimizer instance, optimizer class),
or the optimizer instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams()["optimizer"])
opt = hparams["type"]
if isinstance(opt, tf.train.Optimizer):
return opt, type(opt)
else:
opt_modules = [
'tensorflow.train', 'tensorflow.contrib.opt', 'texar.custom'
]
try:
opt_class = utils.check_or_get_class(opt, opt_modules,
tf.train.Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"tf.train.Optimizer, or an instance of the subclass of "
"Optimizer.")
def _get_opt(learning_rate=None):
opt_kwargs = hparams["kwargs"].todict()
fn_args = set(inspect.getargspec(opt_class.__init__).args)
if 'learning_rate' in fn_args and learning_rate is not None:
opt_kwargs["learning_rate"] = learning_rate
return opt_class(**opt_kwargs)
return _get_opt, opt_class
def __init__(self,
hparams: Optional[Union[HParams, Dict[str, Any]]] = None):
super().__init__()
if not hasattr(self, '_hparams'):
self._hparams = HParams(hparams, self.default_hparams())
else:
# Probably already parsed by subclasses. We rely on subclass
# implementations to get this right.
# As a sanity check, we require `hparams` to be `None` in this case.
if hparams is not None:
raise ValueError(
"`self._hparams` is already assigned, but `hparams` "
"argument is not None.")
def test_typecheck(self):
"""Tests type-check functionality.
"""
def _foo():
pass
def _bar():
pass
default_hparams = {"fn": _foo, "fn_2": _foo}
hparams = {"fn": _foo, "fn_2": _bar}
hparams_ = HParams(hparams, default_hparams)
self.assertEqual(hparams_.fn, default_hparams["fn"])
def __init__(self, data_hparams, hparams=None):
ModelBase.__init__(self, hparams)
self._data_hparams = HParams(data_hparams,
PairedTextData.default_hparams())
self._src_vocab = None
self._tgt_vocab = None
self._src_embedder = None
self._tgt_embedder = None
self._connector = None
self._encoder = None
self._decoder = None
def __init__(self,
pretrained_model_name: Optional[str] = None,
cache_dir: Optional[str] = None,
hparams=None):
super().__init__(pretrained_model_name=pretrained_model_name,
cache_dir=cache_dir,
hparams=hparams)
if self.pretrained_model_dir:
self._hparams = HParams(self.pretrained_model_hparams,
self._hparams.todict())
# Word embedding
self.word_embedder = WordEmbedder(vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The GPT2 decoder (a TransformerDecoder)
self.decoder = TransformerDecoder(
vocab_size=self._hparams.vocab_size,
output_layer=self.word_embedder.embedding,
hparams=self._hparams.decoder)
if self.pretrained_model_dir:
gpt2_utils.init_gpt2_checkpoint(self, self.pretrained_model_dir)
elif self._hparams.initializer:
initialize = layers.get_initializer(self._hparams.initializer)
assert initialize is not None
# Do not re-initialize LayerNorm modules.
for name, param in self.named_parameters():
if name.split(
'.')[-1] == 'weight' and 'layer_norm' not in name:
initialize(param)
def get_embedding(hparams=None,
init_value=None,
num_embeds=None,
variable_scope='Embedding'):
"""Creates embedding variable if not exists.
Args:
hparams (dict or HParams, optional): Embedding hyperparameters. Missing
hyperparameters are set to default values. See
:func:`~texar.modules.default_embedding_hparams`
for all hyperparameters and default values.
If :attr:`init_value` is given, :attr:`hparams["initializer"]`,
and :attr:`hparams["dim"]` are ignored.
init_value (Tensor or numpy array, optional): Initial values of the
embedding variable. If not given, embedding is initialized as
specified in :attr:`hparams["initializer"]`.
num_embeds (int, optional): The number of embedding items
(e.g., vocabulary size). Required if :attr:`init_value` is
not provided.
variable_scope (str or VariableScope, optional): Variable scope of
the embedding variable.
Returns:
Variable or Tensor: A 2D `Variable` or `Tensor` of the same shape with
:attr:`init_value` or of the shape
:attr:`[num_embeds, hparams["dim"]]`.
"""
with tf.variable_scope(variable_scope):
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_embedding_hparams())
regularizer = layers.get_regularizer(hparams["regularizer"])
if init_value is None:
initializer = layers.get_initializer(hparams["initializer"])
dim = hparams["dim"]
if not isinstance(hparams["dim"], (list, tuple)):
dim = [dim]
embedding = tf.get_variable(name='w',
shape=[num_embeds] + dim,
initializer=initializer,
regularizer=regularizer,
trainable=hparams["trainable"])
else:
init_value = tf.cast(init_value, tf.float32)
embedding = tf.get_variable(name='w',
initializer=init_value,
regularizer=regularizer,
trainable=hparams["trainable"])
return embedding
def __init__(self, hparams):
TextDataBase.__init__(self, hparams)
# Defaultizes hparams of each dataset
datasets_hparams = self._hparams.datasets
defaultized_datasets_hparams = []
for ds_hpms in datasets_hparams:
data_type = ds_hpms.get("data_type", None)
defaultized_ds_hpms = HParams(ds_hpms,
_default_dataset_hparams(data_type))
defaultized_datasets_hparams.append(defaultized_ds_hpms)
self._hparams.datasets = defaultized_datasets_hparams
with tf.name_scope(self.name, self.default_hparams()["name"]):
self._make_data()
def _get_static_lr(learning_rate=None, optimizer_class=None, hparams=None):
"""Return the base static learning_rate.
A helper function for creating the optimization function.
"""
hparams = HParams(hparams, default_optimization_hparams())
opt_hparams = hparams['optimizer']
if learning_rate is None:
learning_rate = opt_hparams["kwargs"].get("learning_rate", None)
if learning_rate is None:
# Try to get learning_rate from the default value of the
# optimizer's argument
opt_argspec = utils.get_default_arg_values(optimizer_class.__init__)
learning_rate = opt_argspec.get("learning_rate", None)
return learning_rate
def get_embedding(num_embeds: Optional[int] = None,
init_value: Optional[torch.Tensor] = None,
hparams=None):
r"""Creates embedding variable if not exists.
Args:
hparams (dict or HParams, optional): Embedding hyperparameters. Missing
hyperparameters are set to default values. See
:func:`~texar.modules.default_embedding_hparams`
for all hyperparameters and default values.
If :attr:`init_value` is given, :attr:`hparams["initializer"]`,
and :attr:`hparams["dim"]` are ignored.
init_value (Tensor or numpy array, optional): Initial values of the
embedding variable. If not given, embedding is initialized as
specified in :attr:`hparams["initializer"]`.
num_embeds (int, optional): The number of embedding items
(e.g., vocabulary size). Required if :attr:`init_value` is
not provided.
Returns:
A 2D :tensor:`Tensor` of the same shape with :attr:`init_value` or of
the shape ``[num_embeds, hparams["dim"]]``.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_embedding_hparams())
if init_value is None:
initializer = layers.get_initializer(
getattr(hparams, "initializer", None))
# TODO Shibiao: add regularizer
dim = hparams["dim"]
if not isinstance(hparams["dim"], (list, tuple)):
dim = [dim]
embedding = torch.empty(size=[num_embeds] + dim)
# initializer should be set by layers.get_initializer
if initializer:
embedding = initializer(embedding)
else:
embedding = torch.nn.init.xavier_uniform_(embedding)
else:
if torch.is_tensor(init_value):
embedding = init_value # Do not copy the tensor.
else:
embedding = torch.tensor(init_value, dtype=torch.float)
return embedding
def get_optimizer(
params: Union[List[torch.Tensor], List[Dict[str, List[torch.Tensor]]]],
hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \
Optimizer:
r"""Creates a optimizer instance.
Args:
params: an iterable of :class:`torch.Tensor` or
:class:`dict`. Specifies what Tensors should be optimized.
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
:return:
The :torch_docs:`torch.optim.Optimizer
<optim.html#torch.optim.Optimizer>` instance specified in
:attr:`hparams`.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_opt = hparams["optimizer"]
optimizer_type = hparams_opt["type"]
if isinstance(optimizer_type, Optimizer):
optimizer_class = optimizer_type
else:
optimizer_modules = ['torch.optim', 'texar.custom']
try:
optimizer_class = utils.check_or_get_class( # type: ignore
optimizer_type, optimizer_modules, Optimizer)
except TypeError:
raise ValueError(
"Unrecognized optimizer. Must be string name of the "
"optimizer class, or the class which is a subclass of "
"torch.optim.Optimizer, or an instance of the subclass of "
"Optimizer.")
optimizer_kwargs = hparams_opt["kwargs"].todict()
optimizer_kwargs.update({"params": params})
optimizer = optimizer_class(**optimizer_kwargs) # type: ignore
return optimizer
def get_scheduler(optimizer: Optimizer,
hparams: Optional[Union[HParams, Dict[str, Any]]] = None) -> \
Optional[_LRScheduler]:
r"""Creates a scheduler instance.
Args:
optimizer: A :torch_docs:`torch.optim.Optimizer
<optim.html#torch.optim.Optimizer>` instance.
hparams (dict or HParams, optional): hyperparameters. Missing
hyperparameters are set to default values automatically. See
:func:`~texar.core.default_optimization_hparams` for
all hyperparameters and default values.
:return:
A :torch_docs:`torch.optim.lr_scheduler._LRScheduler
<optim.html#how-to-adjust-learning-rate>` instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_optimization_hparams())
hparams_scheduler = hparams["learning_rate_decay"]
scheduler_type = hparams_scheduler["type"]
if scheduler_type == "" or scheduler_type is None:
scheduler = None
else:
if isinstance(scheduler_type, _LRScheduler):
scheduler_class = scheduler_type
else:
scheduler_modules = ['torch.optim.lr_scheduler', 'texar.custom']
try:
scheduler_class = utils.check_or_get_class( # type: ignore
scheduler_type, scheduler_modules, _LRScheduler)
except TypeError:
raise ValueError(
"Unrecognized lr_scheduler. Must be string name of the "
"lr_scheduler class, or the class which is a subclass of "
"torch.optim._LRScheduler.")
scheduler_kwargs = hparams_scheduler["kwargs"].todict()
scheduler_kwargs.update({"optimizer": optimizer})
scheduler = scheduler_class(**scheduler_kwargs) # type: ignore
return scheduler
def __init__(self, hparams, device: Optional[torch.device] = None):
self._hparams = HParams(hparams, self.default_hparams())
self._other_transforms = self._hparams.dataset.other_transformations
data_type = self._hparams.dataset["data_type"]
self._typecast_func: Union[Type[int], Type[float]]
if data_type == "int":
self._typecast_func = int
self._to_data_type = np.int32
elif data_type == "float":
self._typecast_func = float
self._to_data_type = np.float32
else:
raise ValueError(
"Incorrect 'data_type'. Currently 'int' and "
"'float' are supported. Received {}".format(data_type))
data_source = TextLineDataSource(
self._hparams.dataset.files,
compression_type=self._hparams.dataset.compression_type)
super().__init__(data_source, hparams, device=device)
def get_regularizer(hparams=None):
r"""Returns a variable regularizer instance.
See :func:`~texar.core.default_regularizer_hparams` for all
hyperparameters and default values.
The "type" field can be a subclass
of :class:`~texar.core.regularizers.Regularizer`, its string name
or module path, or a class instance.
Args:
hparams (dict or HParams, optional): Hyperparameters. Missing
hyperparameters are set to default values.
Returns:
A :class:`~texar.core.regularizers.Regularizer` instance.
`None` if :attr:`hparams` is `None` or taking the default
hyperparameter value.
Raises:
ValueError: The resulting regularizer is not an instance of
:class:`~texar.core.regularizers.Regularizer`.
"""
if hparams is None:
return None
if isinstance(hparams, dict):
hparams = HParams(hparams, default_regularizer_hparams())
rgl = utils.check_or_get_instance(
hparams.type, hparams.kwargs.todict(),
["texar.core.regularizers", "texar.custom"])
if not isinstance(rgl, Regularizer):
raise ValueError("The regularizer must be an instance of "
"texar.core.regularizers.Regularizer.")
if isinstance(rgl, L1L2) and rgl.l1 == 0. and rgl.l2 == 0.:
return None
return rgl
