def test_get_layer(self):
"""Tests :func:`texar.core.layers.get_layer`.
"""
hparams = {"type": "Conv1D"}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, tf.layers.Conv1D))
hparams = {
"type": "MergeLayer",
"kwargs": {
"layers": [{
"type": "Conv1D"
}, {
"type": "Conv1D"
}]
}
}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, tx.core.MergeLayer))
hparams = {"type": tf.layers.Conv1D}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, tf.layers.Conv1D))
hparams = {"type": tf.layers.Conv1D(filters=10, kernel_size=2)}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, tf.layers.Conv1D))
def test_get_layer(self):
r"""Tests :func:`texar.core.layers.get_layer`.
"""
hparams = {
"type": "Conv1d",
"kwargs": {
"in_channels": 16,
"out_channels": 32,
"kernel_size": 2
}
}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, nn.Conv1d))
hparams = {
"type": "MergeLayer",
"kwargs": {
"layers": [{
"type": "Conv1d",
"kwargs": {
"in_channels": 16,
"out_channels": 32,
"kernel_size": 2
}
}, {
"type": "Conv1d",
"kwargs": {
"in_channels": 16,
"out_channels": 32,
"kernel_size": 2
}
}]
}
}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, tx.core.MergeLayer))
hparams = {
"type": "Conv1d",
"kwargs": {
"in_channels": 16,
"out_channels": 32,
"kernel_size": 2
}
}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, nn.Conv1d))
hparams = {
"type": nn.Conv1d(in_channels=16, out_channels=32, kernel_size=2)
}
layer = layers.get_layer(hparams)
self.assertTrue(isinstance(layer, nn.Conv1d))
def _build_dense_output_layer(hparams):
nlayers = hparams.num_layers
if nlayers <= 0:
return None
layer_size = _to_list(
hparams.layer_size, 'output_layer.layer_size', nlayers)
other_kwargs = hparams.other_dense_kwargs or {}
if isinstance(other_kwargs, HParams):
other_kwargs = other_kwargs.todict()
if not isinstance(other_kwargs, dict):
raise ValueError(
"hparams 'output_layer.other_dense_kwargs' must be a dict.")
dense_layers = []
for i in range(nlayers):
if i == nlayers - 1:
activation = hparams.final_layer_activation
else:
activation = hparams.activation
kwargs_i = {"units": layer_size[i],
"activation": activation,
"name": "dense_%d" % (i+1)}
kwargs_i.update(other_kwargs)
layer_hparams = {"type": "Dense", "kwargs": kwargs_i}
dense_layers.append(layers.get_layer(hparams=layer_hparams))
if len(dense_layers) == 1:
dense_layers = dense_layers[0]
return dense_layers
def _build_layers(network, layers=None, layer_hparams=None):
"""Builds layers.
Either :attr:`layer_hparams` or :attr:`layers` must be
provided. If both are given, :attr:`layers` will be used.
Args:
network: An instance of a subclass of
:class:`~texar.modules.networks.network_base.FeedForwardNetworkBase`
layers (optional): A list of layer instances.
layer_hparams (optional): A list of layer hparams, each to which
is fed to :func:`~texar.core.layers.get_layer` to create the
layer instance.
"""
with tf.variable_scope(network.variable_scope):
if layers is not None:
network._layers = layers
else:
if layer_hparams is None:
raise ValueError(
'Either `layer` or `layer_hparams` is required.')
network._layers = []
for _, hparams in enumerate(layer_hparams):
network._layers.append(get_layer(hparams=hparams))
for layer in network._layers:
layer_name = uniquify_str(layer.name, network._layer_names)
network._layer_names.append(layer_name)
network._layers_by_name[layer_name] = layer
def _build_layers(network: FeedForwardNetworkBase,
layers: Optional[nn.ModuleList] = None,
layer_hparams: Optional[List[Union[HParams,
Dict[str, Any]]]] = None):
r"""Builds layers.
Either :attr:`layer_hparams` or :attr:`layers` must be
provided. If both are given, :attr:`layers` will be used.
Args:
network: An instance of a subclass of
:class:`~texar.modules.networks.FeedForwardNetworkBase`.
layers (optional): A list of layer instances supplied as an instance of
:torch_nn:`ModuleList`.
layer_hparams (optional): A list of layer hparams, each to which
is fed to :func:`~texar.core.layers.get_layer` to create the
layer instance.
"""
if layers is not None:
network._layers = layers
else:
if layer_hparams is None:
raise ValueError('Either `layer` or `layer_hparams` is required.')
network._layers = nn.ModuleList()
for _, hparams in enumerate(layer_hparams):
network._layers.append(get_layer(hparams=hparams))
for layer in network._layers:
layer_name = uniquify_str(layer._get_name(), network._layer_names)
network._layer_names.append(layer_name)
network._layers_by_name[layer_name] = layer
def _build_dense_output_layer(cell_output_size: int,
hparams: HParams) -> Optional[nn.Sequential]:
r"""Build the output layers.
Args:
cell_output_size: The output size of the rnn cell.
hparams (dict or HParams): Hyperparameters. Missing hyperparameters
will be set to default values. See
:meth:`default_hparams` for the hyperparameter structure and
default values.
Returns:
A :torch_nn:`Sequential` module containing the output layers.
"""
nlayers = hparams.num_layers
if nlayers <= 0:
return None
layer_size = _to_list(hparams.layer_size, 'output_layer.layer_size',
nlayers)
dropout_layer_ids = _to_list(hparams.dropout_layer_ids)
other_kwargs = hparams.other_dense_kwargs or {}
if isinstance(other_kwargs, HParams):
other_kwargs = other_kwargs.todict()
if not isinstance(other_kwargs, dict):
raise ValueError(
"hparams 'output_layer.other_dense_kwargs' must be a dict.")
output_layers: List[nn.Module] = []
for i in range(nlayers):
if i in dropout_layer_ids:
# TODO: Variational dropout is not implemented.
output_layers.append(nn.Dropout(p=hparams.dropout_rate))
dense_layer = nn.Linear(
in_features=(cell_output_size if i == 0 else layer_size[i - 1]),
out_features=layer_size[i],
**other_kwargs)
output_layers.append(dense_layer)
if i == nlayers - 1:
activation = hparams.final_layer_activation
else:
activation = hparams.activation
if activation is not None:
layer_hparams = {"type": activation, "kwargs": {}}
activation_layer = layers.get_layer(hparams=layer_hparams)
output_layers.append(activation_layer)
if nlayers in dropout_layer_ids:
output_layers.append(nn.Dropout(p=hparams.dropout_rate))
return nn.Sequential(*output_layers)
def __init__(self,
pretrained_model_name=None,
cache_dir=None,
hparams=None):
ClassifierBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
# Creates the underlying encoder
encoder_hparams = utils.dict_fetch(
hparams, BertEncoder.default_hparams())
if encoder_hparams is not None:
encoder_hparams['name'] = None
self._encoder = BertEncoder(
pretrained_model_name=pretrained_model_name,
cache_dir=cache_dir,
hparams=encoder_hparams)
# Creates an dropout layer
drop_kwargs = {"rate": self._hparams.dropout}
layer_hparams = {"type": "Dropout", "kwargs": drop_kwargs}
self._dropout_layer = layers.get_layer(hparams=layer_hparams)
# Creates an additional classification layer if needed
self._num_classes = self._hparams.num_classes
if self._num_classes <= 0:
self._logit_layer = None
else:
logit_kwargs = self._hparams.logit_layer_kwargs
if logit_kwargs is None:
logit_kwargs = {}
elif not isinstance(logit_kwargs, HParams):
raise ValueError(
"hparams['logit_layer_kwargs'] must be a dict.")
else:
logit_kwargs = logit_kwargs.todict()
logit_kwargs.update({"units": self._num_classes})
if 'name' not in logit_kwargs:
logit_kwargs['name'] = "logit_layer"
layer_hparams = {"type": "Dense", "kwargs": logit_kwargs}
self._logit_layer = layers.get_layer(hparams=layer_hparams)
def append_layer(self, layer: Union[nn.Module, HParams, Dict[str, Any]]):
r"""Appends a layer to the end of the network.
Args:
layer: A subclass of :torch_nn:`Module`, or a dict of layer
hyperparameters.
"""
layer_ = layer
if not isinstance(layer_, nn.Module):
layer_ = get_layer(hparams=layer_)
self._layers.append(layer_)
layer_name = uniquify_str(layer_._get_name(), self._layer_names)
self._layer_names.append(layer_name)
self._layers_by_name[layer_name] = layer_
def append_layer(self, layer):
"""Appends a layer to the end of the network. The method is only
feasible before :attr:`_build` is called.
Args:
layer: A :tf_main:`tf.layers.Layer <layers/Layer>` instance, or
a dict of layer hyperparameters.
"""
if self._built:
raise TexarError("`FeedForwardNetwork.append_layer` can be "
"called only before `_build` is called.")
with tf.variable_scope(self.variable_scope):
layer_ = layer
if not isinstance(layer_, tf.layers.Layer):
layer_ = get_layer(hparams=layer_)
self._layers.append(layer_)
layer_name = uniquify_str(layer_.name, self._layer_names)
self._layer_names.append(layer_name)
self._layers_by_name[layer_name] = layer_
def __init__(self,
pretrained_model_name=None,
cache_dir=None,
hparams=None):
EncoderBase.__init__(self, hparams)
BertBase.__init__(self, pretrained_model_name, cache_dir,
hparams) # put these things to BertBase
if self.pretrained_model:
self._hparams = HParams(self.pretrained_model_hparams,
self._hparams.todict())
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
# Word embedding
self.word_embedder = WordEmbedder(
vocab_size=self._hparams.vocab_size,
hparams=self._hparams.embed)
# Segment embedding for each type of tokens
self.segment_embedder = WordEmbedder(
vocab_size=self._hparams.type_vocab_size,
hparams=self._hparams.segment_embed)
# Position embedding
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embed)
# The BERT encoder (a TransformerEncoder)
self.encoder = TransformerEncoder(hparams=self._hparams.encoder)
with tf.variable_scope("pooler"):
kwargs_i = {
"units": self._hparams.hidden_size,
"activation": tf.tanh
}
layer_hparams = {"type": "Dense", "kwargs": kwargs_i}
self.pooler = layers.get_layer(hparams=layer_hparams)
def __init__(self,
cell=None,
cell_dropout_mode=None,
output_layer=None,
hparams=None):
ClassifierBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
# Creates the underlying encoder
encoder_hparams = utils.dict_fetch(
hparams, UnidirectionalRNNEncoder.default_hparams())
if encoder_hparams is not None:
encoder_hparams['name'] = None
self._encoder = UnidirectionalRNNEncoder(
cell=cell,
cell_dropout_mode=cell_dropout_mode,
output_layer=output_layer,
hparams=encoder_hparams)
# Creates an additional classification layer if needed
self._num_classes = self._hparams.num_classes
if self._num_classes <= 0:
self._logit_layer = None
else:
logit_kwargs = self._hparams.logit_layer_kwargs
if logit_kwargs is None:
logit_kwargs = {}
elif not isinstance(logit_kwargs, HParams):
raise ValueError(
"hparams['logit_layer_kwargs'] must be a dict.")
else:
logit_kwargs = logit_kwargs.todict()
logit_kwargs.update({"units": self._num_classes})
if 'name' not in logit_kwargs:
logit_kwargs['name'] = "logit_layer"
layer_hparams = {"type": "Dense", "kwargs": logit_kwargs}
self._logit_layer = layers.get_layer(hparams=layer_hparams)