def test_get_layer(self): """Tests :func:`texar.core.layers.get_layer`. """ hparams = {"type": "Conv1D"} layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, tf.layers.Conv1D)) hparams = { "type": "MergeLayer", "kwargs": { "layers": [{ "type": "Conv1D" }, { "type": "Conv1D" }] } } layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, tx.core.MergeLayer)) hparams = {"type": tf.layers.Conv1D} layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, tf.layers.Conv1D)) hparams = {"type": tf.layers.Conv1D(filters=10, kernel_size=2)} layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, tf.layers.Conv1D))
def test_get_layer(self): r"""Tests :func:`texar.core.layers.get_layer`. """ hparams = { "type": "Conv1d", "kwargs": { "in_channels": 16, "out_channels": 32, "kernel_size": 2 } } layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, nn.Conv1d)) hparams = { "type": "MergeLayer", "kwargs": { "layers": [{ "type": "Conv1d", "kwargs": { "in_channels": 16, "out_channels": 32, "kernel_size": 2 } }, { "type": "Conv1d", "kwargs": { "in_channels": 16, "out_channels": 32, "kernel_size": 2 } }] } } layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, tx.core.MergeLayer)) hparams = { "type": "Conv1d", "kwargs": { "in_channels": 16, "out_channels": 32, "kernel_size": 2 } } layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, nn.Conv1d)) hparams = { "type": nn.Conv1d(in_channels=16, out_channels=32, kernel_size=2) } layer = layers.get_layer(hparams) self.assertTrue(isinstance(layer, nn.Conv1d))
def _build_dense_output_layer(hparams): nlayers = hparams.num_layers if nlayers <= 0: return None layer_size = _to_list( hparams.layer_size, 'output_layer.layer_size', nlayers) other_kwargs = hparams.other_dense_kwargs or {} if isinstance(other_kwargs, HParams): other_kwargs = other_kwargs.todict() if not isinstance(other_kwargs, dict): raise ValueError( "hparams 'output_layer.other_dense_kwargs' must be a dict.") dense_layers = [] for i in range(nlayers): if i == nlayers - 1: activation = hparams.final_layer_activation else: activation = hparams.activation kwargs_i = {"units": layer_size[i], "activation": activation, "name": "dense_%d" % (i+1)} kwargs_i.update(other_kwargs) layer_hparams = {"type": "Dense", "kwargs": kwargs_i} dense_layers.append(layers.get_layer(hparams=layer_hparams)) if len(dense_layers) == 1: dense_layers = dense_layers[0] return dense_layers
def _build_layers(network, layers=None, layer_hparams=None): """Builds layers. Either :attr:`layer_hparams` or :attr:`layers` must be provided. If both are given, :attr:`layers` will be used. Args: network: An instance of a subclass of :class:`~texar.modules.networks.network_base.FeedForwardNetworkBase` layers (optional): A list of layer instances. layer_hparams (optional): A list of layer hparams, each to which is fed to :func:`~texar.core.layers.get_layer` to create the layer instance. """ with tf.variable_scope(network.variable_scope): if layers is not None: network._layers = layers else: if layer_hparams is None: raise ValueError( 'Either `layer` or `layer_hparams` is required.') network._layers = [] for _, hparams in enumerate(layer_hparams): network._layers.append(get_layer(hparams=hparams)) for layer in network._layers: layer_name = uniquify_str(layer.name, network._layer_names) network._layer_names.append(layer_name) network._layers_by_name[layer_name] = layer
def _build_layers(network: FeedForwardNetworkBase, layers: Optional[nn.ModuleList] = None, layer_hparams: Optional[List[Union[HParams, Dict[str, Any]]]] = None): r"""Builds layers. Either :attr:`layer_hparams` or :attr:`layers` must be provided. If both are given, :attr:`layers` will be used. Args: network: An instance of a subclass of :class:`~texar.modules.networks.FeedForwardNetworkBase`. layers (optional): A list of layer instances supplied as an instance of :torch_nn:`ModuleList`. layer_hparams (optional): A list of layer hparams, each to which is fed to :func:`~texar.core.layers.get_layer` to create the layer instance. """ if layers is not None: network._layers = layers else: if layer_hparams is None: raise ValueError('Either `layer` or `layer_hparams` is required.') network._layers = nn.ModuleList() for _, hparams in enumerate(layer_hparams): network._layers.append(get_layer(hparams=hparams)) for layer in network._layers: layer_name = uniquify_str(layer._get_name(), network._layer_names) network._layer_names.append(layer_name) network._layers_by_name[layer_name] = layer
def _build_dense_output_layer(cell_output_size: int, hparams: HParams) -> Optional[nn.Sequential]: r"""Build the output layers. Args: cell_output_size: The output size of the rnn cell. hparams (dict or HParams): Hyperparameters. Missing hyperparameters will be set to default values. See :meth:`default_hparams` for the hyperparameter structure and default values. Returns: A :torch_nn:`Sequential` module containing the output layers. """ nlayers = hparams.num_layers if nlayers <= 0: return None layer_size = _to_list(hparams.layer_size, 'output_layer.layer_size', nlayers) dropout_layer_ids = _to_list(hparams.dropout_layer_ids) other_kwargs = hparams.other_dense_kwargs or {} if isinstance(other_kwargs, HParams): other_kwargs = other_kwargs.todict() if not isinstance(other_kwargs, dict): raise ValueError( "hparams 'output_layer.other_dense_kwargs' must be a dict.") output_layers: List[nn.Module] = [] for i in range(nlayers): if i in dropout_layer_ids: # TODO: Variational dropout is not implemented. output_layers.append(nn.Dropout(p=hparams.dropout_rate)) dense_layer = nn.Linear( in_features=(cell_output_size if i == 0 else layer_size[i - 1]), out_features=layer_size[i], **other_kwargs) output_layers.append(dense_layer) if i == nlayers - 1: activation = hparams.final_layer_activation else: activation = hparams.activation if activation is not None: layer_hparams = {"type": activation, "kwargs": {}} activation_layer = layers.get_layer(hparams=layer_hparams) output_layers.append(activation_layer) if nlayers in dropout_layer_ids: output_layers.append(nn.Dropout(p=hparams.dropout_rate)) return nn.Sequential(*output_layers)
def __init__(self, pretrained_model_name=None, cache_dir=None, hparams=None): ClassifierBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): # Creates the underlying encoder encoder_hparams = utils.dict_fetch( hparams, BertEncoder.default_hparams()) if encoder_hparams is not None: encoder_hparams['name'] = None self._encoder = BertEncoder( pretrained_model_name=pretrained_model_name, cache_dir=cache_dir, hparams=encoder_hparams) # Creates an dropout layer drop_kwargs = {"rate": self._hparams.dropout} layer_hparams = {"type": "Dropout", "kwargs": drop_kwargs} self._dropout_layer = layers.get_layer(hparams=layer_hparams) # Creates an additional classification layer if needed self._num_classes = self._hparams.num_classes if self._num_classes <= 0: self._logit_layer = None else: logit_kwargs = self._hparams.logit_layer_kwargs if logit_kwargs is None: logit_kwargs = {} elif not isinstance(logit_kwargs, HParams): raise ValueError( "hparams['logit_layer_kwargs'] must be a dict.") else: logit_kwargs = logit_kwargs.todict() logit_kwargs.update({"units": self._num_classes}) if 'name' not in logit_kwargs: logit_kwargs['name'] = "logit_layer" layer_hparams = {"type": "Dense", "kwargs": logit_kwargs} self._logit_layer = layers.get_layer(hparams=layer_hparams)
def append_layer(self, layer: Union[nn.Module, HParams, Dict[str, Any]]): r"""Appends a layer to the end of the network. Args: layer: A subclass of :torch_nn:`Module`, or a dict of layer hyperparameters. """ layer_ = layer if not isinstance(layer_, nn.Module): layer_ = get_layer(hparams=layer_) self._layers.append(layer_) layer_name = uniquify_str(layer_._get_name(), self._layer_names) self._layer_names.append(layer_name) self._layers_by_name[layer_name] = layer_
def append_layer(self, layer): """Appends a layer to the end of the network. The method is only feasible before :attr:`_build` is called. Args: layer: A :tf_main:`tf.layers.Layer <layers/Layer>` instance, or a dict of layer hyperparameters. """ if self._built: raise TexarError("`FeedForwardNetwork.append_layer` can be " "called only before `_build` is called.") with tf.variable_scope(self.variable_scope): layer_ = layer if not isinstance(layer_, tf.layers.Layer): layer_ = get_layer(hparams=layer_) self._layers.append(layer_) layer_name = uniquify_str(layer_.name, self._layer_names) self._layer_names.append(layer_name) self._layers_by_name[layer_name] = layer_
def __init__(self, pretrained_model_name=None, cache_dir=None, hparams=None): EncoderBase.__init__(self, hparams) BertBase.__init__(self, pretrained_model_name, cache_dir, hparams) # put these things to BertBase if self.pretrained_model: self._hparams = HParams(self.pretrained_model_hparams, self._hparams.todict()) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) # Word embedding self.word_embedder = WordEmbedder( vocab_size=self._hparams.vocab_size, hparams=self._hparams.embed) # Segment embedding for each type of tokens self.segment_embedder = WordEmbedder( vocab_size=self._hparams.type_vocab_size, hparams=self._hparams.segment_embed) # Position embedding self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embed) # The BERT encoder (a TransformerEncoder) self.encoder = TransformerEncoder(hparams=self._hparams.encoder) with tf.variable_scope("pooler"): kwargs_i = { "units": self._hparams.hidden_size, "activation": tf.tanh } layer_hparams = {"type": "Dense", "kwargs": kwargs_i} self.pooler = layers.get_layer(hparams=layer_hparams)
def __init__(self, cell=None, cell_dropout_mode=None, output_layer=None, hparams=None): ClassifierBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): # Creates the underlying encoder encoder_hparams = utils.dict_fetch( hparams, UnidirectionalRNNEncoder.default_hparams()) if encoder_hparams is not None: encoder_hparams['name'] = None self._encoder = UnidirectionalRNNEncoder( cell=cell, cell_dropout_mode=cell_dropout_mode, output_layer=output_layer, hparams=encoder_hparams) # Creates an additional classification layer if needed self._num_classes = self._hparams.num_classes if self._num_classes <= 0: self._logit_layer = None else: logit_kwargs = self._hparams.logit_layer_kwargs if logit_kwargs is None: logit_kwargs = {} elif not isinstance(logit_kwargs, HParams): raise ValueError( "hparams['logit_layer_kwargs'] must be a dict.") else: logit_kwargs = logit_kwargs.todict() logit_kwargs.update({"units": self._num_classes}) if 'name' not in logit_kwargs: logit_kwargs['name'] = "logit_layer" layer_hparams = {"type": "Dense", "kwargs": logit_kwargs} self._logit_layer = layers.get_layer(hparams=layer_hparams)