def __init__(self,
input_size: int,
hidden_size: int,
num_layers: int,
mode='LSTM',
require_grad=True,
dropout=0.3):
super(ElmoEncoder, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.mode = mode
self.require_grad = require_grad
self.dropout = nn.Dropout(dropout)
self.forwards = nn.ModuleList(
nn.RNNBase(mode,
input_size if i == 0 else hidden_size,
hidden_size,
1,
bidirectional=False) for i in range(num_layers))
self.backwards = nn.ModuleList(
nn.RNNBase(mode,
input_size if i == 0 else hidden_size,
hidden_size,
1,
bidirectional=False) for i in range(num_layers))
def __init__(self,
input_size,
hidden_size,
num_layers=3,
rnn="GRU",
dropout=0.1,
batch_first=False):
super(RNNEncoder, self).__init__()
if rnn not in _avaialable_rnns:
raise ValueError("Invalid RNN type `{}`. Use one of {}.".format(
rnn, _avaialable_rnns))
self.batch_first = batch_first
self.num_layers = num_layers
self.rnn = nn.RNNBase(rnn,
input_size,
hidden_size,
num_layers,
dropout=dropout,
bidirectional=True,
batch_first=batch_first)
self.concat = nn.Sequential(nn.SELU(),
nn.Linear(hidden_size * 2, hidden_size),
nn.SELU())
def __init__(self,
input_size,
hidden_size,
num_layers=3,
rnn="GRU",
attention="dot",
dropout=0.1,
batch_first=False):
super(RNNDecoder, self).__init__()
if rnn not in _avaialable_rnns:
raise ValueError("Invalid RNN type `{}`. Use one of {}.".format(
rnn, _avaialable_rnns))
self.batch_first = batch_first
self.rnn = nn.RNNBase(rnn,
input_size,
hidden_size,
num_layers,
dropout=dropout,
bidirectional=False,
batch_first=batch_first)
self.attention = LuongAttention(attention,
batch_first=batch_first,
query_size=hidden_size,
key_size=hidden_size)
self.out = nn.Sequential(nn.SELU(),
nn.Linear(2 * hidden_size, hidden_size),
nn.SELU())
def __build_model(self):
self.embedding_layer = nn.Embedding.from_pretrained(self.embedding)
if self.cell_type == "PEEP":
self.peephole_cell = PeepholeCell(self.embedding_dim,
self.rnn_layer_size)
else:
self.rnn_layer = nn.RNNBase(mode=self.cell_type,
input_size=self.embedding_dim,
hidden_size=self.rnn_layer_size,
num_layers=self.rnn_layer_number,
bidirectional=self.bidirectional,
batch_first=True)
self.hidden_layer = None
self.dense_layer = nn.Linear(self.rnn_out_size, self.tagset_size)
self.activation_layer = self.activation(dim=1)
def __init__(self, rnnType, ntoken, ninp, nhid, nlayers):
super(RNNModel, self).__init__(
encoder = nn.sparse.Embedding(ntoken, ninp),
rnn = nn.RNNBase(rnnType, ninp, nhid, nlayers, bias=False),
decoder = nn.Linear(nhid, ntoken),
)
# FIXME: add stdv named argument to reset_parameters
# (and/or to the constructors)
initrange = 0.1
self.encoder.weight.data.uniform_(-initrange, initrange)
self.decoder.bias.data.fill_(0)
self.decoder.weight.data.uniform_(-initrange, initrange)
self.rnnType = rnnType
self.nhid = nhid
self.nlayers = nlayers
def __init__(self, n_features, n_output, n_layers, batch_size, dropout=0., rnn_type="RNN_RELU"):
super(RNNModel, self).__init__()
n_hidden = 100
self._rnn_type = rnn_type
self._dropout = dropout
self.drop = nn.Dropout(self._dropout)
if self._rnn_type not in self.RNN_TYPES:
raise ValueError("An invalid rnn_type, options are %s" % (self.RNN_TYPES,))
self._rnn_in, self._hidden_size, self._rnn_layers = n_features, n_hidden, n_layers
self.rnn = nn.RNNBase(self._rnn_type, self._rnn_in, self._hidden_size, self._rnn_layers, bias=False,
dropout=self._dropout, batch_first=True)
self._linear = nn.Linear(n_hidden, n_output)
self.log_softmax = nn.LogSoftmax(dim=2)
self._activation_func = functional.relu
self._batch_size = batch_size
self._hidden = self._init_hidden()
def __init__(self, feat_x_n, topo_x_n, n_output, h_layers, dropout=0., rnn_type="RNN_RELU"):
super(RNNModel, self).__init__()
self._comb = "topo"
n_input = {"feat": feat_x_n, "topo": topo_x_n, "comb": feat_x_n + topo_x_n}[self._comb]
all_layers = [n_input] + h_layers + [n_output]
self._rnn_type = rnn_type
self._dropout = dropout
self.drop = nn.Dropout(self._dropout)
if self._rnn_type not in self.RNN_TYPES:
raise ValueError("An invalid rnn_type, options are %s" % (self.RNN_TYPES,))
self._rnn_in, self._rnn_out, self._rnn_layers = all_layers[0], all_layers[0], 1
self.rnn = nn.RNNBase(self._rnn_type, self._rnn_in, self._rnn_out, self._rnn_layers,
dropout=self._dropout)
self.gcn_layers = nn.ModuleList([GraphConvolution(first, second)
for first, second in zip(all_layers[:-1], all_layers[1:])])
self._activation_func = functional.relu
def __init__(self):
super().__init__()
# self.embed = nn.Embedding(V, D, max_norm=config.max_norm)
self.embed = nn.Embedding(opt.vocab_size, opt.vocab_dim)
if opt.pretrained_embedding:
self.embed.weight.data.copy_(opt.pretrained_embedding)
# self.embed.weight.requires_grad = False # 冻结词向量
self.rnn = nn.RNNBase(
mode=opt.mode,
input_size=opt.vocab_dim,
hidden_size=opt.rnn_hidden_size,
dropout=opt.rnn_dropout,
num_layers=opt.rnn_layers_num,
bidirectional=opt.bidirectional,
batch_first=True # input/output shape (batch, time_step, input_size)
)
__in_features = self.rnn.hidden_size * (self.rnn.bidirectional + 1)
self.fc1 = nn.Linear(__in_features,
__in_features // 2)
self.fc2 = nn.Linear(self.fc1.out_features, opt.class_num)
def __init__(self,
input_sizes,
hidden_sizes, mode="GRU", bias=True,
bidirectional=False, batch_first=True):
super(LinearRnnBase, self).__init__()
if len(input_sizes) != len(hidden_sizes):
err_msg = "Size mismatch between len(input_sizes) and len(hidden_sizes)"
raise ValueError(err_msg)
self.num_layers = len(hidden_sizes)
self.batch_first = batch_first
self.multilayer_rnn = nn.ModuleList([
nn.RNNBase(
mode=mode,
input_size=input_sizes[i],
hidden_size=hidden_sizes[i],
bias=bias,
bidirectional=bidirectional
) for i in range(self.num_layers)
])
def __init__(self,
mode,
input_size,
hidden_size,
num_layers=1,
bias=True,
batch_first=False,
dropout=0,
bidirectional=False,
input_module=None,
output_module=None):
"""
Args:
mode: The type of RNN to use. One of ['LSTM', 'GRU',
'RNN_TANH', 'RNN_RELU']
input_size: The number of expected features in the input `x`
hidden_size: The number of features in the hidden state `h`
num_layers: Number of recurrent layers. E.g., setting ``num_layers=2``
would mean stacking two RNNs together to form a `stacked RNN`,
with the second RNN taking in outputs of the first RNN and
computing the final results. Default: 1
bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`.
Default: ``True``
batch_first: If ``True``, then the input and output tensors are provided
as (batch, seq, feature)
dropout: If non-zero, introduces a `Dropout` layer on the outputs of each
RNN layer except the last layer, with dropout probability equal to
:attr:`dropout`. Default: 0
bidirectional: If ``True``, becomes a bidirectional RNN. Default: ``False``
input_module: A module to apply on the packed input data before feeding into
the RNN
output_module A module to apply to the packed output data of the RNN
Inputs: hidden_state, packed_inputs
- **packed_inputs** A list of input `PackedSequence`s whose data are fed into the
input module (the data list is exapnded using * and fed).
See :func:`torch.nn.utils.rnn.pack_padded_sequence` or
:func:`torch.nn.utils.rnn.pack_sequence` for details.
- **hidden_state** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
containing the initial hidden state for each element in the batch.
- An additional cell state is also needed if mode is 'LSTM'
Outputs: output, hidden_state
- **output** A packed variable length sequence with the data from the output_module.
See :func:`torch.nn.utils.rnn.pack_padded_sequence` or
:func:`torch.nn.utils.rnn.pack_sequence` for details.
- **hidden_state** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
containing the hidden state for `t = seq_len`
- An additional cell state is also returned if mode is 'LSTM'
"""
super().__init__()
self.input_module = input_module
self.output_module = output_module
self.rnn = nn.RNNBase(mode,
input_size,
hidden_size,
num_layers,
bias,
batch_first,
dropout,
bidirectional)