def __init__(self, input_size, hidden_size, num_layers=1, batch_first=False, \
bidirectional=False, dropout_in=0, dropout_out=0):
super(HighwayBiLSTM, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.batch_first = batch_first
self.bidirectional = bidirectional
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.num_directions = 2 if bidirectional else 1
self.fcells, self.f_dropout, self.f_hidden_dropout = [], [], []
self.bcells, self.b_dropout, self.b_hidden_dropout = [], [], []
for layer in range(num_layers):
layer_input_size = input_size if layer == 0 else hidden_size
self.fcells.append(
MyHighwayLSTMCell(input_size=layer_input_size,
hidden_size=hidden_size))
self.f_dropout.append(DropoutLayer(hidden_size, self.dropout_out))
self.f_hidden_dropout.append(
DropoutLayer(hidden_size, self.dropout_out))
if self.bidirectional:
self.bcells.append(
MyHighwayLSTMCell(input_size=hidden_size,
hidden_size=hidden_size))
self.b_dropout.append(
DropoutLayer(hidden_size, self.dropout_out))
self.b_hidden_dropout.append(
DropoutLayer(hidden_size, self.dropout_out))
self.fcells, self.bcells = nn.ModuleList(self.fcells), nn.ModuleList(
self.bcells)
self.f_dropout, self.b_dropout = nn.ModuleList(
self.f_dropout), nn.ModuleList(self.b_dropout)
def __init__(self, input_size, hidden_size, num_layers=1, batch_first=False, \
bidirectional=False, dropout_in=0, dropout_out=0):
super(Highway_Concat_BiLSTM, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.batch_first = batch_first
self.bidirectional = bidirectional
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.num_directions = 2 if bidirectional else 1
self.fcells, self.f_dropout, self.f_hidden_dropout = [], [], []
self.bcells, self.b_dropout, self.b_hidden_dropout = [], [], []
self.f_initial, self.b_initial = [], []
for layer in range(num_layers):
layer_input_size = input_size if layer == 0 else 2 * hidden_size if self.bidirectional else hidden_size
self.fcells.append(MyLSTMCell(input_size=layer_input_size, hidden_size=hidden_size))
self.f_dropout.append(DropoutLayer(hidden_size, self.dropout_out))
self.f_hidden_dropout.append(DropoutLayer(hidden_size, self.dropout_out))
self.f_initial.append(nn.Parameter(torch.Tensor(2, self.hidden_size)))
assert self.bidirectional is True
self.bcells.append(MyLSTMCell(input_size=layer_input_size, hidden_size=hidden_size))
self.b_dropout.append(DropoutLayer(hidden_size, self.dropout_out))
self.b_hidden_dropout.append(DropoutLayer(hidden_size, self.dropout_out))
self.b_initial.append(nn.Parameter(torch.Tensor(2, self.hidden_size)))
self.lstm_project_layer = nn.ModuleList([nn.Linear(2 * self.hidden_size, 2 * self.hidden_size)
for _ in range(num_layers - 1)])
self.fcells, self.bcells = nn.ModuleList(self.fcells), nn.ModuleList(self.bcells)
self.f_dropout, self.b_dropout = nn.ModuleList(self.f_dropout), nn.ModuleList(self.b_dropout)
self.f_hidden_dropout, self.b_hidden_dropout = \
nn.ModuleList(self.f_hidden_dropout), nn.ModuleList(self.b_hidden_dropout)
self.f_initial, self.b_initial = nn.ParameterList(self.f_initial), nn.ParameterList(self.b_initial)
self.reset_parameters()
def __init__(self, in_dim, hidden_dim, cuda_id=""):
super(HBiLSTM, self).__init__()
self.batch_size = 1
self.cuda_id = cuda_id
self.in_dim = in_dim
self.hidden_dim = hidden_dim
self.num_layers = 1 # this is a BiLSTM with Highway
self.bilstm = nn.LSTM(self.in_dim, self.hidden_dim, num_layers=self.num_layers, \
batch_first=True, bidirectional=True)
self.in_dropout_layer = DropoutLayer(in_dim, 0.1)
self.out_dropout_layer = DropoutLayer(2 * hidden_dim, 0.1)
# Highway gate layer T in the Highway formula
self.gate_layer = nn.Linear(self.in_dim, self.hidden_dim * 2)
# self.dropout_layer = nn.Dropout(0.1)
self.reset_parameters()
def __init__(self,
input_dim,
hidden_dim,
dropout=None,
with_batch=True,
name='DeepLSTM'):
"""
Initialize neural network.
"""
if type(hidden_dim) is int:
hidden_dim = [hidden_dim]
assert type(hidden_dim) is list and len(hidden_dim) >= 1
assert dropout is None or type(dropout) is float
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.dropout = dropout
self.with_batch = with_batch
self.name = name
# Create all LSTMs
input_dims = [input_dim] + hidden_dim[:-1]
self.lstms = [
LSTM(a, b, with_batch=with_batch, name='%s_%i' % (name, i))
for i, (a, b) in enumerate(zip(input_dims, hidden_dim))
]
# Create dropout layers
if dropout is not None:
self.dropout_layers = [
DropoutLayer(p=dropout) for _ in xrange(len(hidden_dim))
]