def forward(self, # pylint: disable=arguments-differ
sequence_tensor: PackedSequence,
initial_state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None) :
if not isinstance(inputs, PackedSequence):
raise ConfigurationError('inputs must be PackedSequence but got %s' % (type(inputs)))
sequence_tensor, batch_lengths = pad_packed_sequence(inputs, batch_first=True)
batch_size = sequence_tensor.size()[0]
length = sequence_tensor.size()[1]
hidden = torch.tensor([])
cell = torch.tensor([])
#初始化所有层隐藏状态
if initial_state is None:
hidden = sequence_tensor.new_zeros(self.num_layers, batch_size, self.hidden_size)
cell = sequence_tensor.new_zeros(self.num_layers, batch_size, self.n_chunk, self.chunk_size)
else:
hidden = initial_state[0].squeeze(0)
cell = initial_state[1].squeeze(0)
if self.training:
for c in self.cells:
c.sample_masks()
final_hidden = []
final_cell = []
for l in range(len(self.cells)):
curr_layer = [None] * length
t_input = self.cells[l].ih(sequence_tensor)
hx = hidden[l].squeeze(0)
cx = cell[l].squeeze(0)
for t in range(length):
hidden, cell = self.cells[l](None, hx, cx, transformed_input=t_input[:, t])
# length, dim
hx, cx = hidden, cell # overwritten every timestep
curr_layer[t] = hidden
final_hidden.append(hx)
final_cell.append(cx)
# batch, length, dim
sequence_tensor = torch.stack(curr_layer, dim=1)
if l < len(self.cells) - 1:
sequence_tensor = self.lockdrop(sequence_tensor, self.dropout) #每一层LSTM后加lockdrop
output = pack_padded_sequence(sequence_tensor, batch_lengths, batch_first=True)
final_state = (torch.stack(final_hidden), torch.stack(final_cell))
return sequence_tensor, final_state
def forward(self, x, h=None):
out, h = self.lstm_forward(x, h)
if isinstance(out, PackedSequence):
data = out.data
out = PackedSequence(self.linear(data), out.batch_sizes)
else:
# Reshape output to (batch_size*sequence_length, hidden_size)
# this block works as intended whether batch_first is True or False, though the var names are deceptive
batch, seq_len = out.size(0), out.size(1)
data = out.contiguous().view(batch * seq_len, self.hidden_size)
# Decode hidden states of all time steps
out = self.linear(data).view(batch, seq_len, self.vocab_size)
return out
def forward(self, x):
if self.lm:
h = self.embed(x)
else:
if type(x) is PackedSequence:
h = self.embed(x.data)
h = PackedSequence(h, x.batch_sizes)
else:
h = self.embed(x)
h,_ = self.rnn(h)
if type(h) is PackedSequence:
h = h.data
h = self.dropout(h)
z = self.proj(h)
z = PackedSequence(z, x.batch_sizes)
else:
h = h.view(-1, h.size(2))
h = self.dropout(h)
z = self.proj(h)
z = z.view(x.size(0), x.size(1), -1)
return z
def forward(
self,
inputs: PackedSequence, # pylint: disable=arguments-differ
# pylint: disable=unused-argument
initial_state: torch.Tensor = None
) -> Tuple[PackedSequence, torch.Tensor]:
"""
Parameters
----------
inputs : ``PackedSequence``, required.
A batch first ``PackedSequence`` to run the stacked LSTM over.
initial_state : Tuple[torch.Tensor, torch.Tensor], optional, (default = None)
Currently, this is ignored.
Returns
-------
output_sequence : ``PackedSequence``
The encoded sequence of shape (batch_size, sequence_length, hidden_size)
final_states: ``torch.Tensor``
The per-layer final (state, memory) states of the LSTM, each with shape
(num_layers, batch_size, hidden_size).
"""
inputs, lengths = pad_packed_sequence(inputs, batch_first=True)
# Kernel takes sequence length first tensors.
inputs = inputs.transpose(0, 1)
sequence_length, batch_size, _ = inputs.size()
accumulator_shape = [
self.num_layers, sequence_length + 1, batch_size, self.hidden_size
]
state_accumulator = inputs.new_zeros(*accumulator_shape)
memory_accumulator = inputs.new_zeros(*accumulator_shape)
dropout_weights = inputs.new_ones(self.num_layers, batch_size,
self.hidden_size)
if self.training:
# Normalize by 1 - dropout_prob to preserve the output statistics of the layer.
dropout_weights.bernoulli_(1 - self.recurrent_dropout_probability)\
.div_((1 - self.recurrent_dropout_probability))
gates = inputs.new_tensor((self.num_layers, sequence_length,
batch_size, 6 * self.hidden_size))
lengths_variable = torch.LongTensor(lengths)
implementation = _AlternatingHighwayLSTMFunction(
self.input_size,
self.hidden_size,
num_layers=self.num_layers,
train=self.training)
output, _ = implementation(inputs, self.weight, self.bias,
state_accumulator, memory_accumulator,
dropout_weights, lengths_variable, gates)
# TODO(Mark): Also return the state here by using index_select with the lengths so we can use
# it as a Seq2VecEncoder.
output = output.transpose(0, 1)
output = pack_padded_sequence(output, lengths, batch_first=True)
return output, None
def forward(self, packed_x: PackedSequence):
"""
forward expects a PackedSequence as input.
"""
num_chunks = max(1, packed_x.size(0) // self.config.chunk_size)
chunks = torch.chunk(packed_x, num_chunks, 0)
out = [self.process(chunk) for chunk in chunks]
encoded = torch.cat(out, 0)
return encoded
def forward(self, inputs: PackedSequence, # pylint: disable=arguments-differ
# pylint: disable=unused-argument
initial_state: torch.Tensor = None)-> Tuple[PackedSequence, torch.Tensor]:
"""
Parameters
----------
inputs : ``PackedSequence``, required.
A batch first ``PackedSequence`` to run the stacked LSTM over.
initial_state : Tuple[torch.Tensor, torch.Tensor], optional, (default = None)
Currently, this is ignored.
Returns
-------
output_sequence : ``PackedSequence``
The encoded sequence of shape (batch_size, sequence_length, hidden_size)
final_states: ``torch.Tensor``
The per-layer final (state, memory) states of the LSTM, each with shape
(num_layers, batch_size, hidden_size).
"""
inputs, lengths = pad_packed_sequence(inputs, batch_first=True)
# Kernel takes sequence length first tensors.
inputs = inputs.transpose(0, 1)
sequence_length, batch_size, _ = inputs.size()
accumulator_shape = [self.num_layers, sequence_length + 1, batch_size, self.hidden_size]
state_accumulator = Variable(inputs.data.new(*accumulator_shape).zero_(), requires_grad=False)
memory_accumulator = Variable(inputs.data.new(*accumulator_shape).zero_(), requires_grad=False)
dropout_weights = inputs.data.new().resize_(self.num_layers, batch_size, self.hidden_size).fill_(1.0)
if self.training:
# Normalize by 1 - dropout_prob to preserve the output statistics of the layer.
dropout_weights.bernoulli_(1 - self.recurrent_dropout_probability)\
.div_((1 - self.recurrent_dropout_probability))
dropout_weights = Variable(dropout_weights, requires_grad=False)
gates = Variable(inputs.data.new().resize_(self.num_layers,
sequence_length,
batch_size, 6 * self.hidden_size))
lengths_variable = Variable(torch.IntTensor(lengths))
implementation = _AlternatingHighwayLSTMFunction(self.input_size,
self.hidden_size,
num_layers=self.num_layers,
train=self.training)
output, _ = implementation(inputs, self.weight, self.bias, state_accumulator,
memory_accumulator, dropout_weights, lengths_variable, gates)
# TODO(Mark): Also return the state here by using index_select with the lengths so we can use
# it as a Seq2VecEncoder.
output = output.transpose(0, 1)
output = pack_padded_sequence(output, lengths, batch_first=True)
return output, None
def forward(self, state, length):
# length should be sorted
assert len(state.size()) == 3 # batch x n_features x input_dim
# input_dim == n_features + 1
batch_size = state.size()[0]
self.weight = np.zeros((int(batch_size), self.n_features))#state.data.new(int(batch_size), self.n_features).fill_(0.)
nonzero = torch.sum(length > 0).cpu().numpy() # encode only nonzero points
if nonzero == 0:
return state.new(int(batch_size), self.lstm_size + self.embedded_dim).fill_(0.)
length_ = list(length[:nonzero].cpu().numpy())
packed = pack(state[:nonzero], length_, batch_first=True)
embedded = self.embedder(packed.data)
if self.normalize:
embedded = F.normalize(embedded, dim=1)
embedded = PackedSequence(embedded, packed.batch_sizes)
embedded, _ = pad(embedded, batch_first=True) # nonzero x max(length) x embedded_dim
# define initial state
qt = embedded.new(embedded.size()[0], self.lstm_size).fill_(0.)
ct = embedded.new(embedded.size()[0], self.lstm_size).fill_(0.)
###########################
# shuffling (set encoding)
###########################
for i in range(self.n_shuffle):
attended, weight = self.attending(qt, embedded, length[:nonzero])
# attended : nonzero x embedded_dim
qt, ct = self.lstm(attended, (qt, ct))
# TODO edit here!
weight = weight.detach().cpu().numpy()
tmp = state[:, :, 1:]
val, acq = torch.max(tmp, 2) # batch x n_features
tmp = (val.long() * acq).cpu().numpy()
#tmp = tmp.cpu().numpy()
tmp = tmp[:weight.shape[0], :weight.shape[1]]
self.weight[np.arange(nonzero).reshape(-1, 1), tmp] = weight
encoded = torch.cat((attended, qt), dim=1)
if batch_size > nonzero:
encoded = torch.cat(
(encoded,
encoded.new(int(batch_size - nonzero),
encoded.size()[1]).fill_(0.)),
dim=0
)
return encoded
def forward(self, x):
# x's are already flanked by the star/stop token as:
# [stop, x, stop]
z_fwd, z_rvs = self.embed_and_split(x, pad=False)
h_fwd, h_rvs = self.transform(z_fwd, z_rvs, last_only=True)
packed = type(z_fwd) is PackedSequence
if packed:
h_flat = h_fwd.data
logp_fwd = self.linear(h_flat)
logp_fwd = PackedSequence(logp_fwd, h_fwd.batch_sizes)
h_flat = h_rvs.data
logp_rvs = self.linear(h_flat)
logp_rvs = PackedSequence(logp_rvs, h_rvs.batch_sizes)
logp_fwd, batch_sizes = pad_packed_sequence(logp_fwd,
batch_first=True)
logp_rvs, batch_sizes = pad_packed_sequence(logp_rvs,
batch_first=True)
else:
b = h_fwd.size(0)
n = h_fwd.size(1)
h_flat = h_fwd.contiguous().view(-1, h_fwd.size(2))
logp_fwd = self.linear(h_flat)
logp_fwd = logp_fwd.view(b, n, -1)
h_flat = h_rvs.contiguous().view(-1, h_rvs.size(2))
logp_rvs = self.linear(h_flat)
logp_rvs = logp_rvs.view(b, n, -1)
# prepend forward logp with zero
# postpend reverse logp with zero
b = h_fwd.size(0)
zero = h_fwd.data.new(b, 1, logp_fwd.size(2)).zero_()
logp_fwd = torch.cat([zero, logp_fwd], 1)
logp_rvs = torch.cat([logp_rvs, zero], 1)
logp = F.log_softmax(logp_fwd + logp_rvs, dim=2)
if packed:
batch_sizes = [s + 1 for s in batch_sizes]
logp = pack_padded_sequence(logp, batch_sizes, batch_first=True)
return logp
def forward(self, input: PackedSequence):
input, batch_sizes = input
seq_len = batch_sizes.size()[0]
max_batch_size = batch_sizes[0]
output = input.new_zeros(input.size(0), self.hidden_size)
hidden_state = input.new_zeros(max_batch_size, self.hidden_size)
cell_state = input.new_zeros(max_batch_size, self.hidden_size)
recurrent_mask = get_dropout_mask(
self.recurrent_dropout_prob,
hidden_state) if self.training else None
cumsum_sizes = torch.cumsum(batch_sizes, dim=0)
for timestep in range(seq_len):
timestep = timestep if self.go_forward else seq_len - timestep - 1
len_t = batch_sizes[timestep]
begin, end = (cumsum_sizes[timestep] - len_t,
cumsum_sizes[timestep])
input_t = input[begin:end]
hidden_t, cell_t = self.cell(
input_t, (hidden_state[0:len_t], cell_state[0:len_t]))
if self.training:
hidden_t = hidden_t * recurrent_mask[:len_t]
output[begin:end] = hidden_t
hidden_state = hidden_state.clone()
cell_state = cell_state.clone()
hidden_state[0:batch_sizes[timestep]] = hidden_t
cell_state[0:batch_sizes[timestep]] = cell_t
return PackedSequence(output, batch_sizes), (hidden_state, cell_state)