def __init__(self,
input_size,
output_size,
hidden_size,
n_layers,
word_vector=None,
bidirectional=False,
fine_tuned=False,
datasetType=None):
super(FTIA, self).__init__()
self.hidden_size = hidden_size
self.n_layers = n_layers
self.n_directions = int(bidirectional) + 1
self.output_size = output_size
self.word_embedding = nn.Embedding(input_size, hidden_size)
if word_vector is not None:
self.word_embedding.weight = nn.Parameter(word_vector,
requires_grad=fine_tuned)
self.label_embedding = nn.Embedding(output_size,
hidden_size * self.n_directions)
self.label_embedding.weight = nn.Parameter(create_variable(
torch.randn(output_size, hidden_size * self.n_directions)),
requires_grad=True)
self.mp = nn.MaxPool2d((1, self.output_size))
self.lstm = nn.LSTM(hidden_size,
hidden_size,
n_layers,
bidirectional=bidirectional)
self.attention = nn.Linear(hidden_size, hidden_size)
self.fc = nn.Linear(hidden_size * self.n_directions, output_size)
self.datasetType = datasetType
def forward(self, input, seq_lengths, labels):
#preprocess label embedding
labels_cuda = create_variable(
torch.LongTensor([range(self.output_size) for label in labels]))
input = input.t()
self.batch_size = input.size(1)
cell = self._init_hidden(self.batch_size)
hidden = self._init_hidden(self.batch_size)
# word embedding
#print input
word_embedded = self.word_embedding(input)
# label embedding
label_embedded = self.label_embedding(labels_cuda)
self.lstm.flatten_parameters()
# to compact weights again call flatten paramters
output, (final_hidden_state,
final_cell_state) = self.lstm(word_embedded, (hidden, cell))
# attention layers
att_output, weights = self.label_attention(output, label_embedded)
#save text representations
saveTextRepresentations(self.datasetType, 'FTIA',
att_output.detach().cpu().numpy())
#penalty
penalty = self.calc_penalty(weights)
return self.fc(att_output), penalty
def pad_sequences(vectorized_seqs, seq_lengths, labels, global_same_length):
#CNN requires same seq length
if global_same_length:
seq_tensor = torch.zeros(
(len(vectorized_seqs), hyperparams['global_max_seq_len'])).long()
else:
seq_tensor = torch.zeros(
(len(vectorized_seqs), seq_lengths.max())).long()
for idx, (seq, seq_len) in enumerate(zip(vectorized_seqs, seq_lengths)):
seq_tensor[idx, :seq_len] = torch.LongTensor(seq)
#sort tensors by their length
seq_lengths, perm_idx = seq_lengths.sort(0, descending=True)
seq_tensor = seq_tensor[perm_idx]
#also sort label in the same order
labels = torch.LongTensor(labels)[perm_idx]
#return variables
#data,parallel requires everything to be a variable
return create_variable(seq_tensor), create_variable(
seq_lengths), create_variable(labels)
def _init_hidden(self, batch_size):
hidden = torch.zeros(self.n_layers * self.n_directions, batch_size,
self.hidden_size)
return create_variable(hidden)