def __init__(self, device, vocab_dir):
super(UnbiasedTopicalExtractorGloVe, self).__init__()
self.device = device
vocab = Vocabulary()
self.vocab = vocab.from_files(vocab_dir)
self.embedding = Embedding(
embedding_dim=300,
trainable=True,
num_embeddings=self.vocab.get_vocab_size("tokens"),
pretrained_file=
"https://allennlp.s3.amazonaws.com/datasets/glove/glove.840B.300d.txt.gz"
)
self.tokenizer = WordTokenizer()
self.token_indexer = SingleIdTokenIndexer(lowercase_tokens=True)
self.rnn_input_dropout = InputVariationalDropout(0.5)
self.encoder = _Seq2SeqWrapper(torch.nn.LSTM)(input_size=300,
hidden_size=300,
num_layers=1,
bidirectional=True)
self._feature_feedforward_layer = torch.nn.Linear(600, 300)
self._feature_feedforward_dropout = torch.nn.Dropout(0.5)
self._feature_feedforward_activation = torch.nn.ReLU()
self._class_classification_layer = torch.nn.Linear(300, 2)
self._group_classification_layer = torch.nn.Linear(300, 2)
init.xavier_uniform_(self._feature_feedforward_layer.weight)
init.zeros_(self._feature_feedforward_layer.bias)
init.xavier_uniform_(self._class_classification_layer.weight)
init.zeros_(self._class_classification_layer.bias)
init.xavier_uniform_(self._group_classification_layer.weight)
init.zeros_(self._group_classification_layer.bias)
self._class_loss = torch.nn.CrossEntropyLoss()
self._group_loss = torch.nn.CrossEntropyLoss()
def __init__(self, device):
super(UnbiasedTopicalExtractorELMo, self).__init__()
self.device = device
self.tokenizer = WordTokenizer()
options_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
self.elmo_model = Elmo(options_file,
weight_file,
1,
do_layer_norm=False,
dropout=0.5)
self.rnn_input_dropout = InputVariationalDropout(0.0)
self.encoder = _Seq2SeqWrapper(torch.nn.LSTM)(input_size=1024,
hidden_size=300,
num_layers=1,
bidirectional=True)
self._feature_feedforward_layer = torch.nn.Linear(600, 300)
self._feature_feedforward_dropout = torch.nn.Dropout(0.5)
self._feature_feedforward_activation = torch.nn.ReLU()
self._class_classification_layer = torch.nn.Linear(300, 2)
self._group_classification_layer = torch.nn.Linear(300, 2)
init.xavier_uniform_(self._feature_feedforward_layer.weight)
init.zeros_(self._feature_feedforward_layer.bias)
init.xavier_uniform_(self._class_classification_layer.weight)
init.zeros_(self._class_classification_layer.bias)
init.xavier_uniform_(self._group_classification_layer.weight)
init.zeros_(self._group_classification_layer.bias)
self._class_loss = torch.nn.CrossEntropyLoss()
self._group_loss = torch.nn.CrossEntropyLoss()
The encoded sequence of shape (batch_size, sequence_length, hidden_size)
final_states: Tuple[torch.Tensor, torch.Tensor]
The per-layer final (state, memory) states of the LSTM, each with shape
(num_layers, batch_size, hidden_size).
"""
if not initial_state:
hidden_states = [None] * len(self.lstm_layers)
elif initial_state[0].size()[0] != len(self.lstm_layers):
raise ConfigurationError(
"Initial states were passed to forward() but the number of "
"initial states does not match the number of layers.")
else:
hidden_states = list(
zip(initial_state[0].split(1, 0), initial_state[1].split(1,
0)))
output_sequence = inputs
final_states = []
for i, state in enumerate(hidden_states):
layer = getattr(self, 'layer_{}'.format(i))
# The state is duplicated to mirror the Pytorch API for LSTMs.
output_sequence, final_state = layer(output_sequence, state)
final_states.append(final_state)
final_hidden_state, final_cell_state = tuple(
torch.cat(state_list, 0) for state_list in zip(*final_states))
return output_sequence, (final_hidden_state, final_cell_state)
Seq2SeqEncoder.register("stacked_lstm")(_Seq2SeqWrapper(StackedLstm))
output, final_state = layer(output_sequence, state)
output_sequence, lengths = pad_packed_sequence(output,
batch_first=True)
# Apply layer wise dropout on each output sequence apart from the
# first (input) and last
if i < (self.num_layers - 1):
output_sequence = self.layer_dropout(output_sequence)
if self.use_residual:
res_proj = getattr(self, 'res_proj_{}'.format(i))
tmp = output_sequence
output_sequence = output_sequence + res_proj(prev_sequence)
prev_sequence = tmp
output_sequence = pack_padded_sequence(output_sequence,
lengths,
batch_first=True)
final_h.append(final_state[0])
final_c.append(final_state[1])
final_h = torch.cat(final_h, dim=0)
final_c = torch.cat(final_c, dim=0)
final_state_tuple = (final_h, final_c)
return output_sequence, final_state_tuple
Seq2SeqEncoder.register("residual_bidirectional_lstm")(
_Seq2SeqWrapper(ResidualBidirectionalLstm))
final_states: Tuple[torch.Tensor, torch.Tensor]
The per-layer final (state, memory) states of the LSTM, each with shape
(num_layers, batch_size, hidden_size).
"""
def init_hidden(tensor, shape):
return (tensor.new_zeros(shape),
tensor.new_zeros(shape))
sequence_tensor, batch_lengths = pad_packed_sequence(inputs, batch_first=True)
batch_size, _, _ = sequence_tensor.size()
hidden_shape = torch.Size((batch_size, self.hidden_channel * self.hidden_size)) if self.conv \
else torch.Size((batch_size, self.hidden_size))
if not initial_state:
hidden_states = [[init_hidden(sequence_tensor, hidden_shape) for _ in range(2)] if self.bidirectional
else init_hidden(sequence_tensor, hidden_shape)
for _ in range(self.num_layers)]
elif initial_state[0].size()[0] != self.num_layers:
raise ConfigurationError("Initial states were passed to forward() but the number of "
"initial states does not match the number of layers.")
else:
hidden_states = list(zip(initial_state[0].split(1, 0),
initial_state[1].split(1, 0)))
outputs, states = self.rnn(sequence_tensor.transpose(1, 0), hidden_states)
outputs = pack_padded_sequence(outputs, batch_lengths)
return outputs, states
Seq2SeqEncoder.register("stacked_custom_lstm")(_Seq2SeqWrapper(StackedCustomLstm))