def __init__(self, embeddings,
embeddings_dropout,
is_gru,
cell_hidden_size,
stacked_layers,
bidirectional,
att_mlp_layers,
att_mlp_dropout,
top_mlp_layers,
top_mlp_activation,
top_mlp_outer_activation,
targets,
top_mlp_dropout):
super(MultiAttentionRNN, self).__init__()
self.name = "MultiAttentionRNN"
self.word_embedding_layer = PretrainedEmbeddingLayer(embeddings, dropout=embeddings_dropout, trainable=False)
self.cell = CellLayer(is_gru, self.word_embedding_layer.get_output_size(),
cell_hidden_size, bidirectional, stacked_layers)
large_size = cell_hidden_size * 2 if bidirectional else cell_hidden_size
decision_layers = [MLP(num_of_layers=top_mlp_layers,
init_size=large_size,
out_size=1,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(top_mlp_activation),
outer_activation=ActivationFactory.get_activation(top_mlp_outer_activation)) for _ in range(targets)]
self.decision_layers = ModuleList(decision_layers)
self.attentions = ModuleList(
[AttendedState(att_mlp_layers, large_size, att_mlp_dropout, relu) for _ in range(targets)])
self.seq = SequentialModel([self.word_embedding_layer, self.cell])
self.trainable_weights = list(filter(lambda p: p.requires_grad, self.parameters()))
def __init__(self, embeddings,
embeddings_dropout=0.0,
is_gru=True,
cell_hidden_size=128,
stacked_layers=1,
bidirectional=False,
top_mlp_layers=2,
top_mlp_activation="relu",
top_mlp_outer_activation=None, targets=11,
top_mlp_dropout=0.0):
super(LastStateRNN, self).__init__()
self.name = "LastStateRNN"
self.word_embedding_layer = PretrainedEmbeddingLayer(embeddings, dropout=embeddings_dropout, trainable=False)
self.cell = CellLayer(is_gru, self.word_embedding_layer.get_output_size(),
cell_hidden_size, bidirectional, stacked_layers)
self.decision_layer = MLP(num_of_layers=top_mlp_layers,
init_size=self.cell.get_output_size(),
out_size=targets,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(top_mlp_activation),
outer_activation=ActivationFactory.get_activation(top_mlp_outer_activation))
self.state = LastState(self.cell.get_output_size())
self.seq = SequentialModel([self.word_embedding_layer, self.cell, self.state, self.decision_layer])
self.trainable_weights = list(filter(lambda p: p.requires_grad, self.parameters()))
def __init__(self, embeddings, embeddings_dropout, att_mlp_layers,
att_mlp_dropout, top_mlp_layers, top_mlp_activation,
top_mlp_outer_activation, targets, top_mlp_dropout):
super(MLPWithAttentionModel, self).__init__()
self.name = "MLPWithAttentionModel"
self.word_embedding_layer = PretrainedEmbeddingLayer(
embeddings, dropout=embeddings_dropout, trainable=False)
fs_size = self.word_embedding_layer.get_output_size()
decision_layers = [
MLP(num_of_layers=top_mlp_layers,
init_size=fs_size,
out_size=1,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(
top_mlp_activation),
outer_activation=ActivationFactory.get_activation(
top_mlp_outer_activation)) for _ in range(targets)
]
self.decision_layers = ModuleList(decision_layers)
attentions = [
AttendedState(att_mlp_layers, fs_size, att_mlp_dropout, relu)
for _ in range(targets)
]
self.attentions = ModuleList(attentions)
self.trainable_weights = list(
filter(lambda p: p.requires_grad, self.parameters()))
def __init__(self, embeddings, embeddings_dropout, filters, min_kernel,
max_kernel, top_mlp_layers, top_mlp_activation,
top_mlp_outer_activation, targets, top_mlp_dropout):
super(CNN, self).__init__()
self.name = "CNN"
self.word_embedding_layer = PretrainedEmbeddingLayer(
embeddings, dropout=embeddings_dropout, trainable=False)
conv_blocks = [
ConvBlock(in_channels=self.word_embedding_layer.get_output_size(),
filters=filters,
window=i) for i in range(min_kernel, max_kernel + 1)
]
self.conv_blocks = ModuleList(conv_blocks)
concat_size = len(conv_blocks) * filters
self.decision_layer = MLP(
num_of_layers=top_mlp_layers,
init_size=concat_size,
out_size=targets,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(
top_mlp_activation),
outer_activation=ActivationFactory.get_activation(
top_mlp_outer_activation))
self.trainable_weights = list(
filter(lambda p: p.requires_grad, self.parameters()))
class MultiAttentionRNNConcatCNN(Module):
def __init__(self, embeddings, embeddings_dropout, is_gru, filters,
min_kernel, max_kernel, cell_hidden_size, stacked_layers,
bidirectional, att_mlp_layers, att_mlp_dropout,
top_mlp_layers, top_mlp_activation, top_mlp_outer_activation,
targets, top_mlp_dropout):
super(MultiAttentionRNNConcatCNN, self).__init__()
self.name = "MultiAttentionRNNConcatCNN"
self.targets = targets
self.word_embedding_layer = PretrainedEmbeddingLayer(
embeddings, dropout=embeddings_dropout, trainable=False)
conv_blocks = [
ConvBlock(in_channels=self.word_embedding_layer.get_output_size(),
filters=filters,
window=i) for i in range(min_kernel, max_kernel + 1)
]
self.conv_blocks = ModuleList(conv_blocks)
self.cell = CellLayer(is_gru,
self.word_embedding_layer.get_output_size(),
cell_hidden_size, bidirectional, stacked_layers)
concat_size = self.cell.get_output_size() + filters * len(conv_blocks)
decision_layers = [
MLP(num_of_layers=top_mlp_layers,
init_size=concat_size,
out_size=1,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(
top_mlp_activation),
outer_activation=ActivationFactory.get_activation(
top_mlp_outer_activation)) for _ in range(targets)
]
self.decision_layers = ModuleList(decision_layers)
attentions = [
AttendedState(att_mlp_layers, self.cell.get_output_size(),
att_mlp_dropout, relu) for _ in range(targets)
]
self.attentions = ModuleList(attentions)
self.trainable_weights = list(
filter(lambda p: p.requires_grad, self.parameters()))
def forward(self, x):
embeddings = self.word_embedding_layer(x)
n_grams = [block(embeddings) for block in self.conv_blocks]
n_grams_concat = torch.cat(n_grams, dim=1)
sequence_encoder = self.cell(embeddings)
states = [attention(sequence_encoder) for attention in self.attentions]
states = [
torch.cat((state, n_grams_concat), dim=1) for state in states
]
decisions = [
decision(state)
for decision, state in zip(self.decision_layers, states)
]
out = torch.cat(decisions, dim=1)
return out
def __init__(self, embeddings, embeddings_dropout, is_gru, filters,
min_kernel, max_kernel, cell_hidden_size, stacked_layers,
bidirectional, att_mlp_layers, att_mlp_dropout,
top_mlp_layers, top_mlp_activation, top_mlp_outer_activation,
targets, top_mlp_dropout):
super(MultiAttentionRNNConcatCNN, self).__init__()
self.name = "MultiAttentionRNNConcatCNN"
self.targets = targets
self.word_embedding_layer = PretrainedEmbeddingLayer(
embeddings, dropout=embeddings_dropout, trainable=False)
conv_blocks = [
ConvBlock(in_channels=self.word_embedding_layer.get_output_size(),
filters=filters,
window=i) for i in range(min_kernel, max_kernel + 1)
]
self.conv_blocks = ModuleList(conv_blocks)
self.cell = CellLayer(is_gru,
self.word_embedding_layer.get_output_size(),
cell_hidden_size, bidirectional, stacked_layers)
concat_size = self.cell.get_output_size() + filters * len(conv_blocks)
decision_layers = [
MLP(num_of_layers=top_mlp_layers,
init_size=concat_size,
out_size=1,
dropout=top_mlp_dropout,
inner_activation=ActivationFactory.get_activation(
top_mlp_activation),
outer_activation=ActivationFactory.get_activation(
top_mlp_outer_activation)) for _ in range(targets)
]
self.decision_layers = ModuleList(decision_layers)
attentions = [
AttendedState(att_mlp_layers, self.cell.get_output_size(),
att_mlp_dropout, relu) for _ in range(targets)
]
self.attentions = ModuleList(attentions)
self.trainable_weights = list(
filter(lambda p: p.requires_grad, self.parameters()))