def __init__(self, rnn_type,
bidirectional_encoder, num_layers,
hidden_size, attn_type=None,
dropout=0.0, embedding=None):
super(DecoderBase, self).__init__()
assert embedding is not None
# Basic attributes.
self.decoder_type = 'rnn'
self.bidirectional_encoder = bidirectional_encoder
self.num_layers = num_layers
self.hidden_size = hidden_size
self.embedding = embedding
self.dropout = nn.Dropout(dropout)
self.attn_type = attn_type
# Build the RNN.
self.rnn = rnn_factory(rnn_type,
input_size=self._input_size,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=dropout)
# Set up the standard attention.
if self.attn_type is not None:
self.attn = GlobalAttention(
self.hidden_size, attn_type=self.attn_type)
def __init__(self,
rnn_type,
bidirectional,
num_layers,
hidden_size,
dropout=0,
embedding=None,
device=None):
super(RNNEncoder, self).__init__()
assert embedding is not None
self.num_directions = 2 if bidirectional else 1
self.num_layers = num_layers
assert hidden_size % self.num_directions == 0
self.hidden_size = hidden_size // self.num_directions
self.embedding = embedding
self.rnn = rnn_factory(rnn_type,
input_size=embedding.embedding_size,
hidden_size=self.hidden_size,
num_layers=num_layers,
dropout=dropout,
bidirectional=bidirectional)
self.encoder_type = 'rnn'
self.rnn_type = rnn_type
self.device = device
def __init__(self, rnn_type, bidirectional, num_layers,
hidden_size, dropout=0.0, embeddings=None,
use_bridge=False):
super(RNNEncoder, self).__init__()
assert embeddings is not None
num_directions = 2 if bidirectional else 1
assert hidden_size % num_directions == 0
hidden_size = hidden_size // num_directions
self.embeddings = embeddings
self.rnn, self.no_pack_padded_seq = \
rnn_factory(rnn_type,
input_size=embeddings.embedding_size,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=dropout,
bidirectional=bidirectional)
# Initialize the bridge layer
self.use_bridge = use_bridge
if self.use_bridge:
self._initialize_bridge(rnn_type,
hidden_size,
num_layers)
def __init__(self, config):
# inp_size, hid_size, dropout=0, RNN_cell=nn.GRU):
super(AttentionalBiRNN, self).__init__()
self.bidirection_num = 2 if config.bidirectional else 1
self.hidden_size = config.hidden_size // self.bidirection_num
self.n_classes = config.n_classes
# rnn
self.rnn = rnn_factory(rnn_type=config.rnn_type,
input_size=self.embedding_size,
hidden_size=self.hidden_size,
num_layers=config.num_layers,
bidirectional=config.bidirectional,
dropout=config.dropout)
rnn_init(config.rnn_type, self.rnn)
self.lin = nn.Linear(self.hidden_size, self.hidden_size)
self.att_w = nn.Linear(self.hidden_size, 1, bias=False)
self.emb_att = EmbedAttention(self.hidden_size)
def __init__(self, config, embedding):
super(RCNNEncoder, self).__init__()
self.problem = config.problem
self.model_type = config.model_type
self.rnn_type = config.rnn_type
# embedding
self.embedding = embedding
self.embedding_size = embedding.embedding_dim
self.bidirection_num = 2 if config.bidirectional else 1
self.hidden_size = config.hidden_size // self.bidirection_num
self.n_classes = config.n_classes
# dropout
self.dropout = nn.Dropout(config.dropout)
# rnn
self.rnn = rnn_factory(rnn_type=config.rnn_type,
input_size=self.embedding_size,
hidden_size=self.hidden_size,
num_layers=config.num_layers,
bidirectional=config.bidirectional,
dropout=config.dropout)
rnn_init(config.rnn_type, self.rnn)
self.W2 = nn.Linear(
self.hidden_size * self.bidirection_num + self.embedding_size,
self.hidden_size)
if self.problem == 'classification':
self.linear_final = nn.Linear(self.hidden_size, self.n_classes)
else:
self.linear_regression_dense = nn.Linear(
self.hidden_size, config.regression_dense_size)
self.linear_regression_final = nn.Linear(
config.regression_dense_size, 1)
def __init__(self,
config,
embedding=None):
super(RNNEncoder, self).__init__()
self.problem = config.problem
self.model_type = config.model_type
self.rnn_type = config.rnn_type
self.max_len = config.max_len
# embedding
if embedding is not None:
self.embedding = embedding
self.embedding_size = embedding.embedding_dim
self.from_other = False
else:
self.from_other = True
self.bidirection_num = 2 if config.bidirectional else 1
self.hidden_size = config.hidden_size // self.bidirection_num
self.n_classes = config.n_classes
self.num_layers = config.num_layers
# dropout
self.dropout = nn.Dropout(config.dropout)
# rnn
self.rnn = rnn_factory(
rnn_type=self.rnn_type,
input_size=config.embedding_size,
hidden_size=self.hidden_size,
num_layers=self.num_layers,
bidirectional=config.bidirectional,
dropout=0.5 if self.num_layers > 1 else 0
# dropout=config.dropout if self.num_layers > 1 else 0
)
rnn_init(self.rnn_type, self.rnn)
self.reduce_state = ReduceState(self.rnn_type)
if self.model_type == 'rnn_bert':
self.bert = BERT(config, None)
if not self.from_other:
if self.problem == 'classification':
if self.model_type == 'rnn_cnn':
self.cnn = CNNEncoder(config)
self.linear_final = nn.Linear(
len(config.kernel_heights) * config.out_channels, config.n_classes)
elif self.model_type in ['rnn_avg_hidden', 'rnn_max_hidden']:
self.linear_final = nn.Linear(
self.max_len, self.n_classes)
else:
self.linear_final = nn.Linear(
self.hidden_size * self.bidirection_num, self.n_classes)
else:
# self.linear_regression_dense = nn.Linear(
# self.hidden_size * self.bidirection_num, config.regression_dense_size)
# self.linear_regression_final = nn.Linear(config.regression_dense_size, 1)
self.linear_regression_final = nn.Linear(
self.hidden_size * self.bidirection_num, 1)
def _build_rnn(self, rnn_type, **kwargs):
rnn, _ = rnn_factory(rnn_type, **kwargs)
return rnn
def __init__(self, config, embedding):
"""
Initializes parameters suggested in paper
Args:
batch_size : {int} batch_size used for training
hidden_size: {int} hidden dimension for lstm
dense_size : {int} hidden dimension for the dense layer
num_heads : {int} attention-hops or attention num_heads
max_len : {int} number of lstm timesteps
embedding_size : {int} embedding dimension
vocab_size : {int} size of the vocabulary
use_pretrained_embeddings: {bool} use or train your own embedding
embedding : {torch.FloatTensor} loaded pretrained embedding
type : [0,1] 0-->binary_classification 1-->multiclass classification
n_classes : {int} number of classes
Returns:
self
Raises:
Exception
"""
super(StructuredSelfAttention, self).__init__()
self.problem = config.problem
# embedding
self.embedding = embedding
self.embedding_size = embedding.embedding_dim
self.bidirection_num = 2 if config.bidirectional else 1
self.hidden_size = config.hidden_size // self.bidirection_num
self.n_classes = config.n_classes
self.num_heads = config.num_heads
# dropout
self.dropout = nn.Dropout(config.dropout)
# self.rnn = nn.LSTM(self.embedding_size, self.hidden_size, config.num_layers)
self.rnn = rnn_factory(rnn_type=config.rnn_type,
input_size=self.embedding_size,
hidden_size=self.hidden_size,
num_layers=config.num_layers,
bidirectional=config.bidirectional,
dropout=0.5 if config.num_layers > 1 else 0)
rnn_init(config.rnn_type, self.rnn)
# W_s1
self.linear_first = torch.nn.Linear(
self.hidden_size * self.bidirection_num, config.dense_size)
self.linear_first.bias.data.fill_(0)
# W_s2
self.linear_second = torch.nn.Linear(config.dense_size, self.num_heads)
self.linear_second.bias.data.fill_(0)
if self.problem == 'classification':
self.linear_final = nn.Linear(
self.hidden_size * self.bidirection_num, config.n_classes)
else:
self.linear_regression_dense = nn.Linear(
self.hidden_size * self.bidirection_num,
config.regression_dense_size)
self.linear_regression_final = nn.Linear(
config.regression_dense_size, 1)
def __init__(self, config, embedding):
super().__init__()
self.problem = config.problem
self.embedding = embedding
self.embedding_size = embedding.embedding_dim
self.model_type = config.model_type
self.n_classes = config.n_classes
self.dense_size = config.dense_size
# self.transformer_size = config.transformer_size
self.max_len = config.max_len
# self.encoder = Encoder(config, embedding)
self.use_pos = config.use_pos
self.dropout = nn.Dropout(config.dropout)
if self.use_pos:
n_position = config.max_len + 1
self.pos_embedding = nn.Embedding.from_pretrained(
get_sinusoid_encoding_table(n_position,
self.embedding_size,
padid=PAD_ID),
freeze=True)
self.layer_stack = nn.ModuleList(
[EncoderLayer(config) for _ in range(config.t_num_layers)])
if self.model_type == 'transformer':
in_feature_size = self.embedding_size * self.max_len
elif self.model_type == 'transformer_avgpool':
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_avgpool_e':
in_feature_size = self.max_len
elif self.model_type == 'transformer_mean':
# self.linear_dense = nn.Linear(
# self.embedding_size,
# self.dense_size
# )
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_max':
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_maxpool':
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_maxpool_e':
in_feature_size = self.max_len
elif self.model_type == 'transformer_maxpool_concat': # and transformer_maxpool_residual
self.W2 = nn.Linear(self.embedding_size * 2, self.embedding_size)
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_maxpool_residual':
# self.layer_norm = nn.LayerNorm(config.embedding_size)
in_feature_size = self.embedding_size
elif self.model_type == 'transformer_rnn':
self.bidirection_num = 2 if config.bidirectional else 1
self.hidden_size = self.embedding_size // self.bidirection_num
self.rnn = rnn_factory(rnn_type=config.rnn_type,
input_size=self.embedding_size,
hidden_size=self.hidden_size,
num_layers=config.num_layers,
bidirectional=config.bidirectional,
dropout=config.dropout)
in_feature_size = self.hidden_size * self.bidirection_num
elif self.model_type == 'transformer_weight':
# W_s1
self.linear_first = torch.nn.Linear(self.embedding_size,
config.dense_size)
self.linear_first.bias.data.fill_(0)
# W_s2
self.linear_second = torch.nn.Linear(config.dense_size,
config.num_heads)
self.linear_second.bias.data.fill_(0)
in_feature_size = config.max_len * config.num_heads
if self.problem == 'classification':
self.linear_final = nn.Linear(in_feature_size, config.n_classes)
# nn.init.xavier_normal_(self.linear_final.weight)
else:
self.linear_regression_dense = nn.Linear(
in_feature_size, config.regression_dense_size)
self.linear_regression_final = nn.Linear(
config.regression_dense_size, 1)