def __init__(self, config, src_vocab, target_vocab, s_v, t_v, u):
super(Transformer, self).__init__()
self.config = config
h, N, dropout = self.config.h, self.config.N, self.config.dropout
d_model, d_ff = self.config.d_model, self.config.d_ff
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
attncross = MultiHeadedAttention(h, d_model * 2)
ffcross = PositionwiseFeedForward(d_model * 2, d_ff, dropout)
positioncross = PositionalEncoding(d_model * 2, dropout)
self.encoder = Encoder(
EncoderLayer(config.d_model, deepcopy(attn), deepcopy(ff),
dropout), N)
self.encoder_cross = EncoderCross(
EncoderLayerCross((config.d_model) * 2, deepcopy(attncross),
deepcopy(ffcross), dropout), N)
self.src_embed = nn.Sequential(
Embeddings(config.d_model, src_vocab, s_v, u),
deepcopy(position)) # Embeddings followed by PE
# self.src_embed.weight.data.copy_(src_vocab.vectors)
self.target_embed = nn.Sequential(
Embeddings(config.d_model, target_vocab, t_v, u),
deepcopy(position))
# self.target_embed.weight.data.copy_(target_vocab.vectors)
# Fully-Connected Layer
self.fc = nn.Linear(self.config.d_model, self.config.output_size)
self.sigmoid = nn.Sigmoid()
self.cos = nn.CosineSimilarity(dim=1, eps=1e-6)
self.softmax = nn.Softmax()
def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout):
"""
:param hidden: hidden size of transformer
:param attn_heads: head sizes of multi-head attention
:param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size
:param dropout: dropout rate
"""
super().__init__()
self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden)
self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout)
self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout)
self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout)
self.dropout = nn.Dropout(p=dropout)
def __init__(self, config, src_vocab):
super(Transformer, self).__init__()
self.config = config
h, N, dropout = self.config.h, self.config.N, self.config.dropout
d_model, d_ff = self.config.d_model, self.config.d_ff
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
self.encoder = Encoder(
EncoderLayer(config.d_model, deepcopy(attn), deepcopy(ff),
dropout), N)
self.src_embed = nn.Sequential(Embeddings(config.d_model, src_vocab))
self.fc = nn.Linear(self.config.d_model, self.config.output_size)
self.softmax = nn.Softmax()
def __init__(self, config, src_vocab):
super(Matposer, self).__init__()
self.config = config
d_row, N, dropout = self.config.d_row, self.config.N, self.config.dropout
d_model, d_ff = self.config.d_model, self.config.d_ff
inter = Interactor(d_model, d_ff, out_row=d_row, dropout=dropout)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
self.encoder = Encoder(
EncoderLayer(d_model, deepcopy(inter), deepcopy(ff), dropout), N)
self.src_embed = nn.Sequential(Embeddings(d_model, src_vocab),
deepcopy(position))
self.fc = nn.Linear(d_model, self.config.output_size)
self.softmax = nn.Softmax()
def __init__(self, config):
super(Transformer, self).__init__()
self.config = config
h, N, dropout = self.config.h, self.config.N, self.config.dropout
d_model, d_ff = self.config.d_model, self.config.d_ff
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
self.encoder = Encoder(EncoderLayer(config.d_model, deepcopy(attn), deepcopy(ff), dropout), N)
# self.src_embed = nn.Sequential(Embeddings(config.d_model, src_vocab),
# deepcopy(position)) # Embeddings followed by PE
# Fully-Connected Layer
self.fc = nn.Linear(
self.config.d_model,
self.config.output_size
)
def __init__(self, config, src_vocab):
super(Transformer, self).__init__()
self.config = config
h, N, dropout = self.config.h, self.config.N, self.config.dropout
d_model, d_ff = self.config.d_model, self.config.d_ff
self.src_vocab = src_vocab
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
self.encoder_layer = EncoderLayer(config.d_model, deepcopy(attn),
deepcopy(ff), dropout)
self.encoder = Encoder(self.encoder_layer, N)
self.src_word_emb = nn.Embedding(src_vocab,
config.d_model,
padding_idx=0)
# self.pos_bias = nn.Embedding(src_vocab, config.d_model, padding_idx=0)
# self.pos_bias = nn.Embedding.from_pretrained(get_sinusoid_encoding_table_dim(src_vocab, config.d_model, padding_idx=0),freeze=True)
# self.pos_bias = nn.Embedding.from_pretrained(get_sinusoid_encoding_table_vocab(src_vocab, config.d_model, padding_idx=0),freeze=True)
# self.pos_bias = nn.Embedding(1, config.d_model, padding_idx=0)
# self.pos_bias = nn.Embedding(src_vocab, 1, padding_idx=0)
# self.position_enc = nn.Embedding(src_vocab, config.d_model, padding_idx=0)
self.position_enc = nn.Embedding.from_pretrained(
get_sinusoid_encoding_table(src_vocab,
config.d_model,
padding_idx=0),
freeze=False)
# position_enc = torch.randn(1000, config.d_model)
# position_enc = position_enc.unsqueeze(0)
# self.register_buffer('position_enc', position_enc)
self.drop = nn.Dropout(p=dropout)
self.fc = nn.Linear(self.config.d_model, self.config.output_size)
self.softmax = nn.Softmax()