def __init__(self,
enc_inp_size,
dec_inp_size,
dec_out_size,
N=6,
d_model=512,
d_ff=2048,
heads=8,
dropout=0.1,
mean=[0, 0],
std=[0, 0]):
super(IndividualTF, self).__init__()
"Helper: Construct a model from hyperparameters."
c = copy.deepcopy
attn = MultiHeadAttention(heads, d_model)
ff = PointerwiseFeedforward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
self.mean = np.array(mean)
self.std = np.array(std)
self.model = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout),
N),
nn.Sequential(LinearEmbedding(enc_inp_size, d_model), c(position)),
nn.Sequential(LinearEmbedding(dec_inp_size, d_model), c(position)),
Generator(d_model, dec_out_size))
# This was important from their code.
# Initialize parameters with Glorot / fan_avg.
for p in self.model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def __init__(self,
enc_inp_size,
d_latent,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1,
device='cpu',
d_map_latent=8):
super(EncoderY, self).__init__()
self.d_model = d_model
self.embed_fn = nn.Sequential(
LinearEmbedding(enc_inp_size, d_model - d_map_latent),
PositionalEncoding(d_model - d_map_latent, dropout))
self.encoder = Encoder(
EncoderLayer(d_model, MultiHeadAttention(h, d_model),
PointerwiseFeedforward(d_model, d_ff, dropout),
dropout), N)
self.fc = nn.Linear(d_model, d_latent)
self.init_weights(self.encoder.parameters())
self.init_weights(self.fc.parameters())
self.map_encoder = load_map_encoder(device)
def __init__(self,
src_len,
tgt_len,
enc_inp_size,
dec_inp_size,
dec_out_size,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1,
device='cpu'):
super(Generator, self).__init__()
self.device = device
self.src_len = src_len
self.tgt_len = tgt_len
self.dec_inp_size = dec_inp_size
c = copy.deepcopy
attn = MultiHeadAttention(h, d_model)
ff = PointerwiseFeedforward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
self.generator = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout),
N),
nn.Sequential(LinearEmbedding(enc_inp_size, d_model), c(position)),
nn.Sequential(LinearEmbedding(dec_inp_size, d_model), c(position)),
TFHeadGenerator(d_model, dec_out_size))
# This was important from their code.
# Initialize parameters with Glorot / fan_avg.
for p in self.generator.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def __init__(self,
disc_inp_size,
disc_seq_len,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1,
device='cpu'):
super(Critic, self).__init__()
self.device = device
c = copy.deepcopy
attn = MultiHeadAttention(h, d_model)
ff = PointerwiseFeedforward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
self.critic = nn.ModuleDict({
'src_embed':
nn.Sequential(LinearEmbedding(disc_inp_size, d_model),
c(position)),
'encoder':
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
'disc_head':
nn.Sequential(nn.Flatten(), nn.Linear(d_model * disc_seq_len, 1)),
})
for p in self.critic.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def __init__(self, embedding, vocab_size, num_hiddens, ffn_num_hiddens,
num_heads, num_layers, dropout, **kwargs):
super(TransformerDecoder, self).__init__(**kwargs)
self.num_hiddens = num_hiddens
self.num_layers = num_layers
self.embedding = embedding
self.pos_encoding = PositionalEncoding(num_hiddens, dropout)
self.blks = nn.ModuleList([])
for i in range(num_layers):
self.blks.append(
DecoderBlock(self.embedding.get_output_dim(), num_hiddens,
ffn_num_hiddens, num_heads, dropout, i))
self.dense = nn.Linear(num_hiddens, vocab_size)
def make_model(src_vocab_size, tgt_vocab_size, N=6,
d_model=512, d_ff=2048, h=8, dropout=0.1):
c = copy.deepcopy
multi_head_attn = MultiHeadedAttention(h, d_model)
ff = PositionWiseFeedForward(d_model, d_ff, dropout)
pe = PositionalEncoding(d_model, dropout)
model = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(multi_head_attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(multi_head_attn), c(multi_head_attn), c(ff), dropout), N),
nn.Sequential(Embeddings(d_model, src_vocab_size), c(pe)),
nn.Sequential(Embeddings(d_model, tgt_vocab_size), c(pe)),
Generator(d_model, tgt_vocab_size)
)
for p in model.parameters():
if p.dim() > 1:
# nn.init.xavier_uniform(p)
nn.init.xavier_uniform_(p)
return model
def setUp(self):
self.pos_enc = PositionalEncoding(20, 0)
self.input = torch.zeros(1, 100, 20) # pylint: disable=no-member