def build_encoder_layer(params): p = HyperParams() p.d_f = params.filter p.n_I = params.n_heads p.d_x = params.hidden p.d_v = p.d_x // p.n_I p.d_r = p.d_x // p.n_I p.d_k = p.d_x // p.n_I p.d_q = p.d_x // p.n_I p.dropout = params.dropout update_vis_params(params,p) encoderlayer = EncoderLayer(p) return encoderlayer
def build_transformer(params, pad_idx):
p = HyperParams()
p.d_vocab = params.input_dim
p.d_pos = 200 # max input size
p.d_f = params.filter
p.n_L = params.n_layers
p.n_I = params.n_heads
p.d_x = params.hidden # token embedding dimension
p.d_p = params.hidden # position embedding dimension
p.d_v = p.d_x // p.n_I # value dimension
p.d_r = p.d_x // p.n_I # role dimension
p.d_k = p.d_x // p.n_I # key dimension
p.d_q = p.d_x // p.n_I # query dimension
p.dropout = params.dropout
update_vis_params(params,p)
embedding = EmbeddingMultilinearSinusoidal(d_vocab=params.input_dim,
d_x=p.d_x,
d_r=p.d_r,
dropout=params.dropout,
max_length=200)
encoder = Encoder(p=p)
decoder = Decoder(p=p)
model = Seq2Seq(p=p,
embedding=embedding,
encoder=encoder,
decoder=decoder,
pad_idx=pad_idx)
return model