def __init__(self, emb, outlin, embdim, encdims, diadims, decdims, **kw):
super(BasicHierarchicalEncoderDecoder, self).__init__()
self.emb = emb
self.utterance_enc = q.FastestLSTMEncoder(embdim, *encdims)
self.dialog_enc = q.FastestLSTMEncoder(encdims[-1], *diadims)
self.utterance_dec = q.FastestLSTMEncoder(
embdim + diadims[-1], *decdims) # make timestep wise network
self.outlin = outlin
self.sm = torch.nn.Softmax(-1)
def test_it(self):
batsize = 3
seqlen = 4
lstm = q.FastestLSTMEncoder(8, 9, 10)
x = torch.nn.Parameter(torch.randn(batsize, seqlen, 8))
x_mask = torch.tensor([[1, 1, 1, 0], [1, 0, 0, 0], [1, 1, 0, 0]],
dtype=torch.int64)
y, states = lstm(x, mask=x_mask, ret_states=True)
l = states[-1][0][1].sum()
l.backward(retain_graph=True)
self.assertTrue(x.grad[0].norm() == 0)
self.assertTrue(x.grad[1].norm() > 0)
self.assertTrue(x.grad[2].norm() == 0)
self.assertTrue(x.grad[1][0].norm() > 0)
self.assertTrue(x.grad[1][1].norm() == 0)
self.assertTrue(x.grad[1][2].norm() == 0)
self.assertTrue(x.grad[1][3].norm() == 0)
x.grad = None
l = states[-1][0][2].sum()
l.backward(retain_graph=True)
self.assertTrue(x.grad[0].norm() == 0)
self.assertTrue(x.grad[1].norm() == 0)
self.assertTrue(x.grad[2].norm() > 0)
self.assertTrue(x.grad[2][0].norm() > 0)
self.assertTrue(x.grad[2][1].norm() > 0)
self.assertTrue(x.grad[2][2].norm() == 0)
self.assertTrue(x.grad[2][3].norm() == 0)
print("done")
def __init__(self, D, embdim, *innerdim, **kw):
super(Discriminator, self).__init__()
self.emb = q.WordEmb(embdim, worddic=D)
self.core = q.FastestLSTMEncoder(embdim, *innerdim)
self.outlin1 = torch.nn.Linear(innerdim[-1], innerdim[-1])
self.outlin1_sigm = torch.nn.Sigmoid()
self.outlin2 = torch.nn.Linear(innerdim[-1], 1)
self.outlin2_sigm = torch.nn.Sigmoid()
def setUp(self):
batsize = 5
seqlen = 4
lstm = q.FastestLSTMEncoder(20, 26, 30, bidir=True)
x = torch.nn.Parameter(torch.randn(batsize, seqlen, 20))
y = lstm(x)
self.batsize, self.seqlen = batsize, seqlen
self.x, self.y = x, y
self.lstm = lstm
def setUp(self):
batsize = 5
seqlen = 4
lstm = q.FastestLSTMEncoder(20, 26, 30)
x = torch.nn.Parameter(torch.randn(batsize, seqlen, 20))
y = lstm(x)
self.batsize = batsize
self.seqlen = seqlen
self.x = x
self.y = y
self.lstm = lstm
def test_init_states(self):
batsize = 5
seqlen = 4
lstm = q.FastestLSTMEncoder(20, 26, 30)
lstm.train(False)
x = torch.nn.Parameter(torch.randn(batsize, seqlen * 2, 20))
y_whole = lstm(x)
y_first, states = lstm(x[:, :seqlen], ret_states=True)
states = list(zip(*states))
y_second = lstm(x[:, seqlen:], y_0s=states[0], c_0s=states[1])
y_part = torch.cat([y_first, y_second], 1)
self.assertTrue(
np.allclose(y_whole.detach().numpy(),
y_part.detach().numpy()))
def __init__(self,
embdim,
dims,
word_dic,
bidir=False,
dropout_in=0.,
dropout_rec=0.,
gfrac=0.):
"""
:param word_dic: dictionary mapping words (strings) to ids as they occur in the input x to .forward()
:param gfrac:
"""
""" embdim for embedder, dims is a list of dims for RNN"""
super(FlatEncoder, self).__init__()
self.emb = q.PartiallyPretrainedWordEmb(embdim,
worddic=word_dic,
gradfracs=(1., gfrac))
self.lstm = q.FastestLSTMEncoder(embdim,
*dims,
bidir=bidir,
dropout_in=dropout_in,
dropout_rec=dropout_rec)
def test_dropout_in(self):
batsize = 5
seqlen = 8
lstm = q.FastestLSTMEncoder(20,
30,
bidir=False,
dropout_in=0.3,
dropout_rec=0.)
x = torch.nn.Parameter(torch.randn(batsize, seqlen, 20))
mask = np.zeros((batsize, seqlen)).astype("int64")
mask[0, :3] = 1
mask[1, :] = 1
mask[2, :5] = 1
mask[3, :1] = 1
mask[4, :4] = 1
mask = torch.tensor(mask)
assert (lstm.training)
y = lstm(x, mask=mask)
self.batsize, self.seqlen = batsize, seqlen
self.x, self.y = x, y
self.mask = mask
self.lstm = lstm
y_t0_r0 = self.lstm(self.x, mask=self.mask)[:, 0, :30]
y_t0_r1 = self.lstm(self.x, mask=self.mask)[:, 0, :30]
y_t0_r2 = self.lstm(self.x, mask=self.mask)[:, 0, :30]
self.assertTrue(not np.allclose(y_t0_r0.detach().numpy(),
y_t0_r1.detach().numpy()))
self.assertTrue(not np.allclose(y_t0_r1.detach().numpy(),
y_t0_r2.detach().numpy()))
self.assertTrue(not np.allclose(y_t0_r0.detach().numpy(),
y_t0_r2.detach().numpy()))
def setUp(self):
batsize = 5
seqlen = 8
lstm = q.FastestLSTMEncoder(20, 26, 30, bidir=True)
x = torch.nn.Parameter(torch.randn(batsize, seqlen, 20))
mask = np.zeros((batsize, seqlen)).astype("int64")
mask[0, :3] = 1
mask[1, :] = 1
mask[2, :5] = 1
mask[3, :1] = 1
mask[4, :4] = 1
mask = torch.tensor(mask)
y, yT = lstm(x, mask=mask, ret_states=True)
self.yT = lstm.y_n[-1]
self.batsize, self.seqlen = batsize, seqlen
self.x, self.y = x, y
self.mask = mask
self.lstm = lstm
# reference
self.rf_lstm = LSTMEncoder(20, 26, 30, bidir=True)
self.rf_lstm.layers[0].weight_ih_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[0].layer.weight_ih_l0.detach().numpy() +
0))
self.rf_lstm.layers[0].weight_ih_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[0].layer.weight_ih_l0_reverse.detach().numpy() +
0))
self.rf_lstm.layers[0].weight_hh_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[0].layer.weight_hh_l0.detach().numpy() +
0))
self.rf_lstm.layers[0].weight_hh_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[0].layer.weight_hh_l0_reverse.detach().numpy() +
0))
self.rf_lstm.layers[0].bias_ih_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[0].layer.bias_ih_l0.detach().numpy() + 0))
self.rf_lstm.layers[0].bias_ih_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[0].layer.bias_ih_l0_reverse.detach().numpy() + 0))
self.rf_lstm.layers[0].bias_hh_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[0].layer.bias_hh_l0.detach().numpy() + 0))
self.rf_lstm.layers[0].bias_hh_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[0].layer.bias_hh_l0_reverse.detach().numpy() + 0))
self.rf_lstm.layers[1].weight_ih_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[1].layer.weight_ih_l0.detach().numpy() +
0))
self.rf_lstm.layers[1].weight_ih_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[1].layer.weight_ih_l0_reverse.detach().numpy() +
0))
self.rf_lstm.layers[1].weight_hh_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[1].layer.weight_hh_l0.detach().numpy() +
0))
self.rf_lstm.layers[1].weight_hh_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[1].layer.weight_hh_l0_reverse.detach().numpy() +
0))
self.rf_lstm.layers[1].bias_ih_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[1].layer.bias_ih_l0.detach().numpy() + 0))
self.rf_lstm.layers[1].bias_ih_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[1].layer.bias_ih_l0_reverse.detach().numpy() + 0))
self.rf_lstm.layers[1].bias_hh_l0 = torch.nn.Parameter(
torch.tensor(lstm.layers[1].layer.bias_hh_l0.detach().numpy() + 0))
self.rf_lstm.layers[1].bias_hh_l0_reverse = torch.nn.Parameter(
torch.tensor(
lstm.layers[1].layer.bias_hh_l0_reverse.detach().numpy() + 0))
rf_x = torch.nn.Parameter(torch.tensor(x.detach().numpy() + 0))
assert (rf_x is not x)
self.rf_y, rf_yT = self.rf_lstm(rf_x, mask=mask, ret_states=True)
self.rf_yT = rf_yT
self.rf_x = x
def run_normal_seqvae_toy(
lr=0.001,
embdim=64,
encdim=100,
zdim=64,
batsize=50,
epochs=100,
):
# test
vocsize = 100
seqlen = 12
wD = dict((chr(xi), xi) for xi in range(vocsize))
# region encoder
encoder_emb = q.WordEmb(embdim, worddic=wD)
encoder_lstm = q.FastestLSTMEncoder(embdim, encdim)
class EncoderNet(torch.nn.Module):
def __init__(self, emb, core):
super(EncoderNet, self).__init__()
self.emb, self.core = emb, core
def forward(self, x):
embs, mask = self.emb(x)
out, states = self.core(embs, mask, ret_states=True)
top_state = states[-1][0][:, 0]
# top_state = top_state.unsqueeze(1).repeat(1, out.size(1), 1)
return top_state # (batsize, encdim)
encoder_net = EncoderNet(encoder_emb, encoder_lstm)
encoder = Posterior(encoder_net, encdim, zdim)
# endregion
# region decoder
decoder_emb = q.WordEmb(embdim, worddic=wD)
decoder_lstm = q.LSTMCell(embdim + zdim, encdim)
decoder_outlin = q.WordLinout(encdim, worddic=wD)
class DecoderCell(torch.nn.Module):
def __init__(self, emb, core, out, **kw):
super(DecoderCell, self).__init__()
self.emb, self.core, self.out = emb, core, out
def forward(self, xs, z=None):
embs, mask = self.emb(xs)
core_inp = torch.cat([embs, z], 1)
core_out = self.core(core_inp)
out = self.out(core_out)
return out
decoder_cell = DecoderCell(decoder_emb, decoder_lstm, decoder_outlin)
decoder = q.TFDecoder(decoder_cell)
# endregion
likelihood = Likelihood()
vae = SeqVAE(encoder, decoder, likelihood)
x = torch.randint(0, vocsize, (batsize, seqlen), dtype=torch.int64)
ys = vae(x)
optim = torch.optim.Adam(q.params_of(vae), lr=lr)
x = torch.randint(0, vocsize, (batsize * 100, seqlen), dtype=torch.int64)
dataloader = q.dataload(x, batch_size=batsize, shuffle=True)
trainer = q.trainer(vae).on(dataloader).optimizer(optim).loss(4).epochs(
epochs)
trainer.run()
print("done \n\n")