def launcher_predict(net,
data_generator,
criterion,
optimizer,
n_epoch,
n_iter,
idx,
plot=False):
if plot:
losses = []
fig, ax = plt.subplots(1, 1)
idx_in, idx_target = idx
start = time.time()
for epoch in range(n_epoch):
for i in range(n_iter):
z = np.random.normal(size=(data_generator.dim_z, 1))
res = data_generator.sample(z)
# Training iteration
value = res[idx_in]
target = res[idx_target]
#target = torch.FloatTensor(z).view(1 ,1, -1)
# Initialize hidden, grad, loss
net.zero_grad()
loss = 0
input = value.view(1, 1, -1)
target = target.view(1, 1, -1)
output = net(input)
loss = criterion(output, target)
# Gradient step
loss.backward()
optimizer.step()
if plot:
losses.append(loss.data.numpy())
ax.plot(losses)
plt.draw()
plt.show()
plt.pause(0.01)
print("Epoch {0}, {1} : {2}".format(epoch, timeSince(start),
loss.data.view(-1).numpy()[0]))
input = torch.cat(inputs, dim=0)
# Initialize hidden, grad, loss
rnn.zero_grad()
loss = 0
h0 = torch.zeros(2, batch_size,
hidden_size) # num_layers / batch / dim
output = rnn(input, h0)
loss = criterion(output, input) # Not sure here with the dimensions
# Gradient step
loss.backward()
optimizer.step()
print("Epoch {0}, {1} : {2}".format(epoch, timeSince(start),
loss.data.view(-1).numpy()[0]))
#%% Visualization of results
i = np.random.randint(0, 1000)
# Generate inputs
samples = [generator.sample() for j in range(batch_size)]
inputs = [
torch.FloatTensor(sample.values[:, 2:]).view(1, dim_time, dim_input)
for sample in samples
]
input = torch.cat(inputs, dim=0)
h0 = torch.zeros(2, batch_size, hidden_size) # num_layers / batch / dim
output = rnn(input, h0)
start = time.time()
for epoch in range(n_epoch):
for i in range(n_iter):
z_in = np.random.normal(size=(dim_z, 1))
multi_x = subset3.sample(z_in, noise=True)
# Randomly remove modalities
# Initialize hidden, grad, loss
multiencoder.zero_grad()
loss = 0
output = multiencoder(multi_x)
loss = criterion(torch.cat(output).view(N, 1, dim_input), torch.cat(multi_x)) # Not sure here with the dimensions
#loss = criterion(output, torch.FloatTensor(z_in).view(1, -1)) # Not sure here with the dimensions
# Gradient step
loss.backward(retain_graph=True)
optimizer.step()
print("Epoch {0}, {1} : {2}".format(epoch, timeSince(start), loss.data.view(-1).numpy()[0]))
#%% Learned Representation
z_in = np.random.normal(size=(dim_z, 1))
multi_x = subset3.sample(z_in, noise=True)
z_latent = multiencoder(multi_x)
def trainIters(model, n_iters, print_every=1000, plot_every=1, learning_rate=0.01):
start = time.time()
plot_losses = []
plot_kl_losses = []
plot_ce_losses = []
plot_bleu_score = []
print_loss_total = 0 # Reset every print_every
plot_loss_total = 0 # Reset every plot_every
best_bleu_score = 0
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
training_pairs = []
training_cs = []
for i in range(n_iters):
pair = random.choice(pairs)
tfp = tensorsFromPair(pair)
training_pairs.append(tfp[0])
training_cs.append(tfp[1])
criterion = nn.CrossEntropyLoss()
for iter in range(1, n_iters + 1):
training_pair = training_pairs[iter - 1]
training_c = training_cs[iter - 1]
input_tensor = training_pair[0]
target_tensor = training_pair[1]
input_c = training_c[0]
target_c = training_c[1]
loss, ce_loss, kl_loss = train(input_tensor, target_tensor, input_c, target_c, model, optimizer, criterion, kl_annealing(iter))
print_loss_total += loss
plot_loss_total += loss
bleu = evaluateByTestData(model)
if best_bleu_score < bleu:
best_bleu_score = bleu
if iter % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
iter, iter / n_iters * 100, print_loss_avg))
print("BestBleu: {:.4f}".format(best_bleu_score))
print("Bleu: {:.4f}".format(bleu))
if iter % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_losses.append(plot_loss_avg)
plot_kl_losses.append(kl_loss.item())
plot_ce_losses.append(ce_loss)
plot_bleu_score.append(bleu)
plot_loss_total = 0
if bleu > 0.7:
save_checkpoint({
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'bleu': bleu,
'ce_loss': ce_loss,
'kl_loss': kl_loss
}, f'./checkpoint/{bleu}_{iter}.pth')
showPlot(plot_losses)
torch.save({
'kl_loss': plot_kl_losses,
'ce_loss': plot_ce_losses,
'loss': plot_losses,
'bleu_score': plot_bleu_score
}, "plot_result2")
