def step(engine, batch):
model.train()
x, v = batch
x, v = x.to(device), v.to(device)
x, v, x_q, v_q = partition(x, v)
# Reconstruction, representation and divergence
x_mu, _, kl = model(x, v, x_q, v_q)
# Log likelihood
sigma = next(sigma_scheme)
ll = Normal(x_mu, sigma).log_prob(x_q)
likelihood = torch.mean(torch.sum(ll, dim=[1, 2, 3]))
kl_divergence = torch.mean(torch.sum(kl, dim=[1, 2, 3]))
# Evidence lower bound
elbo = likelihood - kl_divergence
loss = -elbo
loss.backward()
optimizer.step()
optimizer.zero_grad()
with torch.no_grad():
# Anneal learning rate
mu = next(mu_scheme)
i = engine.state.iteration
for group in optimizer.param_groups:
group["lr"] = mu * math.sqrt(1 - 0.999 ** i) / (1 - 0.9 ** i)
return {"elbo": elbo.item(), "kl": kl_divergence.item(), "sigma": sigma, "mu": mu}
def save_images(engine):
with torch.no_grad():
x, v = engine.state.batch
x, v = x.to(device), v.to(device)
x, v, x_q, v_q = partition(x, v)
x_mu, r, _ = model(x, v, x_q, v_q)
r = r.view(-1, 1, 16, 16)
# Send to CPU
x_mu = x_mu.detach().cpu().float()
r = r.detach().cpu().float()
writer.add_image("representation", make_grid(r), engine.state.epoch)
writer.add_image("reconstruction", make_grid(x_mu), engine.state.epoch)
def step(engine, batch):
model.train()
x, v = batch
x, v = x.to(device), v.to(device)
x, v, x_q, v_q = partition(x, v)
# Reconstruction, representation and divergence
x_mu, _, kl = model(x, v, x_q, v_q)
# Log likelihood
sigma = next(sigma_scheme)
ll = Normal(x_mu, sigma).log_prob(x_q)
likelihood = torch.mean(torch.sum(ll, dim=[1, 2, 3]))
kl_divergence = torch.mean(torch.sum(kl, dim=[1, 2, 3]))
# Evidence lower bound
elbo = likelihood - kl_divergence
loss = -elbo
loss.backward()
optimizer.step()
optimizer.zero_grad()
with torch.no_grad():
# Anneal learning rate
mu = next(mu_scheme)
i = engine.state.iteration
for group in optimizer.param_groups:
group['lr'] = mu * math.sqrt(1 - 0.999**i) / (1 - 0.9**i)
print(
f'e{engine.state.epoch},i{engine.state.iteration}/{engine.state.epoch_length}: elbo {elbo.item()}, kl {kl_divergence.item()}, sigma {sigma}, mu {mu}'
)
return {
'elbo': elbo.item(),
'kl': kl_divergence.item(),
'sigma': sigma,
'mu': mu
}
def validate(engine):
with torch.no_grad():
x, v = next(iter(valid_loader))
x, v = x.to(device), v.to(device)
x, v, x_q, v_q = partition(x, v)
# Reconstruction, representation and divergence
x_mu, _, kl = model(x, v, x_q, v_q)
# Validate at last sigma
ll = Normal(x_mu, sigma_scheme.recent).log_prob(x_q)
likelihood = torch.mean(torch.sum(ll, dim=[1, 2, 3]))
kl_divergence = torch.mean(torch.sum(kl, dim=[1, 2, 3]))
# Evidence lower bound
elbo = likelihood - kl_divergence
writer.add_scalar("validation/elbo", elbo.item(), engine.state.epoch)
writer.add_scalar("validation/kl", kl_divergence.item(), engine.state.epoch)
im.set_data(image)
fig.canvas.draw()
plt.pause(0.05)
datapath = "D:\\Projekte\\MachineLearning\\DataSets\\temp"
#train_dataset = ShepardMetzler(root_dir=datapath, fraction=1)
#train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
#valid_dataset = ShepardMetzler(root_dir=datapath, fraction=1.0, train=False)
#valid_loader = DataLoader(valid_dataset, batch_size=1, shuffle=True)
train_dataset = rooms_free_camera_no_object_rotations(root_dir=datapath,
fraction=1)
train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True)
train_imgs, train_viewpoints = next(iter(train_loader))
#valid_imgs, valid_viewpoints = next(iter(valid_loader))
part_train_imgs, part_train_viewpoints, context_imgs, context_viewpoints = partition(
train_imgs, train_viewpoints)
for parts in part_train_imgs:
for img in parts:
showimg = img.numpy().transpose((1, 2, 0))
displayImage(showimg)
#test = representation numpy().transpose((1,2,0))
x = 5
from gqn import GenerativeQueryNetwork, partition, Annealer
from dataset import GQN_Dataset
import glob
import os
import numpy as np
import cv2
import torch
valid_dataset = GQN_Dataset(root_dir="../data/rooms_ring_camera/", train=False)
x, v = valid_dataset[4]
x = x.view((1, *x.shape))
v = v.view((1, *v.shape))
max_m = 5
x, v, x_q, v_q = partition(x, v, max_m, 4)
batch, *_ = x.shape
device = torch.device("cpu")
model = GenerativeQueryNetwork(x_dim=3,
v_dim=7,
r_dim=256,
h_dim=128,
z_dim=64,
L=10).to(device)
models = glob.glob("../checkpoints/checkpoint_model_*.pth")
models.sort(key=lambda x: os.path.getmtime(x))
last_checkpoint = models[-1]
checkpoint = torch.load(last_checkpoint, map_location="cpu")
model.load_state_dict(checkpoint)
scenes = [1, 2]
print("Batch Tensor: ")
print(batch[0].shape)
print(batch[1].shape)
#x, v = x.to(device), v.to(device)
#x, v, x_q, v_q = partition(x, v)
print("img Batch Tensor: ")
print(imgs.shape)
print("viewpoints Batch Tensor: ")
print(viewpoints.shape)
#def step(batch):
model.train()
x, v = batch
x, v = x.to(device), v.to(device)
x, v, x_q, v_q = partition(x, v)
# Reconstruction, representation and divergence
x_mu, _, kl = model(x, v, x_q, v_q)
# Log likelihood
sigma = next(sigma_scheme)
ll = Normal(x_mu, sigma).log_prob(x_q)
likelihood = torch.mean(torch.sum(ll, dim=[1, 2, 3]))
kl_divergence = torch.mean(torch.sum(kl, dim=[1, 2, 3]))
# Evidence lower bound
elbo = likelihood - kl_divergence
loss = -elbo
loss.backward()
v_dim=7,
r_dim=256,
h_dim=128,
z_dim=64,
L=8)
pretrained_dict = torch.load(modelFullPath, map_location='cpu') #.to(device)
model_trained.load_state_dict(pretrained_dict)
model_trained = model_trained.to(device)
# datapath
datapath = "D:\\Projekte\\MachineLearning\\DataSets\\shepard_metzler_7_parts"
valid_dataset = ShepardMetzler(root_dir=datapath, fraction=1.0, train=False)
valid_loader = DataLoader(valid_dataset, batch_size=1, shuffle=True)
valid_imgs, valid_viewpoints = next(iter(valid_loader))
part_valid_imgs, part_valid_viewpoints, part_context_imgs, part_context_viewpoints = partition(
valid_imgs, valid_viewpoints)
batch_size, num_views, channels, height, width = part_valid_imgs.shape
model_trained.eval()
with torch.no_grad():
for valid_imgs, viewpoints, context_img, context_viewpoint in zip(
part_valid_imgs, part_valid_viewpoints, part_context_imgs,
part_context_viewpoints):
# Reconstruction, representation and divergence
#x_ = valid_imgs.view(-1, channels, height, width)
#v_ = viewpoints.view(-1, 7)
