def predict(params):
inverter = PrectNormalizer(xr.open_dataset(params['norm_fn']),
params['output_vars'],
params['input_transform'][0],
params['input_transform'][1],
params['var_cut_off'], params['model_type'])
model = CVAE(params)
optimizer = optim.Adam(model.parameters(), lr=0.00001, weight_decay=0.001)
### Load model
checkpoint = torch.load('./runs/VAE_model_DNN_classifier_exp_VAE_Exp04.pt')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']
model.cuda()
model.eval()
valid_dataset = spcamDataset(params, phase="validation")
valid_loader = DataLoader(valid_dataset,
sampler=SubsetSampler(valid_dataset.indices))
result_predicted, result_actual = [], []
with torch.no_grad():
for batch_idx, (data, target) in enumerate(valid_loader):
target = target.squeeze(0).type(torch.float32).to(params['device'])
z = torch.randn(data.shape[1], 16)
z = torch.cat((z, data.squeeze(0)), dim=1).cuda()
predPrecit = model.decoder(z)
print("Batch MSE {}".format(
metrics.mean_squared_error(
predPrecit.detach().cpu().numpy(),
target.squeeze(0).detach().cpu().numpy())))
#val_loss = compute_loss(target, sampled_precit, mean, log_var) #.type(torch.FloatTensor).to(params['device']))
#assert val_loss.requires_grad == False
result_predicted.extend(predPrecit.cpu().detach().numpy())
result_actual.extend(target.squeeze(0).cpu().detach().numpy())
mse = metrics.mean_squared_error(np.array(result_actual),
np.array(result_predicted))
print("MSE {}".format(mse))
def __init__(self):
super(ConvolutionalVariationalAutoencoder, self).__init__()
train_images, test_images = get_dataset()
self.train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(cfg.train_buf).batch(cfg.batch_size)
self.test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(cfg.test_buf).batch(cfg.batch_size)
self.optimizer = tf.keras.optimizers.Adam(lr=cfg.learning_rate)
self.global_step = 0
self.model = CVAE(cfg.latent_dim)
self.build_writers()
class TestCVAE(unittest.TestCase):
def setUp(self) -> None:
# self.model2 = VAE(3, 10)
self.model = CVAE(3, 40, 10)
def test_forward(self):
x = torch.randn(16, 3, 64, 64)
c = torch.randn(16, 40)
y = self.model(x, c)
print("Model Output size:", y[0].size())
# print("Model2 Output size:", self.model2(x)[0].size())
def test_loss(self):
x = torch.randn(16, 3, 64, 64)
c = torch.randn(16, 40)
result = self.model(x, labels=c)
loss = self.model.loss_function(*result, M_N=0.005)
print(loss)
def training(dataset, train_log, loss_func):
train_images, test_images, sample_train, sample_test, input_shape = \
create_data_iter(dataset, train_buf, test_buf, batch_size)
num_channel = input_shape[-1]
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
sample_vectors = load_or_create_vectors('sample.npy')
random_vector_for_generation = tf.constant(sample_vectors,
dtype=tf.float32)
model = CVAE(input_shape, latent_dim)
model.build_graph()
generate_and_save_images(model, random_vector_for_generation,
'sampling_epoch_0.png', num_channel=num_channel)
for epoch in range(1, epochs + 1):
start_time = time.time()
train_loss = tf.keras.metrics.Mean()
for train_x in train_images:
gradients, loss = compute_gradients(model, train_x, loss_func)
train_loss(loss)
apply_gradients(optimizer, gradients, model.trainable_variables)
end_time = time.time()
train_elbo = -train_loss.result()
if epoch % log_every == 0:
loss = tf.keras.metrics.Mean()
for test_x in test_images:
loss(compute_loss(model, test_x, loss_func))
elbo = -loss.result()
print('Epoch: {}, Train ELBO: {}, Test ELBO: {}, '
'Time {}'.format(
epoch, train_elbo, elbo, end_time - start_time))
generate_and_save_images(model, random_vector_for_generation,
'sampling_epoch_{}.png'.format(epoch),
num_channel=num_channel)
visualize_reconstruction(model, sample_train, sample_test, epoch)
model.save_weights(os.path.join(train_log, 'model'))
create_gif(del_remain=False)
zip_file(os.path.join(train_log, 'images'),
os.path.join(train_log, 'images.zip'))
from utils import trainEpochs, plot_results
from dataloader import get_train_loader
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# ----------Hyper Parameters---------- #
hidden_size = 256
cond_embedding_size = 8
latent_size = 32
n_epochs = 900
batch_size = 16
learning_rate = 0.001
train_loader = get_train_loader(batch_size=batch_size)
for KL_annealing_method in ['cyclical', 'monotonic']:
cvae = CVAE(hidden_size, latent_size, cond_embedding_size).to(device)
history = trainEpochs(train_loader=train_loader,
cvae=cvae,
n_epochs=n_epochs,
learning_rate=learning_rate,
KL_annealing_method=KL_annealing_method)
plot_results(history, KL_annealing_method)
filename = f'./histories/{KL_annealing_method}.pkl'
with open(filename, "wb") as fp:
pickle.dump(history, fp)
train=True,
transform=transforms.ToTensor())
trainLoader = torch.utils.data.DataLoader(trainDataset,
batch_size=opts.batchSize,
shuffle=True)
testDataset = CELEBA(root=opts.root,
train=False,
transform=transforms.ToTensor())
testLoader = torch.utils.data.DataLoader(testDataset,
batch_size=opts.batchSize,
shuffle=False)
print 'Data loaders ready.'
####### Create model #######
cvae = CVAE(nz=opts.nz, imSize=64, fSize=opts.fSize)
dis = DISCRIMINATOR(imSize=64, fSize=opts.fSize)
if cvae.useCUDA:
print 'using CUDA'
cvae.cuda()
dis.cuda()
else:
print '\n *** NOT USING CUDA ***\n'
print cvae
print dis
####### Define optimizer #######
optimizerCVAE = optim.RMSprop(
cvae.parameters(),
for i in range(predictions.shape[0]):
plt.subplot(4, 4, i + 1, facecolor=(0, 0, 0))
plt.imshow(predictions[i, :, :, 0], cmap='gray')
# plt.imshow(predictions[i, :, :, 0])
plt.axis('off')
plt.axis('off')
# plt.imshow(predictions[0,:,:,0])#Single img show
plt.savefig(localpath + '/genimg/image_at_epoch_{:04d}.jpg'.format(epoch))
#plt.show()
# this will be easier to see the improvement.
random_vector_for_generation = tf.random.normal(
shape=[num_examples_to_generate, latent_dim])
model = CVAE(latent_dim)
#get some samples from test to gen output
assert batch_size >= num_examples_to_generate
test_sample = 0
for test_batch in test_dataset.take(1): #Changed for full img
# test_sample = test_batch[0:num_examples_to_generate, :, :, :]
test_sample = test_batch[0:num_examples_to_generate, :, :, :]
generate_and_save_images(model, 0, test_sample)
for epoch in range(1, epochs + 1):
start_time = time.time()
for train_x in train_dataset:
train_step(model, train_x, optimizer)
end_time = time.time()
loss = tf.keras.metrics.Mean()
if __name__ == "__main__":
data_dir = 'data/coco2017'
log_dir = 'logs'
input_image_size = (256, 256, 3)
batch_size = 10
latent_dim = 32
optimizer = tf.keras.optimizers.Adam(1e-3)
# Initialize and compile models
incept_model = InceptionV3(include_top=False,
pooling='avg',
input_shape=input_image_size)
ae_model = AE(latent_dim, input_image_size)
cae_model = CAE(latent_dim, input_image_size)
vae_model = VAE(latent_dim, input_image_size)
cvae_model = CVAE(latent_dim, input_image_size)
memcae_model = MemCAE(latent_dim, True, input_image_size, batch_size, 500,
optimizer)
for classes in [['cat']]:
# Load and augment training data
ds_train = dataloader(classes, data_dir, input_image_size, batch_size,
'train2019')
ds_val = dataloader(classes, data_dir, input_image_size, batch_size,
'val2019')
class_label = classes[0] if len(classes) == 1 else "similar"
# Train each model for comparison
for m in [memcae_model]:
print("Training {} on {} class...".format(m.architecture,
class_label))
def setUp(self) -> None:
# self.model2 = VAE(3, 10)
self.model = CVAE(3, 40, 10)
)
test_dataset = datasets.MNIST(
'./data',
train=False,
download=True,
transform=transforms.Compose(
[transforms.ToTensor()]
)
)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size)
model = CVAE(input_size, hidden_size, latent_size, num_of_classes).to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
for epoch in tqdm.tqdm(range(n_epochs)):
model.train()
train_loss = 0
for x, y in train_dataloader:
x = x.view(-1, input_size).to(device)
y = utils.y_to_onehot(y, batch_size, num_of_classes).to(device)
optimizer.zero_grad()
x_mu, x_logvar, z, z_mu, z_logvar = model(x, y)
loss = model.loss_calc(x, x_mu, z_mu, z_logvar)
return opt
opt = parse_opt()
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
transform = transforms.ToTensor()
dataset = torchvision.datasets.MNIST(root='../data', train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batch_size, shuffle=True, num_workers=4)
data_shape = dataset[0][0].shape[1:]
data_size = dataset[0][0].numel()
net = CVAE(1, data_shape, data_size, 20, 400, 10)
net = net.to(device)
optimizer = optim.Adam(net.parameters(), lr=opt.learning_rate)
summary = Summary()
iteration = 0
def loss_fn(inputs, outputs, mean, logvar):
MSE = F.mse_loss(outputs.view(-1, data_size), inputs.view(-1, data_size), size_average=False)
KLD = -0.5 * torch.sum(1 + logvar - mean.pow(2) - logvar.exp())
return MSE + KLD
def train():
global iteration
net.train()
class ConvolutionalVariationalAutoencoder(object):
def __init__(self):
super(ConvolutionalVariationalAutoencoder, self).__init__()
train_images, test_images = get_dataset()
self.train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(cfg.train_buf).batch(cfg.batch_size)
self.test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(cfg.test_buf).batch(cfg.batch_size)
self.optimizer = tf.keras.optimizers.Adam(lr=cfg.learning_rate)
self.global_step = 0
self.model = CVAE(cfg.latent_dim)
self.build_writers()
def log_normal_pdf(self, sample, mean, logvar, raxis=1):
log2pi = tf.math.log(2. * np.pi)
return tf.reduce_sum(
-.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi),
axis=raxis)
def build_writers(self):
if not Path(cfg.save_dir).is_dir():
os.mkdir(cfg.save_dir)
if not Path(cfg.image_dir).is_dir():
os.mkdir(cfg.image_dir)
if cfg.extension is None:
cfg.extension = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
self.log_path = cfg.log_dir + cfg.extension
self.writer = tf.summary.create_file_writer(self.log_path)
self.writer.set_as_default()
self.save_path = cfg.save_dir + cfg.extension
self.ckpt_prefix = self.save_path + '/ckpt'
# self.saver = tf.train.Checkpoint(generator=self.generator, gen_optim=self.gen_optim, discriminator=self.discriminator, disc_optim=self.disc_optim, global_step=self.global_step, epoch=self.epoch)
tf.summary.experimental.set_step(0)
def logger(self, loss):
if self.global_step % cfg.log_freq == 0:
tf.summary.scalar('loss', loss, step=self.global_step)
def log_img(self, name, img):
if self.global_step % (cfg.log_freq*4) == 0:
tf.summary.image(name, img, step=self.global_step, max_outputs=6)
def generate_and_save_images(self, model, epoch, test_input):
predictions = model.sample(test_input)
fig = plt.figure(figsize=(4,4))
for i in range(predictions.shape[0]):
plt.subplot(4, 4, i+1)
plt.imshow(predictions[i, :, :, 0], cmap='gray')
plt.axis('off')
# tight_layout minimizes the overlap between 2 sub-plots
# holy hack is bad
plt.savefig('{}image_at_epoch_{:04d}.png'.format(cfg.image_dir,epoch))
img = np.asarray(PIL.Image.open('{}image_at_epoch_{:04d}.png'.format(cfg.image_dir,epoch)).convert('RGB'))
self.log_img("predictions", np.array([img]))
def make_gif(self):
anim_file = 'evaluation/cvae.gif'
with imageio.get_writer(anim_file, mode='I') as writer:
filenames = glob.glob(cfg.image_dir+'image*.png')
filenames = sorted(filenames)
last = -1
for i,filename in enumerate(filenames):
frame = 2*(i**0.5)
if round(frame) > round(last):
last = frame
else:
continue
image = imageio.imread(filename)
writer.append_data(image)
image = imageio.imread(filename)
writer.append_data(image)
def train(self):
# keeping the random vector constant for generation (prediction) so
# it will be easier to see the improvement.
random_vector_for_generation = tf.random.normal(shape=[cfg.num_examples_to_generate, cfg.latent_dim])
print("Training Begins")
for epoch in trange(1, cfg.epochs + 1):
for train_x in self.train_dataset:
with tf.GradientTape() as tape:
mean, logvar = self.model.encode(train_x)
z = self.model.reparameterize(mean, logvar)
x_logit = self.model.decode(z)
cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=train_x)
logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])
logpz = self.log_normal_pdf(z, 0., 0.)
logqz_x = self.log_normal_pdf(z, mean, logvar)
loss = -tf.reduce_mean(logpx_z + logpz - logqz_x)
self.logger(loss)
gradients = tape.gradient(loss, self.model.trainable_variables)
gradient_and_vars = zip(gradients, self.model.trainable_variables)
self.optimizer.apply_gradients(gradient_and_vars)
self.global_step += 1
self.generate_and_save_images(self.model, epoch, random_vector_for_generation)
if cfg.make_gif == True:
self.make_gif()
if iter % save_every == 0:
torch.save(
{
"model": model.state_dict(),
"teacher_forcing_ratio": teacher_forcing_ratio,
"KLD_weight": KLD_weight,
}, 'results/{}-Model-{}.ckpt'.format(time_now, iter))
adjust_teacher_forcing(iter)
adjust_kld_weight(iter)
model.teacher_forcing_ratio = teacher_forcing_ratio
if __name__ == "__main__":
cvae = CVAE(vocab_size,
latent_size,
hidden_size,
condition_size,
condition_embedding_size,
teacher_forcing_ratio=teacher_forcing_ratio,
SOS=SOS_token,
EOS=EOS_token).to(device)
trainIters(cvae,
75000,
print_every=10,
plot_every=plot_every,
save_every=save_every,
learning_rate=LR)
