def concat_items(images_path, capture_folder, manual_contrast_folder, our_method_folder, groundtruth_folder, output_folder):
for im in os.listdir(os.path.join(images_path, capture_folder)):
captured_image = Image.open(os.path.join(images_path, capture_folder, im))
manual_contrast_image = Image.open(os.path.join(images_path, manual_contrast_folder, im))
our_method_image = Image.open(os.path.join(images_path, our_method_folder, im))
groundtruth_image = Image.open(os.path.join(images_path, groundtruth_folder, im))
concat_images([captured_image, manual_contrast_image, our_method_image, groundtruth_image],
os.path.join(images_path, output_folder, im))
def saveResult(save_path, results, inputs):
if len(results) == 0:
return
count = 0
for item in results:
image_name, img = item
original_image_name = image_name.split('_')[0] + '.png'
##original_image_name = image_name.split('_')[0] + '.jpg'
# item = (item * 255).astype('uint8')
processed_image = Image.fromarray((img[0]*255).astype('uint8'))
original_image = Image.open(os.path.join('disjoint/originals', original_image_name))
##original_image = Image.open(os.path.join('origim', original_image_name))
input_image = Image.fromarray((next(inputs)[0][0]*255).astype('uint8'))
concat_images([input_image, processed_image, original_image], os.path.join(save_path, image_name))
##misc.imsave(os.path.join(save_path, image_name),processed_image)
# misc.imsave(os.path.join(save_path, image_name), )
count += 1
def forecast_sub(sess, trainer, data_index):
image_dir = flags.save_dir + "/forecast"
if not os.path.exists(image_dir):
os.mkdir(image_dir)
original_images, forecasted_images = trainer.forecast(
sess, data_index=data_index)
seq_length = len(original_images)
# Concatenate images
original_concated_before = utils.concat_images(
original_images[0:seq_length // 2])
original_concated_after = utils.concat_images(original_images[seq_length //
2:])
forecasted_concated_after = utils.concat_images(forecasted_images)
imsave(image_dir + "/original_before{0:0>2}.png".format(data_index),
original_concated_before)
imsave(image_dir + "/original_after{0:0>2}.png".format(data_index),
original_concated_after)
imsave(image_dir + "/forecast_after{0:0>2}.png".format(data_index),
forecasted_concated_after)
# Construct model
pred = colorize_net(tensors)
pred_yuv = tf.concat(3, [tf.split(3, 3, grayscale_yuv)[0], pred])
pred_rgb = yuv2rgb(pred_yuv)
init_op = tf.group(tf.initialize_all_variables(),
tf.initialize_local_variables())
# start session
sess = tf.Session()
sess.run(init_op)
# import model
saver = tf.train.import_meta_graph('full_model/colorize_model_2.ckpt.meta')
saver.restore(sess, 'full_model/colorize_model_2.ckpt.data-00000-of-00001')
pred_, pred_rgb_, colorimage_, grayscale_rgb_ = sess.run(
[pred, pred_rgb, colorimage, grayscale_rgb],
feed_dict={phase_train: False})
summary_image = concat_images(grayscale_rgb_[0], pred_rgb_[0])
summary_image = concat_images(summary_image, colorimage_[0])
# save color output
scipy.misc.imsave("summary_1.jpg", colorimage_[0])
# save summary image
scipy.misc.imsave("summary_1.jpg", summary_image)
sess.close()
def train_color_net(graph, phase_train, uv, grayscale):
pred_rgb = tf.placeholder(tf.float32, name="pred_rgb")
pred = color_net(graph, phase_train, grayscale)
pred_yuv = tf.concat([tf.split(grayscale_yuv, 3, 3)[0], pred],3)
pred_rgb = yuv2rgb(pred_yuv)
colorimage_yuv = rgb2yuv(colorimage)
loss = tf.square(tf.subtract(pred, tf.concat([tf.split(colorimage_yuv, 3, 3)[1], tf.split(colorimage_yuv, 3, 3)[2]],3)))
if uv == 1:
loss = tf.split(loss, 2,3)[0]
elif uv == 2:
loss = tf.split(loss, 2, 3)[1]
else:
loss = (tf.split(loss,2, 3)[0] + tf.split(loss,2, 3)[1]) / 2
global_step = tf.Variable(0, name='global_step', trainable=False)
if phase_train is not None:
optimizer = tf.train.GradientDescentOptimizer(0.0001)
opt = optimizer.minimize(loss, global_step=global_step, gate_gradients=optimizer.GATE_NONE)
# Saver.
saver = tf.train.Saver()
sess = tf.Session()
# Initialize the variables.
sess.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
while not coord.should_stop():
# Run training steps
training_opt = sess.run(opt, feed_dict={phase_train: True, uv: 1})
training_opt = sess.run(opt, feed_dict={phase_train: True, uv: 2})
step = sess.run(global_step)
if step % 1 == 0:
pred_, pred_rgb_, colorimage_, grayscale_rgb_, cost = sess.run(
[pred, pred_rgb, colorimage, grayscale_rgb, loss], feed_dict={phase_train: False, uv: 3})
print({"step": step, "cost": np.mean(cost)})
if step % 10 == 0:
summary_image = concat_images(grayscale_rgb_[0], pred_rgb_[0])
summary_image = concat_images(summary_image, colorimage_[0])
plt.imsave("summary/" + str(step) + "_0.jpg", summary_image)
if step % 100000 == 500:
if not os.path.exists(checkpoints_dir):
os.makedirs(checkpoints_dir)
save_path = saver.save(sess, checkpoints_dir + "/color_net_model"+str(step)+".ckpt")
print("Model saved in file: %s" % save_path)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
def predict_sub(sess, trainer, data_index, off_forecast):
""" Confirm prediction error. """
print("predict data")
if off_forecast:
prefix = "off"
else:
prefix = "on"
image_dir = flags.save_dir + "/predicted_{}_{}".format(prefix, data_index)
if not os.path.exists(image_dir):
os.mkdir(image_dir)
out = trainer.calculate(sess,
data_index=data_index,
off_forecast=off_forecast)
errors, inputs, enc_mus, enc_sigma_sqs, dec_outs, prior_mus, prior_sigma_sqs = out
layer_size = len(errors)
for i in range(layer_size):
save_figure(errors[i],
"predict_pp_kl{}.png".format(i),
image_dir,
ylabel="KLD",
title="Posterior/Prior KLD: Layer{}".format(i))
save_figure(enc_mus[i],
"predict_enc_mu{}.png".format(i),
image_dir,
ylabel="Mean",
title="Posterior mean: Layer{}".format(i))
save_figure(enc_sigma_sqs[i],
"predict_enc_sigma_sq{}.png".format(i),
image_dir,
ylabel="Variance",
title="Posterior variance: Layer{}".format(i))
save_figure(prior_mus[i],
"predict_prior_mu{}.png".format(i),
image_dir,
ylabel="Mean",
title="Prior mean: Layer{}".format(i))
save_figure(prior_sigma_sqs[i],
"predict_prior_sigma_sq{}.png".format(i),
image_dir,
ylabel="Variance",
title="Prior variance: Layer{}".format(i))
if i == 0:
# Show input images and reconstruction images.
dec_out_i = dec_outs[i]
seq_length = inputs[0].shape[0]
for j in range(seq_length):
img_data = inputs[0][j, :, :]
img_pred = dec_out_i[j, :, :]
img = np.hstack([img_data, img_pred])
imsave(image_dir + "/img{0:0>2}.png".format(j), img)
# Save original images by concatenating.
org_concated = utils.concat_images(inputs[0])
imsave(image_dir + "/org_concated_{0:0>2}.png".format(data_index),
org_concated)
def train():
# Seeds
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
np.random.seed(SEED)
random.seed(SEED)
# Device
device = ("cuda" if torch.cuda.is_available() else "cpu")
print("Device: {}".format(device))
# Prepare dataset to make it usable with ImageFolder. Please only do this once
# Uncomment this when you encounter "RuntimeError: Found 0 files in subfolders of:"
# prepare_dataset_and_folder(DATASET_PATH, [IMAGE_SAVE_FOLDER, MODEL_SAVE_FOLDER])
# Tranform, Dataset, DataLoaders
transform = transforms.Compose([
transforms.Resize(TRAIN_IMAGE_SIZE),
transforms.CenterCrop(TRAIN_IMAGE_SIZE),
transforms.ToTensor()
])
x_trainloader, x_testloader = utils.prepare_loader(DATASET_PATH, X_CLASS, transform=transform, batch_size=BATCH_SIZE, shuffle=True)
y_trainloader, y_testloader = utils.prepare_loader(DATASET_PATH, Y_CLASS, transform=transform, batch_size=BATCH_SIZE, shuffle=True)
# [Iterators]: We need iterators for DataLoader because we
# are fetching training samples from 2 or more DataLoaders
x_trainiter, x_testiter = iter(x_trainloader), iter(x_testloader)
y_trainiter, y_testiter = iter(y_trainloader), iter(y_testloader)
# Load Networks
Gxy = models.Generator(64).to(device)
Gyx = models.Generator(64).to(device)
Dxy = models.Discriminator(64).to(device)
Dyx = models.Discriminator(64).to(device)
# Initialize weights with Normal(0, 0.02)
Gxy.apply(models.init_weights)
Gyx.apply(models.init_weights)
Dxy.apply(models.init_weights)
Dyx.apply(models.init_weights)
# Optimizers
# Concatenate Generator ParamsSee Training Loop for explanation
# Reference: https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/cycle_gan_model.py#L94
G_optim = optim.Adam(itertools.chain(Gxy.parameters(), Gyx.parameters()), lr=LR, betas=[BETA_1, BETA_2])
Dxy_optim = optim.Adam(Dxy.parameters(), lr=LR, betas=[BETA_1, BETA_2])
Dyx_optim = optim.Adam(Dyx.parameters(), lr=LR, betas=[BETA_1, BETA_2])
# LR Scheduler
# Reference: https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py#L51
def lambda_rule(epoch):
lr_l = 1.0 - max(0, epoch - START_LR_DECAY)/(NUM_EPOCHS - START_LR_DECAY)
return lr_l
lr_scheduler_G = torch.optim.lr_scheduler.LambdaLR(G_optim, lr_lambda=lambda_rule)
lr_scheduler_Dxy = torch.optim.lr_scheduler.LambdaLR(Dxy_optim, lr_lambda=lambda_rule)
lr_scheduler_Dyx = torch.optim.lr_scheduler.LambdaLR(Dyx_optim, lr_lambda=lambda_rule)
# Some Helper Functions!
# Output of generator is Tanh! so we need to scale real images accordingly
def scale(tensor, mini=-1, maxi=1):
return tensor * (maxi-mini) + mini
# Fixed test samples
fixed_X, _ = next(x_testiter)
fixed_X = fixed_X.to(device)
fixed_Y, _ = next(y_testiter)
fixed_Y = fixed_Y.to(device)
# Loss History for the Whole Training Process
loss_hist = losses.createLogger(["Gxy", "Gyx", "Dxy", "Dyx", "cycle"])
# Number of batches
iter_per_epoch = min(len(x_trainiter), len(y_trainiter))
print("There are {} batches per epoch".format(iter_per_epoch))
for epoch in range(1, NUM_EPOCHS+1):
print("========Epoch {}/{}========".format(epoch, NUM_EPOCHS))
start_time = time.time()
# Loss Logger for the Current Batch
curr_loss_hist = losses.createLogger(["Gxy", "Gyx", "Dxy", "Dyx", "cycle"])
# Reset Iterators every epoch, otherwise, we'll have unequal batch sizes
# or worse, reach the end of iterator and get a Stop Iteration Error
x_trainiter = iter(x_trainloader)
y_trainiter = iter(y_trainloader)
for i in range(1, iter_per_epoch):
# Get current batches
x_real, _ = next(x_trainiter)
x_real = scale(x_real)
x_real = x_real.to(device)
y_real, _ = next(y_trainiter)
y_real = scale(y_real)
y_real = y_real.to(device)
# ========= Discriminator ==========
# In training the discriminators, we fix the generators' parameters
# It is alright to train both discriminators seperately because
# their forward pass don't share any parameters with each other
# Discriminator Y -> X Adversarial Loss
Dyx_optim.zero_grad() # Zero-out Gradients
Dyx_real_out = Dyx(x_real) # Dyx Forward Pass
Dyx_real_loss = real_loss(Dyx_real_out) # Dyx Real Loss
Dyx_fake_out = Dyx(Gyx(y_real)) # Gyx produces fake-x images
Dyx_fake_loss = fake_loss(Dyx_fake_out) # Dyx Fake Loss
Dyx_loss = Dyx_real_loss + Dyx_fake_loss # Dyx Total Loss
Dyx_loss.backward() # Dyx Backprop
Dyx_optim.step() # Dyx Gradient Descent
# Discriminator X -> Y Adversarial Loss
Dxy_optim.zero_grad() # Zero-out Gradients
Dxy_real_out = Dxy(y_real) # Dxy Forward Pass
Dxy_real_loss = real_loss(Dxy_real_out) # Dxy Real Loss
Dxy_fake_out = Dxy(Gxy(x_real)) # Gxy produces fake y-images
Dxy_fake_loss = fake_loss(Dxy_fake_out) # Dxy Fake Loss
Dxy_loss = Dxy_real_loss + Dxy_fake_loss # Dxy Total Loss
Dxy_loss.backward() # Dxy Backprop
Dxy_optim.step() # Dxy Gradient Descent
# ============= Generator ==============
# Similar to training discriminator networks, in training
# generator networks, we fix discriminator networks.
# However, cycle consistency prohibits us
# from training generators seperately.
# Generator X -> Y Adversarial Loss
G_optim.zero_grad() # Zero-out Gradients
Gxy_out = Gxy(x_real) # Gxy Forward Pass : generates fake-y images
D_Gxy_out = Dxy(Gxy_out) # Gxy -> Dxy Forward Pass
Gxy_loss = real_loss(D_Gxy_out) # Gxy Real Loss
# Generator Y -> X Adversarial Loss
Gyx_out = Gyx(y_real) # Gyx Forward Pass : generates fake-x images
D_Gyx_out = Dyx(Gyx_out) # Gyx -> Dyx Forward Pass
Gyx_loss = real_loss(D_Gyx_out) # Gyx Real Loss
# Cycle Consistency Loss
yxy = Gxy(Gyx_out) # Reconstruct Y
yxy_cycle_loss = cycle_loss(yxy, y_real) # Y-X-Y L1 Cycle Reconstruction Loss
xyx = Gyx(Gxy_out) # Reconstruct X
xyx_cycle_loss = cycle_loss(xyx, x_real) # X-Y-X L1 Cycle Reconstruction Loss
G_cycle_loss = CYCLE_WEIGHT * (yxy_cycle_loss + xyx_cycle_loss)
# Generator Total Loss
G_loss = Gxy_loss + Gyx_loss + G_cycle_loss
G_loss.backward()
G_optim.step()
# Record Losses
curr_loss_hist = losses.updateEpochLogger(curr_loss_hist, [Gxy_loss, Gyx_loss, Dxy_loss, Dxy_loss, G_cycle_loss])
# Learning Rate Scheduler Step
lr_scheduler_G.step()
lr_scheduler_Dxy.step()
lr_scheduler_Dyx.step()
# Record and Print Losses
print("Dxy: {} Dyx: {} G: {} Cycle: {}".format(Dxy_loss.item(), Dyx_loss.item(), G_loss.item(), G_cycle_loss.item()))
print("Time Elapsed: {}".format(time.time() - start_time))
loss_hist = losses.updateGlobalLogger(loss_hist, curr_loss_hist)
# Generate Fake Images
Gxy.eval()
Gyx.eval()
with torch.no_grad():
# Generate Fake X Images
x_tensor = generate.evaluate(Gyx, fixed_Y) # Generate Image Tensor
x_images = utils.concat_images(x_tensor) # Merge Image Tensors -> Numpy Array
save_path = IMAGE_SAVE_FOLDER + DATASET_PATH[:-1] + "X" + str(epoch) + ".png"
utils.saveimg(x_images, save_path)
# Generate Fake Y Images
y_tensor = generate.evaluate(Gxy, fixed_X) # Generate Image Tensor
y_images = utils.concat_images(y_tensor) # Merge Image Tensors -> Numpy Array
save_path = IMAGE_SAVE_FOLDER + DATASET_PATH[:-1] + "Y" + str(epoch) + ".png"
utils.saveimg(y_images, save_path)
# Save Model Checkpoints
if ((epoch % SAVE_MODEL_EVERY == 0) or (epoch == 1)):
save_str = MODEL_SAVE_FOLDER + DATASET_PATH[:-1] + str(epoch)
torch.save(Gxy.cpu().state_dict(), save_str + "_Gxy.pth")
torch.save(Gyx.cpu().state_dict(), save_str + "_Gyx.pth")
torch.save(Dxy.cpu().state_dict(), save_str + "_Dxy.pth")
torch.save(Dxy.cpu().state_dict(), save_str + "_Dyx.pth")
Gxy.to(device)
Gyx.to(device)
Dxy.to(device)
Dyx.to(device)
Gxy.train();
Gyx.train();
utils.plot_loss(loss_hist)
def test_concat_images(self):
images = np.zeros((20, 64, 64))
concated = utils.concat_images(images)
# (64, 1299)
self.assertEqual(concated.shape, (64, 64 * 20 + 19))