def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z")
mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask")
mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2")
z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new")
istrain = tf.placeholder(tf.bool)
image_ = tf.image.random_flip_up_down(image)
image_ = tf.image.random_flip_left_right(image_)
image_ = tf.image.resize_with_crop_or_pad(image_, 80, 80)
image_ = tf.image.random_crop(image_, size=(FLAGS.batch_size, 64, 64, 3))
image_ = tf.image.rot90(image_,
k=tf.random_uniform(shape=[],
minval=0,
maxval=4,
dtype=tf.int32))
#image_ = tf.image.pad_to_bounding_box(image_, 16,16,96,96)
#z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip")
#z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv")
e = tf.placeholder(tf.float32, shape=[None, 4, 4, 522], name="e")
e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8202], name="e_p")
save_itr = 0
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# mask_ = tf.ones([FLAGS.batch_size,32,64,3])
# mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
zero = tf.zeros([FLAGS.batch_size, 1, 1, 8202])
logits, h = inference((1 - mask) * image + mask * -2.0, keep_probability,
z, 0.0, istrain)
logits_e, h_e = inference((1 - mask) * image + mask * -2.0,
keep_probability, z, e, istrain)
#logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain )
#logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain )
z_pred = predictor(h, z, zero, istrain)
z_pred_e = predictor(h, z, e_p, istrain)
# z_pred_lip = predictor(h,z_lip,istrain)
# z_pred_lip_inv = predictor(h,z_lip_inv,istrain)
# logits = inference(image, keep_probability,z,istrain)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
# loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
loss_mask = 0.001 * tf.reduce_mean(
tf.reduce_sum(
tf.contrib.layers.flatten(tf.abs(
(annotation - logits) * mask)), 1))
loss_mask2 = 0.001 * tf.reduce_mean(
tf.reduce_sum(
tf.contrib.layers.flatten(tf.abs(
(annotation - logits) * mask2)), 1))
annotation_ssim = (annotation + 1.0) * 255 / 2.0
logits_ssim = (logits + 1.0) * 255 / 2.0
# loss_ssim = -0.5*tf.reduce_mean(tf.image.ssim(annotation_ssim , logits_ssim,max_val = 255.0))
loss_ssim = -1.0 * tf.log(
tf.reduce_mean(
tf.image.ssim(
annotation_ssim * mask, logits_ssim * mask, max_val=255.0)))
loss_ = 0.4 * loss_mask + loss_mask2 + loss_ssim
#loss_ = loss_mask# + 0.07*loss_ssim
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss_)
# zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1))
zloss_ = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1))
# zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1))
# zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1))
# z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv
lip_loss_dec = 0.001 * tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(
(logits - logits_e))), 1))
loss = loss_ + 0.1 * lip_loss_dec
lip_loss_pred = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))),
1))
zloss = zloss_ + 0.1 * lip_loss_pred
grads = train_z(loss_, z)
trainable_var = tf.trainable_variables()
trainable_z_pred_var = tf.trainable_variables(scope="predictor")
trainable_d_pred_var = tf.trainable_variables(scope="decoder")
print(trainable_z_pred_var)
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
train_pred = train_predictor(zloss, trainable_z_pred_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
saved = True
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images_, train_annotations_ = train_dataset_reader.next_batch(
FLAGS.batch_size)
train_images = sess.run(image_, feed_dict={image: train_images_})
train_annotations = np.copy(train_images)
print(np.max(train_images))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
# train_images[train_images < 0.] = -1.
# train_annotations[train_annotations < 0.] = -1.
# train_images[train_images >= 0.] = 1.0
# train_annotations[train_annotations >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
cond = random.randint(0, 10)
# saved = True
if False:
saved = False
train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(
FLAGS.batch_size)
train_images_m[train_images_m < 0.] = -1.
train_annotations_m[train_annotations_m < 0.] = -1.
train_images_m[train_images_m >= 0.] = 1.0
train_annotations_m[train_annotations_m >= 0.] = 1.0
train_images = (train_images + 1.) / 2.0 * 255.0
train_annotations = (train_annotations + 1.) / 2.0 * 255.0
train_images_m = (train_images_m + 1.) / 2.0 * 255.0
train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0
train_images_m[:, 32:, :, :] = 0
train_annotations_m[:, 32:, :, :] = 0
train_images = np.clip((train_images + train_images_m), 0.0,
255.0)
train_annotations = np.clip(
(train_annotations + train_annotations_m), 0.0, 255.0)
'''
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
'''
train_annotations_ = np.squeeze(train_annotations, axis=3)
train_images_ = train_images
train_images = train_images / 127.5 - 1.0
train_annotations = train_annotations / 127.5 - 1.0
# for itr_ in range(FLAGS.batch_size):
# utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) )
# utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) )
# train_images[:,x1:w1,y1:h1,:] = 0
# print(train_images)
r_m, r_m2 = random_mask(64)
#feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True }
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
error_dec = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 4, 4, 522))
error_dec_ = np.random.normal(0.0, 0.001,
(FLAGS.batch_size, 1, 1, 8202))
# z_l_inv = z_ + np.random.normal(0.0,0.1)
# feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True }
# z_l = z_ + np.random.normal(0.0,0.001)
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# z_l = z_ + np.random.normal(0.0,0.001)
# print("Step: %d, lip_loss:%g" % (itr,lloss))
for p in range(20):
z_ol = np.copy(z_)
# z_l = z_ol + np.random.normal(0.0,0.001)
# print("666666666666666666666666666666666666666")
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_,
e: error_dec,
mask: r_m,
mask2: r_m2,
istrain: True
}
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss))
z_loss, l_mask, l_ssim, l_lip, summ = sess.run(
[loss, loss_mask, loss_ssim, lip_loss_dec, loss_summary],
feed_dict=feed_dict)
print(
"Step: %d, z_step: %d, Train_loss:%g, Loss_mask:%g, SSIM_loss:%g, lip_loss:%g, "
% (itr, p, z_loss, l_mask, l_ssim, l_lip))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -10.0, 10.0)
'''
m = interp1d([-10.0,10.0],[-1.0,1.0])
print(np.max(z_))
print(np.min(z_))
z_ol_interp = m(z_ol)
z_interp = m(z_)
_,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85})
print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
'''
# _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85})
# print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
mask: r_m,
e: error_dec,
z: z_,
mask2: r_m2,
istrain: True
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.
# valid_images[valid_images < 0.] = -1.
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
mask: r_m,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
e: error_dec,
istrain: False,
mask2: r_m2
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
if itr % 3000 == 0:
save_itr = save_itr + 3000
saver.save(sess, FLAGS.logs_dir + "model.ckpt", save_itr)
elif FLAGS.mode == "visualize":
k = 1
#or k in xrange(20):
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
# valid_images = sess.run(image_, feed_dict={image:valid_images_})
# valid_annotations = np.copy(valid_images)
# valid_annotations[valid_annotations < 0.] = -1.0
# valid_images[valid_images < 0.] = -1.0
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
r_m, r_m2 = random_mask(64)
# z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 0.85,
z: z_,
istrain: False,
mask: r_m,
mask2: r_m2
}
v = 0
# m__ = interp1d([-10.0,10.0],[-1.0,1.0])
# z_ = m__(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
# print(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
# z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict)
# print("z_step: %d, Train_loss:%g" % (p,z_loss))
# z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v
# z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# print("z_____________")
# print(z__)
# print("z_")
# print(z_)
# m__ = interp1d([-10.0,10.0],[-1.0,1.0])
# z_ol = m__(z_ol)
# z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m})
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
# z_ = np.clip(z_, -1.0, 1.0)
# print(z_pred_loss)
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
istrain: False,
mask: r_m,
mask2: r_m2,
keep_probability: 1.0
})
valid_images_masked = ((1 - r_m) * valid_images + 1.) / 2.0 * 255
# valid_images = (valid_images +1.)/2.0*255
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred_ = (pred + 1.) / 2.0 * 255
# pred = pred + 1./2.0*255
pred = valid_images_masked * (1 - r_m) + pred_ * r_m
valid_annotations_ = (valid_annotations + 1.) / 2.0 * 255
# pred = np.squeeze(pred, axis=3)
print(np.max(pred))
print(valid_images.shape)
print(valid_annotations.shape)
print(pred.shape)
# for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) )
# utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) )
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) )
# print("Saved image: %d" % itr)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images_masked[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(85 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="predz_" + str(85 + itr))
utils.save_image(valid_annotations_[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(85 + itr))
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z")
mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask")
z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new")
istrain = tf.placeholder(tf.bool)
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# mask_ = tf.ones([FLAGS.batch_size,32,64,3])
# mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
logits, h = inference((1 - mask) * image + mask * 0.0, keep_probability, z,
istrain)
z_pred = predictor(h, z, istrain)
# logits = inference(image, keep_probability,z,istrain)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
# loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
loss_mask = tf.reduce_mean(
tf.reduce_sum(
tf.contrib.layers.flatten(tf.abs((annotation - logits) * mask)),
1))
loss = loss_mask
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss)
# zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1))
zloss = tf.reduce_mean(
tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1))
grads = train_z(loss_mask, z)
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
train_pred = train_predictor(zloss, trainable_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
saved = True
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
print(np.max(train_images))
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 35)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
cond = random.randint(0, 10)
# saved = True
if cond <= 4 and saved:
saved = False
train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(
FLAGS.batch_size)
train_images_m[train_images_m < 0.] = -1.
train_annotations_m[train_annotations_m < 0.] = -1.
train_images_m[train_images_m >= 0.] = 1.0
train_annotations_m[train_annotations_m >= 0.] = 1.0
train_images = (train_images + 1.) / 2.0 * 255.0
train_annotations = (train_annotations + 1.) / 2.0 * 255.0
train_images_m = (train_images_m + 1.) / 2.0 * 255.0
train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0
train_images_m[:, 32:, :, :] = 0
train_annotations_m[:, 32:, :, :] = 0
train_images = np.clip((train_images + train_images_m), 0.0,
255.0)
train_annotations = np.clip(
(train_annotations + train_annotations_m), 0.0, 255.0)
'''
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
'''
train_annotations_ = np.squeeze(train_annotations, axis=3)
train_images_ = train_images
train_images = train_images / 127.5 - 1.0
train_annotations = train_annotations / 127.5 - 1.0
for itr_ in range(FLAGS.batch_size):
utils.save_image(train_images_[itr_].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr_))
utils.save_image(train_annotations_[itr_].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr_))
# train_images[:,x1:w1,y1:h1,:] = 0
# print(train_images)
r_m = random_mask(64)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 1.0,
z: z_,
mask: r_m,
istrain: True
}
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 1.0,
z: z_,
mask: r_m,
istrain: True
}
z_loss, summ = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -1.0, 1.0)
_, z_pred_loss = sess.run(
[train_pred, zloss],
feed_dict={
image: train_images,
mask: r_m,
z: z_ol,
z_new: z_,
istrain: True,
keep_probability: 1.0
})
print("Step: %d, z_step: %d, z_pred_loss:%g" %
(itr, p, z_pred_loss))
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 1.0,
mask: r_m,
z: z_,
istrain: True
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_annotations[valid_annotations < 0.] = -1.
valid_images[valid_images < 0.] = -1.
valid_annotations[valid_annotations >= 0.] = 1.0
valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
mask: random_mask(64),
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
istrain: False
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
saver.save(sess, FLAGS.logs_dir + "model_z_center.ckpt", 500)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.next_batch(
20)
valid_annotations[valid_annotations < 0.] = -1.0
valid_images[valid_images < 0.] = -1.0
valid_annotations[valid_annotations >= 0.] = 1.0
valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
r_m = random_mask(64)
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 1, 1, 10))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
istrain: False,
mask: r_m
}
v = 0
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
for p in range(10):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
# print(z_)
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_,
istrain: False,
mask: r_m
}
z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict)
print("z_step: %d, Train_loss:%g" % (p, z_loss))
# z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v
# z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# print("z_____________")
# print(z__)
# print("z_")
# print(z_)
# z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# z_ = np.clip(z_, -1.0, 1.0)
# print(z_pred_loss)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
istrain: False,
mask: r_m,
keep_probability: 1.0
})
valid_images_masked = ((1 - r_m) * valid_images + 1.) / 2.0 * 255
# valid_images = (valid_images +1.)/2.0*255
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred_ = (np.squeeze(pred, axis=3) + 1.) / 2.0 * 255
# pred = pred + 1./2.0*255
pred = valid_images_masked[:, :, :, 0] * (
1 - r_m)[:, :, :, 0] + pred_ * r_m[:, :, :, 0]
valid_annotations_ = (np.squeeze(valid_annotations, axis=3) +
1.) / 2.0 * 255
# pred = np.squeeze(pred, axis=3)
print(np.max(pred))
print(valid_images.shape)
print(valid_annotations.shape)
print(pred.shape)
# for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) )
# utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) )
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) )
# print("Saved image: %d" % itr)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images_masked[itr].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(pred[itr].astype(np.uint8),
FLAGS.logs_dir,
name="predz_" + str(5 + itr))
utils.save_image(valid_annotations_[itr].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 4, 4, 10], name="z")
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
mask_ = tf.ones([FLAGS.batch_size, 64, 64, 3])
mask = tf.pad(mask_, [[0, 0], [32, 32], [32, 32], [0, 0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
pred_annotation, logits = inference((1 - mask) * image + mask * 0.5,
keep_probability, z)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth",
tf.cast(annotation, tf.uint8),
max_outputs=2)
tf.summary.image("pred_annotation",
tf.cast(pred_annotation, tf.uint8),
max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
loss_all = tf.reduce_mean(tf.abs(image - logits))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
loss_mask = 0.8 * tf.reduce_mean(tf.abs((image - logits) * mask))
loss = loss_all + loss_mask
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss)
grads = train_z(loss_mask, z)
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
print(np.max(train_images))
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 4, 4, 10))
train_images[train_images < 0.5] = 0.0
train_annotations[train_annotations < 0.5] = 0.0
train_images[train_images >= 0.5] = 1.0
train_annotations[train_annotations >= 0.5] = 1.0
# print(train_images)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_
}
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
for p in range(10):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
z_loss, summ = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.85,
z: z_
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
FLAGS.batch_size)
valid_annotations[valid_annotations < 0.5] = 0.0
valid_images[valid_images < 0.5] = 0.0
valid_annotations[valid_annotations >= 0.5] = 1.0
valid_images[valid_images >= 0.5] = 1.0
valid_loss, summary_sva = sess.run(
[loss, loss_summary],
feed_dict={
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_
})
print("%s ---> Validation_loss: %g" %
(datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
saver.save(sess, FLAGS.logs_dir + "model_z_center.ckpt", 500)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
20)
valid_annotations[valid_annotations < 0.5] = 0.0
valid_images[valid_images < 0.5] = 0.0
valid_annotations[valid_annotations >= 0.5] = 1.0
valid_images[valid_images >= 0.5] = 1.0
z_ = np.random.uniform(low=-1.0,
high=1.0,
size=(FLAGS.batch_size, 4, 4, 10))
feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0,
z: z_
}
v = 0
for p in range(50):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict)
print("z_step: %d, Train_loss:%g" % (p, z_loss))
# print(z_)
g = sess.run([grads], feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001 * v - 0.1 * g[0][0]
z_ += 0.001 * v_prev + (1 + 0.001) * v
# z_ = np.clip(z_, -1.0, 1.0)
pred = sess.run(logits,
feed_dict={
image: valid_images,
annotation: valid_annotations,
z: z_,
keep_probability: 1.0
})
valid_images_masked = (1 - sess.run(mask)) * valid_images
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred = np.squeeze(pred, axis=3)
valid_annotations = np.squeeze(valid_annotations, axis=3)
# pred = np.squeeze(pred, axis=3)
print(valid_images.shape)
print(valid_annotations.shape)
print(pred.shape)
for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
utils.save_image(valid_images_masked[itr],
FLAGS.logs_dir,
name="inp_" + str(5 + itr))
utils.save_image(valid_annotations[itr],
FLAGS.logs_dir,
name="gt_" + str(5 + itr))
utils.save_image(pred[itr],
FLAGS.logs_dir,
name="predz_" + str(5 + itr))
print("Saved image: %d" % itr)
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name="input_image")
annotation = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 2], name="annotation")
z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z")
# mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask")
# mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2")
z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new")
istrain = tf.placeholder(tf.bool)
#z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip")
#z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv")
e = tf.placeholder(tf.float32, shape=[None, 4, 4, 522], name="e")
e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8202], name="e_p")
save_itr = 0
# pred_annotation, logits = inference(image, keep_probability,z)
# tf.summary.image("input_image", image, max_outputs=2)
# tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# mask_ = tf.ones([FLAGS.batch_size,32,64,3])
# mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]])
# mask2__ = tf.ones([FLAGS.batch_size,78,78,3])
# mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]])
# mask2 = mask2_ - mask
zero = tf.zeros([FLAGS.batch_size,1,1,8202])
logits, h = inference(image, keep_probability,z,0.0,istrain)
logits_e, h_e = inference(image, keep_probability,z,e,istrain)
#logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain )
#logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain )
z_pred = predictor(h,z,zero,istrain)
z_pred_e = predictor(h,z,e_p,istrain)
# z_pred_lip = predictor(h,z_lip,istrain)
# z_pred_lip_inv = predictor(h,z_lip_inv,istrain)
# logits = inference(image, keep_probability,z,istrain)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
# tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
# lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3]))
# lossz = 0.1 * tf.reduce_mean(tf.abs(z))
# loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3])))
# loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1))
# loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3])))
g_k = gaussian_kernel(2,0.0,1.0)
gauss_kernel = tf.tile(g_k[ :, :, tf.newaxis, tf.newaxis],[1,1,2,2])
# Convolve.
logits_smooth_ = tf.nn.conv2d(tf.abs(logits), gauss_kernel, strides=[1, 1, 1, 1], padding="SAME")
logits_smooth = tf.maximum(logits_smooth_,0.0001)
logits_smooth_norm = tf.contrib.layers.flatten(logits_smooth)/tf.reduce_sum(logits_smooth,axis=[1,2,3], keep_dims = True)
ones = tf.ones([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2])
zeros = tf.zeros([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2])
normal_dist_d = tf.distributions.Uniform(low = zeros, high = ones)
normal_dist_ = normal_dist_d.sample()
normal_dist = tf.maximum(normal_dist_,0.0001)
normal_dist_norm = tf.contrib.layers.flatten(normal_dist)/tf.reduce_sum(normal_dist,axis=[1,2,3], keep_dims = True)
X = tf.distributions.Categorical(probs=logits_smooth_norm)
Y = tf.distributions.Categorical(probs=normal_dist_norm)
kl_dist = tf.reduce_mean(tf.distributions.kl_divergence(X, Y))
# kl_dist = tf.reduce_sum(logits_smooth * tf.log(logits_smooth/normal_dist))
#kl_dist = tf.contrib.distributions.kl_divergence(logits_smooth_norm,normal_dist_norm)
# logits_std = tf.reduce_std(logits_smooth, axis =[1,2],keep_dims=True )
# logits_mean = tf.reduce_mean(logits_smooth, axis =[1,2], keep_dims=True)
# logits_normalized = (logits_smooth - logits_mean)/logits_std
# annotation_weights_norm = tf.reduce_sum(tf.exp(tf.abs(annotation))/tf.exp(1.0))
# annotation_weights = (tf.exp(tf.abs(annotation))/tf.exp(1.0))
loss_ = 0.2*tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((annotation - logits)) ),1)) + kl_dist
# loss_ = 0.4*loss_mask + loss_mask2
# loss = tf.reduce_mean(tf.squared_difference(logits ,annotation ))
loss_summary = tf.summary.scalar("entropy", loss_)
# zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1))
zloss_ = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))),1))
# zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1))
# zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1))
# z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv
lip_loss_dec = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((logits - logits_e))),1))
loss = loss_ + 0.1*lip_loss_dec
lip_loss_pred = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))),1))
zloss = zloss_ + 0.1*lip_loss_pred
grads = train_z(loss_,z)
trainable_var = tf.trainable_variables()
trainable_z_pred_var = tf.trainable_variables(scope="predictor")
trainable_d_pred_var = tf.trainable_variables(scope="decoder")
print(trainable_z_pred_var)
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
train_pred = train_predictor(zloss,trainable_z_pred_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
# if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records, image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records, image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
saved =True
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(FLAGS.batch_size)
print("$$$$$$$$$$$$$$$$$$$$")
print(train_images.shape)
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
# train_images[train_images < 0.] = -1.
# train_annotations[train_annotations < 0.] = -1.
# train_images[train_images >= 0.] = 1.0
# train_annotations[train_annotations >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
cond = random.randint(0, 10)
# saved = True
if False:
saved = False
train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(FLAGS.batch_size)
train_images_m[train_images_m < 0.] = -1.
train_annotations_m[train_annotations_m < 0.] = -1.
train_images_m[train_images_m >= 0.] = 1.0
train_annotations_m[train_annotations_m >= 0.] = 1.0
train_images = (train_images + 1.)/2.0*255.0
train_annotations = (train_annotations + 1.)/2.0*255.0
train_images_m = (train_images_m + 1.)/2.0*255.0
train_annotations_m = (train_annotations_m + 1.)/2.0*255.0
train_images_m[:,32:,:,:] = 0
train_annotations_m[:,32:,:,:] = 0
train_images = np.clip((train_images + train_images_m),0.0,255.0)
train_annotations = np.clip((train_annotations + train_annotations_m),0.0,255.0)
'''
train_images[train_images < 0.] = -1.
train_annotations[train_annotations < 0.] = -1.
train_images[train_images >= 0.] = 1.0
train_annotations[train_annotations >= 0.] = 1.0
'''
train_annotations_ = np.squeeze(train_annotations,axis = 3)
train_images_ = train_images
train_images = train_images/127.5 - 1.0
train_annotations = train_annotations/127.5 - 1.0
# for itr_ in range(FLAGS.batch_size):
# utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) )
# utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) )
# train_images[:,x1:w1,y1:h1,:] = 0
# print(train_images)
r_m, r_m2 = random_mask(64)
#feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True }
#train_images[:,50:100,50:100,:] =0
v = 0
# print(train_images)
error_dec = np.random.normal(0.0,0.001,(FLAGS.batch_size,4,4,522))
error_dec_ = np.random.normal(0.0,0.001,(FLAGS.batch_size,1,1,8202))
# z_l_inv = z_ + np.random.normal(0.0,0.1)
# feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True }
# z_l = z_ + np.random.normal(0.0,0.001)
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# z_l = z_ + np.random.normal(0.0,0.001)
# print("Step: %d, lip_loss:%g" % (itr,lloss))
for p in range(20):
z_ol = np.copy(z_)
# z_l = z_ol + np.random.normal(0.0,0.001)
# print("666666666666666666666666666666666666666")
feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,e:error_dec, istrain:True }
# lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict)
# print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss))
z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict)
print("Step: %d, z_step: %d, Train_loss:%g" % (itr,p,z_loss))
# print(z_)
g = sess.run([grads],feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001*v - 0.1*g[0][0]
z_ += 0.001 * v_prev + (1+0.001)*v
#z_ = np.clip(z_, -10.0, 10.0)
'''
m = interp1d([-10.0,10.0],[-1.0,1.0])
print(np.max(z_))
print(np.min(z_))
z_ol_interp = m(z_ol)
z_interp = m(z_)
_,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85})
print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
'''
# _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85})
# print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss))
# z_ = np.clip(z_, -1.0, 1.0)
# print(v.shape)
# print(z_.shape)
feed_dict = {image: train_images, annotation: train_annotations, keep_probability:0.85,e:error_dec, z: z_, istrain:True }
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary], feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.
# valid_images[valid_images < 0.] = -1.
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
valid_loss, summary_sva = sess.run([loss, loss_summary], feed_dict={image: valid_images, annotation: valid_annotations,
keep_probability: 1.0, z: z_,e:error_dec, istrain:False })
print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
# add validation loss to TensorBoard
validation_writer.add_summary(summary_sva, itr)
if itr % 3000 == 0:
save_itr = save_itr + 3000
saver.save(sess, FLAGS.logs_dir + "model_fuse.ckpt", save_itr)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = train_dataset_reader.get_random_batch(FLAGS.batch_size)
# valid_annotations[valid_annotations < 0.] = -1.0
# valid_images[valid_images < 0.] = -1.0
# valid_annotations[valid_annotations >= 0.] = 1.0
# valid_images[valid_images >= 0.] = 1.0
x1 = random.randint(0, 10)
w1 = random.randint(30, 54)
y1 = random.randint(0, 10)
h1 = random.randint(30, 54)
# valid_images[:,x1:w1,y1:h1,:] = 0
r_m, r_m2 = random_mask(64)
# z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
# z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5
z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10))
feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False }
v= 0
m__ = interp1d([-10.0,10.0],[-1.0,1.0])
z_ = m__(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
for p in range(20):
z_ol = np.copy(z_)
# print("666666666666666666666666666666666666666")
# print(z_)
# feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m }
# z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict)
# print("z_step: %d, Train_loss:%g" % (p,z_loss))
# z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m})
# print(z_)
g = sess.run([grads],feed_dict=feed_dict)
v_prev = np.copy(v)
# print(g[0][0].shape)
v = 0.001*v - 0.1*g[0][0]
z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# z_ = z_ol + 0.001 * v_prev + (1+0.001)*v
# print("z_____________")
# print(z__)
# print("z_")
# print(z_)
# m__ = interp1d([-10.0,10.0],[-1.0,1.0])
# z_ol = m__(z_ol)
# z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m})
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
# z_ = np.clip(z_, -1.0, 1.0)
# print(z_pred_loss)
# m_ = interp1d([-1.0,1.0],[-10.0,10.0])
# z_ = m_(z_)
pred = sess.run(logits, feed_dict={image: valid_images, annotation: valid_annotations,z:z_, istrain:False,
keep_probability: 0.85})
valid_images = (valid_images +1.)/2.0*100.0
# predicted_patch = sess.run(mask) * pred
# pred = valid_images_masked + predicted_patch
pred_ = pred * 128.0
# pred = pred + 1./2.0*255
print(np.max(pred_))
print(np.min(pred_))
pred = np.reshape(np.concatenate((valid_images,pred_), axis=3),(-1,64,64,3))
valid_annotations_ = valid_annotations * 128.0
valid_annotations = np.reshape(np.concatenate((valid_images, valid_annotations_), axis=3),(-1,64,64,3))
valid_images_gray = np.squeeze(valid_images)
# for itr in range(FLAGS.batch_size):
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
# utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr))
# utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) )
# utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) )
# utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) )
# print("Saved image: %d" % itr)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images_gray[itr], FLAGS.logs_dir, name="inp_" + str(5+itr) )
utils.save_image(color.lab2rgb(pred[itr]), FLAGS.logs_dir, name="predz_" + str(5+itr) )
utils.save_image(color.lab2rgb(valid_annotations[itr]), FLAGS.logs_dir, name="gt_" + str(5+itr) )
