def main(argv=None): keep_probability = tf.placeholder(tf.float32, name="keep_probabilty") image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3], name="input_image") annotation = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3], name="annotation") z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z") mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask") mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2") z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new") istrain = tf.placeholder(tf.bool) image_ = tf.image.random_flip_up_down(image) image_ = tf.image.random_flip_left_right(image_) image_ = tf.image.resize_with_crop_or_pad(image_, 80, 80) image_ = tf.image.random_crop(image_, size=(FLAGS.batch_size, 64, 64, 3)) image_ = tf.image.rot90(image_, k=tf.random_uniform(shape=[], minval=0, maxval=4, dtype=tf.int32)) #image_ = tf.image.pad_to_bounding_box(image_, 16,16,96,96) #z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip") #z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv") e = tf.placeholder(tf.float32, shape=[None, 4, 4, 522], name="e") e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8202], name="e_p") save_itr = 0 # pred_annotation, logits = inference(image, keep_probability,z) # tf.summary.image("input_image", image, max_outputs=2) # tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, # labels=tf.squeeze(annotation, squeeze_dims=[3]), # name="entropy"))) # mask_ = tf.ones([FLAGS.batch_size,32,64,3]) # mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]]) # mask2__ = tf.ones([FLAGS.batch_size,78,78,3]) # mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]]) # mask2 = mask2_ - mask zero = tf.zeros([FLAGS.batch_size, 1, 1, 8202]) logits, h = inference((1 - mask) * image + mask * -2.0, keep_probability, z, 0.0, istrain) logits_e, h_e = inference((1 - mask) * image + mask * -2.0, keep_probability, z, e, istrain) #logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain ) #logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain ) z_pred = predictor(h, z, zero, istrain) z_pred_e = predictor(h, z, e_p, istrain) # z_pred_lip = predictor(h,z_lip,istrain) # z_pred_lip_inv = predictor(h,z_lip_inv,istrain) # logits = inference(image, keep_probability,z,istrain) tf.summary.image("input_image", image, max_outputs=2) tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3])) # lossz = 0.1 * tf.reduce_mean(tf.abs(z)) # loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3]))) # loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1)) # loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3]))) loss_mask = 0.001 * tf.reduce_mean( tf.reduce_sum( tf.contrib.layers.flatten(tf.abs( (annotation - logits) * mask)), 1)) loss_mask2 = 0.001 * tf.reduce_mean( tf.reduce_sum( tf.contrib.layers.flatten(tf.abs( (annotation - logits) * mask2)), 1)) annotation_ssim = (annotation + 1.0) * 255 / 2.0 logits_ssim = (logits + 1.0) * 255 / 2.0 # loss_ssim = -0.5*tf.reduce_mean(tf.image.ssim(annotation_ssim , logits_ssim,max_val = 255.0)) loss_ssim = -1.0 * tf.log( tf.reduce_mean( tf.image.ssim( annotation_ssim * mask, logits_ssim * mask, max_val=255.0))) loss_ = 0.4 * loss_mask + loss_mask2 + loss_ssim #loss_ = loss_mask# + 0.07*loss_ssim # loss = tf.reduce_mean(tf.squared_difference(logits ,annotation )) loss_summary = tf.summary.scalar("entropy", loss_) # zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1)) zloss_ = tf.reduce_mean( tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1)) # zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1)) # zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1)) # z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv lip_loss_dec = 0.001 * tf.reduce_mean( tf.reduce_sum(tf.contrib.layers.flatten(tf.abs( (logits - logits_e))), 1)) loss = loss_ + 0.1 * lip_loss_dec lip_loss_pred = tf.reduce_mean( tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))), 1)) zloss = zloss_ + 0.1 * lip_loss_pred grads = train_z(loss_, z) trainable_var = tf.trainable_variables() trainable_z_pred_var = tf.trainable_variables(scope="predictor") trainable_d_pred_var = tf.trainable_variables(scope="decoder") print(trainable_z_pred_var) if FLAGS.debug: for var in trainable_var: utils.add_to_regularization_and_summary(var) train_op = train(loss, trainable_var) train_pred = train_predictor(zloss, trainable_z_pred_var) print("Setting up summary op...") summary_op = tf.summary.merge_all() print("Setting up image reader...") train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir) print(len(train_records)) print(len(valid_records)) print("Setting up dataset reader") image_options = {'resize': True, 'resize_size': IMAGE_SIZE} if FLAGS.mode == 'train': train_dataset_reader = dataset.BatchDatset(train_records, image_options) validation_dataset_reader = dataset.BatchDatset(valid_records, image_options) sess = tf.Session() print("Setting up Saver...") saver = tf.train.Saver() # create two summary writers to show training loss and validation loss in the same graph # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation') sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") saved = True if FLAGS.mode == "train": for itr in xrange(MAX_ITERATION): train_images_, train_annotations_ = train_dataset_reader.next_batch( FLAGS.batch_size) train_images = sess.run(image_, feed_dict={image: train_images_}) train_annotations = np.copy(train_images) print(np.max(train_images)) # z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5 z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 1, 1, 10)) # train_images[train_images < 0.] = -1. # train_annotations[train_annotations < 0.] = -1. # train_images[train_images >= 0.] = 1.0 # train_annotations[train_annotations >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) cond = random.randint(0, 10) # saved = True if False: saved = False train_images_m, train_annotations_m = train_dataset_reader.get_random_batch( FLAGS.batch_size) train_images_m[train_images_m < 0.] = -1. train_annotations_m[train_annotations_m < 0.] = -1. train_images_m[train_images_m >= 0.] = 1.0 train_annotations_m[train_annotations_m >= 0.] = 1.0 train_images = (train_images + 1.) / 2.0 * 255.0 train_annotations = (train_annotations + 1.) / 2.0 * 255.0 train_images_m = (train_images_m + 1.) / 2.0 * 255.0 train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0 train_images_m[:, 32:, :, :] = 0 train_annotations_m[:, 32:, :, :] = 0 train_images = np.clip((train_images + train_images_m), 0.0, 255.0) train_annotations = np.clip( (train_annotations + train_annotations_m), 0.0, 255.0) ''' train_images[train_images < 0.] = -1. train_annotations[train_annotations < 0.] = -1. train_images[train_images >= 0.] = 1.0 train_annotations[train_annotations >= 0.] = 1.0 ''' train_annotations_ = np.squeeze(train_annotations, axis=3) train_images_ = train_images train_images = train_images / 127.5 - 1.0 train_annotations = train_annotations / 127.5 - 1.0 # for itr_ in range(FLAGS.batch_size): # utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) ) # utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) ) # train_images[:,x1:w1,y1:h1,:] = 0 # print(train_images) r_m, r_m2 = random_mask(64) #feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True } #train_images[:,50:100,50:100,:] =0 v = 0 # print(train_images) error_dec = np.random.normal(0.0, 0.001, (FLAGS.batch_size, 4, 4, 522)) error_dec_ = np.random.normal(0.0, 0.001, (FLAGS.batch_size, 1, 1, 8202)) # z_l_inv = z_ + np.random.normal(0.0,0.1) # feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True } # z_l = z_ + np.random.normal(0.0,0.001) # lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict) # z_l = z_ + np.random.normal(0.0,0.001) # print("Step: %d, lip_loss:%g" % (itr,lloss)) for p in range(20): z_ol = np.copy(z_) # z_l = z_ol + np.random.normal(0.0,0.001) # print("666666666666666666666666666666666666666") feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e: error_dec, mask: r_m, mask2: r_m2, istrain: True } # lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict) # print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss)) z_loss, l_mask, l_ssim, l_lip, summ = sess.run( [loss, loss_mask, loss_ssim, lip_loss_dec, loss_summary], feed_dict=feed_dict) print( "Step: %d, z_step: %d, Train_loss:%g, Loss_mask:%g, SSIM_loss:%g, lip_loss:%g, " % (itr, p, z_loss, l_mask, l_ssim, l_lip)) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ += 0.001 * v_prev + (1 + 0.001) * v # z_ = np.clip(z_, -10.0, 10.0) ''' m = interp1d([-10.0,10.0],[-1.0,1.0]) print(np.max(z_)) print(np.min(z_)) z_ol_interp = m(z_ol) z_interp = m(z_) _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85}) print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss)) ''' # _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85}) # print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss)) # z_ = np.clip(z_, -1.0, 1.0) # print(v.shape) # print(z_.shape) feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 0.85, mask: r_m, e: error_dec, z: z_, mask2: r_m2, istrain: True } sess.run(train_op, feed_dict=feed_dict) if itr % 10 == 0: train_loss, summary_str = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, Train_loss:%g" % (itr, train_loss)) train_writer.add_summary(summary_str, itr) if itr % 500 == 0: valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) # valid_annotations[valid_annotations < 0.] = -1. # valid_images[valid_images < 0.] = -1. # valid_annotations[valid_annotations >= 0.] = 1.0 # valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 valid_loss, summary_sva = sess.run( [loss, loss_summary], feed_dict={ image: valid_images, mask: r_m, annotation: valid_annotations, keep_probability: 1.0, z: z_, e: error_dec, istrain: False, mask2: r_m2 }) print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss)) # add validation loss to TensorBoard validation_writer.add_summary(summary_sva, itr) if itr % 3000 == 0: save_itr = save_itr + 3000 saver.save(sess, FLAGS.logs_dir + "model.ckpt", save_itr) elif FLAGS.mode == "visualize": k = 1 #or k in xrange(20): valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) # valid_images = sess.run(image_, feed_dict={image:valid_images_}) # valid_annotations = np.copy(valid_images) # valid_annotations[valid_annotations < 0.] = -1.0 # valid_images[valid_images < 0.] = -1.0 # valid_annotations[valid_annotations >= 0.] = 1.0 # valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 r_m, r_m2 = random_mask(64) # z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10)) # z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5 z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 1, 1, 10)) feed_dict = { image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain: False, mask: r_m, mask2: r_m2 } v = 0 # m__ = interp1d([-10.0,10.0],[-1.0,1.0]) # z_ = m__(z_) # feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m } for p in range(20): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") # print(z_) # feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m } # z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict) # print("z_step: %d, Train_loss:%g" % (p,z_loss)) # z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m}) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v # z_ = z_ol + 0.001 * v_prev + (1+0.001)*v # print("z_____________") # print(z__) # print("z_") # print(z_) # m__ = interp1d([-10.0,10.0],[-1.0,1.0]) # z_ol = m__(z_ol) # z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m}) # m_ = interp1d([-1.0,1.0],[-10.0,10.0]) # z_ = m_(z_) # z_ = np.clip(z_, -1.0, 1.0) # print(z_pred_loss) # m_ = interp1d([-1.0,1.0],[-10.0,10.0]) # z_ = m_(z_) pred = sess.run(logits, feed_dict={ image: valid_images, annotation: valid_annotations, z: z_, istrain: False, mask: r_m, mask2: r_m2, keep_probability: 1.0 }) valid_images_masked = ((1 - r_m) * valid_images + 1.) / 2.0 * 255 # valid_images = (valid_images +1.)/2.0*255 # predicted_patch = sess.run(mask) * pred # pred = valid_images_masked + predicted_patch pred_ = (pred + 1.) / 2.0 * 255 # pred = pred + 1./2.0*255 pred = valid_images_masked * (1 - r_m) + pred_ * r_m valid_annotations_ = (valid_annotations + 1.) / 2.0 * 255 # pred = np.squeeze(pred, axis=3) print(np.max(pred)) print(valid_images.shape) print(valid_annotations.shape) print(pred.shape) # for itr in range(FLAGS.batch_size): # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)) # utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)) # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) ) # utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) ) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) ) # print("Saved image: %d" % itr) for itr in range(FLAGS.batch_size): utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(85 + itr)) utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(85 + itr)) utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(85 + itr))
def main(argv=None): keep_probability = tf.placeholder(tf.float32, name="keep_probabilty") image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3], name="input_image") annotation = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name="annotation") z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z") mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask") z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new") istrain = tf.placeholder(tf.bool) # pred_annotation, logits = inference(image, keep_probability,z) # tf.summary.image("input_image", image, max_outputs=2) # tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, # labels=tf.squeeze(annotation, squeeze_dims=[3]), # name="entropy"))) # mask_ = tf.ones([FLAGS.batch_size,32,64,3]) # mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]]) # mask2__ = tf.ones([FLAGS.batch_size,78,78,3]) # mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]]) # mask2 = mask2_ - mask logits, h = inference((1 - mask) * image + mask * 0.0, keep_probability, z, istrain) z_pred = predictor(h, z, istrain) # logits = inference(image, keep_probability,z,istrain) tf.summary.image("input_image", image, max_outputs=2) tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3])) # lossz = 0.1 * tf.reduce_mean(tf.abs(z)) # loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3]))) # loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1)) # loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3]))) loss_mask = tf.reduce_mean( tf.reduce_sum( tf.contrib.layers.flatten(tf.abs((annotation - logits) * mask)), 1)) loss = loss_mask # loss = tf.reduce_mean(tf.squared_difference(logits ,annotation )) loss_summary = tf.summary.scalar("entropy", loss) # zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1)) zloss = tf.reduce_mean( tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))), 1)) grads = train_z(loss_mask, z) trainable_var = tf.trainable_variables() if FLAGS.debug: for var in trainable_var: utils.add_to_regularization_and_summary(var) train_op = train(loss, trainable_var) train_pred = train_predictor(zloss, trainable_var) print("Setting up summary op...") summary_op = tf.summary.merge_all() print("Setting up image reader...") train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir) print(len(train_records)) print(len(valid_records)) print("Setting up dataset reader") image_options = {'resize': True, 'resize_size': IMAGE_SIZE} if FLAGS.mode == 'train': train_dataset_reader = dataset.BatchDatset(train_records, image_options) validation_dataset_reader = dataset.BatchDatset(valid_records, image_options) sess = tf.Session() print("Setting up Saver...") saver = tf.train.Saver() # create two summary writers to show training loss and validation loss in the same graph # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation') sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") saved = True if FLAGS.mode == "train": for itr in xrange(MAX_ITERATION): train_images, train_annotations = train_dataset_reader.next_batch( FLAGS.batch_size) print(np.max(train_images)) z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 1, 1, 10)) train_images[train_images < 0.] = -1. train_annotations[train_annotations < 0.] = -1. train_images[train_images >= 0.] = 1.0 train_annotations[train_annotations >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 35) y1 = random.randint(0, 10) h1 = random.randint(30, 54) cond = random.randint(0, 10) # saved = True if cond <= 4 and saved: saved = False train_images_m, train_annotations_m = train_dataset_reader.get_random_batch( FLAGS.batch_size) train_images_m[train_images_m < 0.] = -1. train_annotations_m[train_annotations_m < 0.] = -1. train_images_m[train_images_m >= 0.] = 1.0 train_annotations_m[train_annotations_m >= 0.] = 1.0 train_images = (train_images + 1.) / 2.0 * 255.0 train_annotations = (train_annotations + 1.) / 2.0 * 255.0 train_images_m = (train_images_m + 1.) / 2.0 * 255.0 train_annotations_m = (train_annotations_m + 1.) / 2.0 * 255.0 train_images_m[:, 32:, :, :] = 0 train_annotations_m[:, 32:, :, :] = 0 train_images = np.clip((train_images + train_images_m), 0.0, 255.0) train_annotations = np.clip( (train_annotations + train_annotations_m), 0.0, 255.0) ''' train_images[train_images < 0.] = -1. train_annotations[train_annotations < 0.] = -1. train_images[train_images >= 0.] = 1.0 train_annotations[train_annotations >= 0.] = 1.0 ''' train_annotations_ = np.squeeze(train_annotations, axis=3) train_images_ = train_images train_images = train_images / 127.5 - 1.0 train_annotations = train_annotations / 127.5 - 1.0 for itr_ in range(FLAGS.batch_size): utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5 + itr_)) utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5 + itr_)) # train_images[:,x1:w1,y1:h1,:] = 0 # print(train_images) r_m = random_mask(64) feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 1.0, z: z_, mask: r_m, istrain: True } #train_images[:,50:100,50:100,:] =0 v = 0 # print(train_images) for p in range(20): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 1.0, z: z_, mask: r_m, istrain: True } z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss)) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ += 0.001 * v_prev + (1 + 0.001) * v # z_ = np.clip(z_, -1.0, 1.0) _, z_pred_loss = sess.run( [train_pred, zloss], feed_dict={ image: train_images, mask: r_m, z: z_ol, z_new: z_, istrain: True, keep_probability: 1.0 }) print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr, p, z_pred_loss)) # z_ = np.clip(z_, -1.0, 1.0) # print(v.shape) # print(z_.shape) feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 1.0, mask: r_m, z: z_, istrain: True } sess.run(train_op, feed_dict=feed_dict) if itr % 10 == 0: train_loss, summary_str = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, Train_loss:%g" % (itr, train_loss)) train_writer.add_summary(summary_str, itr) if itr % 500 == 0: valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_annotations[valid_annotations < 0.] = -1. valid_images[valid_images < 0.] = -1. valid_annotations[valid_annotations >= 0.] = 1.0 valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 valid_loss, summary_sva = sess.run( [loss, loss_summary], feed_dict={ image: valid_images, mask: random_mask(64), annotation: valid_annotations, keep_probability: 1.0, z: z_, istrain: False }) print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss)) # add validation loss to TensorBoard validation_writer.add_summary(summary_sva, itr) saver.save(sess, FLAGS.logs_dir + "model_z_center.ckpt", 500) elif FLAGS.mode == "visualize": valid_images, valid_annotations = validation_dataset_reader.next_batch( 20) valid_annotations[valid_annotations < 0.] = -1.0 valid_images[valid_images < 0.] = -1.0 valid_annotations[valid_annotations >= 0.] = 1.0 valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 r_m = random_mask(64) z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 1, 1, 10)) feed_dict = { image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z: z_, istrain: False, mask: r_m } v = 0 # feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m } for p in range(10): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") # print(z_) feed_dict = { image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z: z_, istrain: False, mask: r_m } z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict) print("z_step: %d, Train_loss:%g" % (p, z_loss)) # z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m}) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ = z_ol + 0.001 * v_prev + (1 + 0.001) * v # z_ = z_ol + 0.001 * v_prev + (1+0.001)*v # print("z_____________") # print(z__) # print("z_") # print(z_) # z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m}) # z_ = np.clip(z_, -1.0, 1.0) # print(z_pred_loss) pred = sess.run(logits, feed_dict={ image: valid_images, annotation: valid_annotations, z: z_, istrain: False, mask: r_m, keep_probability: 1.0 }) valid_images_masked = ((1 - r_m) * valid_images + 1.) / 2.0 * 255 # valid_images = (valid_images +1.)/2.0*255 # predicted_patch = sess.run(mask) * pred # pred = valid_images_masked + predicted_patch pred_ = (np.squeeze(pred, axis=3) + 1.) / 2.0 * 255 # pred = pred + 1./2.0*255 pred = valid_images_masked[:, :, :, 0] * ( 1 - r_m)[:, :, :, 0] + pred_ * r_m[:, :, :, 0] valid_annotations_ = (np.squeeze(valid_annotations, axis=3) + 1.) / 2.0 * 255 # pred = np.squeeze(pred, axis=3) print(np.max(pred)) print(valid_images.shape) print(valid_annotations.shape) print(pred.shape) # for itr in range(FLAGS.batch_size): # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)) # utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)) # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) ) # utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) ) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) ) # print("Saved image: %d" % itr) for itr in range(FLAGS.batch_size): utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5 + itr)) utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5 + itr)) utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5 + itr))
def main(argv=None): keep_probability = tf.placeholder(tf.float32, name="keep_probabilty") image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3], name="input_image") annotation = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name="annotation") z = tf.placeholder(tf.float32, shape=[None, 4, 4, 10], name="z") # pred_annotation, logits = inference(image, keep_probability,z) # tf.summary.image("input_image", image, max_outputs=2) # tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, # labels=tf.squeeze(annotation, squeeze_dims=[3]), # name="entropy"))) mask_ = tf.ones([FLAGS.batch_size, 64, 64, 3]) mask = tf.pad(mask_, [[0, 0], [32, 32], [32, 32], [0, 0]]) # mask2__ = tf.ones([FLAGS.batch_size,78,78,3]) # mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]]) # mask2 = mask2_ - mask pred_annotation, logits = inference((1 - mask) * image + mask * 0.5, keep_probability, z) tf.summary.image("input_image", image, max_outputs=2) tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3])) # lossz = 0.1 * tf.reduce_mean(tf.abs(z)) # loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3]))) loss_all = tf.reduce_mean(tf.abs(image - logits)) # loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3]))) loss_mask = 0.8 * tf.reduce_mean(tf.abs((image - logits) * mask)) loss = loss_all + loss_mask # loss = tf.reduce_mean(tf.squared_difference(logits ,annotation )) loss_summary = tf.summary.scalar("entropy", loss) grads = train_z(loss_mask, z) trainable_var = tf.trainable_variables() if FLAGS.debug: for var in trainable_var: utils.add_to_regularization_and_summary(var) train_op = train(loss, trainable_var) print("Setting up summary op...") summary_op = tf.summary.merge_all() print("Setting up image reader...") train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir) print(len(train_records)) print(len(valid_records)) print("Setting up dataset reader") image_options = {'resize': True, 'resize_size': IMAGE_SIZE} if FLAGS.mode == 'train': train_dataset_reader = dataset.BatchDatset(train_records, image_options) validation_dataset_reader = dataset.BatchDatset(valid_records, image_options) sess = tf.Session() print("Setting up Saver...") saver = tf.train.Saver() # create two summary writers to show training loss and validation loss in the same graph # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation') sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") if FLAGS.mode == "train": for itr in xrange(MAX_ITERATION): train_images, train_annotations = train_dataset_reader.next_batch( FLAGS.batch_size) print(np.max(train_images)) z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 4, 4, 10)) train_images[train_images < 0.5] = 0.0 train_annotations[train_annotations < 0.5] = 0.0 train_images[train_images >= 0.5] = 1.0 train_annotations[train_annotations >= 0.5] = 1.0 # print(train_images) feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_ } #train_images[:,50:100,50:100,:] =0 v = 0 # print(train_images) for p in range(10): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss)) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ += 0.001 * v_prev + (1 + 0.001) * v # z_ = np.clip(z_, -1.0, 1.0) # print(v.shape) # print(z_.shape) feed_dict = { image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_ } sess.run(train_op, feed_dict=feed_dict) if itr % 10 == 0: train_loss, summary_str = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, Train_loss:%g" % (itr, train_loss)) train_writer.add_summary(summary_str, itr) if itr % 500 == 0: valid_images, valid_annotations = validation_dataset_reader.next_batch( FLAGS.batch_size) valid_annotations[valid_annotations < 0.5] = 0.0 valid_images[valid_images < 0.5] = 0.0 valid_annotations[valid_annotations >= 0.5] = 1.0 valid_images[valid_images >= 0.5] = 1.0 valid_loss, summary_sva = sess.run( [loss, loss_summary], feed_dict={ image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z: z_ }) print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss)) # add validation loss to TensorBoard validation_writer.add_summary(summary_sva, itr) saver.save(sess, FLAGS.logs_dir + "model_z_center.ckpt", 500) elif FLAGS.mode == "visualize": valid_images, valid_annotations = validation_dataset_reader.get_random_batch( 20) valid_annotations[valid_annotations < 0.5] = 0.0 valid_images[valid_images < 0.5] = 0.0 valid_annotations[valid_annotations >= 0.5] = 1.0 valid_images[valid_images >= 0.5] = 1.0 z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size, 4, 4, 10)) feed_dict = { image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z: z_ } v = 0 for p in range(50): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict) print("z_step: %d, Train_loss:%g" % (p, z_loss)) # print(z_) g = sess.run([grads], feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001 * v - 0.1 * g[0][0] z_ += 0.001 * v_prev + (1 + 0.001) * v # z_ = np.clip(z_, -1.0, 1.0) pred = sess.run(logits, feed_dict={ image: valid_images, annotation: valid_annotations, z: z_, keep_probability: 1.0 }) valid_images_masked = (1 - sess.run(mask)) * valid_images # predicted_patch = sess.run(mask) * pred # pred = valid_images_masked + predicted_patch pred = np.squeeze(pred, axis=3) valid_annotations = np.squeeze(valid_annotations, axis=3) # pred = np.squeeze(pred, axis=3) print(valid_images.shape) print(valid_annotations.shape) print(pred.shape) for itr in range(FLAGS.batch_size): # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)) # utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)) utils.save_image(valid_images_masked[itr], FLAGS.logs_dir, name="inp_" + str(5 + itr)) utils.save_image(valid_annotations[itr], FLAGS.logs_dir, name="gt_" + str(5 + itr)) utils.save_image(pred[itr], FLAGS.logs_dir, name="predz_" + str(5 + itr)) print("Saved image: %d" % itr)
def main(argv=None): keep_probability = tf.placeholder(tf.float32, name="keep_probabilty") image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name="input_image") annotation = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 2], name="annotation") z = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z") # mask = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask") # mask2 = tf.placeholder(tf.float32, shape=[None, 64, 64, 1], name="mask2") z_new = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_new") istrain = tf.placeholder(tf.bool) #z_lip = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip") #z_lip_inv = tf.placeholder(tf.float32, shape=[None, 1, 1, 10], name="z_lip_inv") e = tf.placeholder(tf.float32, shape=[None, 4, 4, 522], name="e") e_p = tf.placeholder(tf.float32, shape=[None, 1, 1, 8202], name="e_p") save_itr = 0 # pred_annotation, logits = inference(image, keep_probability,z) # tf.summary.image("input_image", image, max_outputs=2) # tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, # labels=tf.squeeze(annotation, squeeze_dims=[3]), # name="entropy"))) # mask_ = tf.ones([FLAGS.batch_size,32,64,3]) # mask = tf.pad(mask_, [[0,0],[0,32],[0,0],[0,0]]) # mask2__ = tf.ones([FLAGS.batch_size,78,78,3]) # mask2_ = tf.pad(mask2__, [[0,0],[25,25],[25,25],[0,0]]) # mask2 = mask2_ - mask zero = tf.zeros([FLAGS.batch_size,1,1,8202]) logits, h = inference(image, keep_probability,z,0.0,istrain) logits_e, h_e = inference(image, keep_probability,z,e,istrain) #logits_lip,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip,istrain ) #logits_lip_inv,_ = inference((1-mask)*image + mask*0.0, keep_probability,z_lip_inv,istrain ) z_pred = predictor(h,z,zero,istrain) z_pred_e = predictor(h,z,e_p,istrain) # z_pred_lip = predictor(h,z_lip,istrain) # z_pred_lip_inv = predictor(h,z_lip_inv,istrain) # logits = inference(image, keep_probability,z,istrain) tf.summary.image("input_image", image, max_outputs=2) tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2) # tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2) # lossz = 0.1 * tf.reduce_mean(tf.reduce_sum(tf.abs(z),[1,2,3])) # lossz = 0.1 * tf.reduce_mean(tf.abs(z)) # loss_all = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)),[1,2,3]))) # loss_all = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs(image - logits)),1)) # loss_mask = 0.8*tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square((image - logits)*mask),[1,2,3]))) g_k = gaussian_kernel(2,0.0,1.0) gauss_kernel = tf.tile(g_k[ :, :, tf.newaxis, tf.newaxis],[1,1,2,2]) # Convolve. logits_smooth_ = tf.nn.conv2d(tf.abs(logits), gauss_kernel, strides=[1, 1, 1, 1], padding="SAME") logits_smooth = tf.maximum(logits_smooth_,0.0001) logits_smooth_norm = tf.contrib.layers.flatten(logits_smooth)/tf.reduce_sum(logits_smooth,axis=[1,2,3], keep_dims = True) ones = tf.ones([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2]) zeros = tf.zeros([FLAGS.batch_size,IMAGE_SIZE,IMAGE_SIZE,2]) normal_dist_d = tf.distributions.Uniform(low = zeros, high = ones) normal_dist_ = normal_dist_d.sample() normal_dist = tf.maximum(normal_dist_,0.0001) normal_dist_norm = tf.contrib.layers.flatten(normal_dist)/tf.reduce_sum(normal_dist,axis=[1,2,3], keep_dims = True) X = tf.distributions.Categorical(probs=logits_smooth_norm) Y = tf.distributions.Categorical(probs=normal_dist_norm) kl_dist = tf.reduce_mean(tf.distributions.kl_divergence(X, Y)) # kl_dist = tf.reduce_sum(logits_smooth * tf.log(logits_smooth/normal_dist)) #kl_dist = tf.contrib.distributions.kl_divergence(logits_smooth_norm,normal_dist_norm) # logits_std = tf.reduce_std(logits_smooth, axis =[1,2],keep_dims=True ) # logits_mean = tf.reduce_mean(logits_smooth, axis =[1,2], keep_dims=True) # logits_normalized = (logits_smooth - logits_mean)/logits_std # annotation_weights_norm = tf.reduce_sum(tf.exp(tf.abs(annotation))/tf.exp(1.0)) # annotation_weights = (tf.exp(tf.abs(annotation))/tf.exp(1.0)) loss_ = 0.2*tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((annotation - logits)) ),1)) + kl_dist # loss_ = 0.4*loss_mask + loss_mask2 # loss = tf.reduce_mean(tf.squared_difference(logits ,annotation )) loss_summary = tf.summary.scalar("entropy", loss_) # zloss = tf.reduce_mean(tf.losses.cosine_distance(tf.contrib.layers.flatten(z_new) ,tf.contrib.layers.flatten(z_pred),axis =1)) zloss_ = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_new))),1)) # zloss_lip = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip))),1)) # zloss_lip_inv = -tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_lip_inv))),1)) # z_loss = zloss_ + 0.1* zloss_lip# + zloss_lip_inv lip_loss_dec = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((logits - logits_e))),1)) loss = loss_ + 0.1*lip_loss_dec lip_loss_pred = tf.reduce_mean(tf.reduce_sum(tf.contrib.layers.flatten(tf.abs((z_pred - z_pred_e))),1)) zloss = zloss_ + 0.1*lip_loss_pred grads = train_z(loss_,z) trainable_var = tf.trainable_variables() trainable_z_pred_var = tf.trainable_variables(scope="predictor") trainable_d_pred_var = tf.trainable_variables(scope="decoder") print(trainable_z_pred_var) if FLAGS.debug: for var in trainable_var: utils.add_to_regularization_and_summary(var) train_op = train(loss, trainable_var) train_pred = train_predictor(zloss,trainable_z_pred_var) print("Setting up summary op...") summary_op = tf.summary.merge_all() print("Setting up image reader...") train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir) print(len(train_records)) print(len(valid_records)) print("Setting up dataset reader") image_options = {'resize': True, 'resize_size': IMAGE_SIZE} # if FLAGS.mode == 'train': train_dataset_reader = dataset.BatchDatset(train_records, image_options) validation_dataset_reader = dataset.BatchDatset(valid_records, image_options) sess = tf.Session() print("Setting up Saver...") saver = tf.train.Saver() # create two summary writers to show training loss and validation loss in the same graph # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation') sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") saved =True if FLAGS.mode == "train": for itr in xrange(MAX_ITERATION): train_images, train_annotations = train_dataset_reader.next_batch(FLAGS.batch_size) print("$$$$$$$$$$$$$$$$$$$$") print(train_images.shape) # z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5 z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10)) # train_images[train_images < 0.] = -1. # train_annotations[train_annotations < 0.] = -1. # train_images[train_images >= 0.] = 1.0 # train_annotations[train_annotations >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) cond = random.randint(0, 10) # saved = True if False: saved = False train_images_m, train_annotations_m = train_dataset_reader.get_random_batch(FLAGS.batch_size) train_images_m[train_images_m < 0.] = -1. train_annotations_m[train_annotations_m < 0.] = -1. train_images_m[train_images_m >= 0.] = 1.0 train_annotations_m[train_annotations_m >= 0.] = 1.0 train_images = (train_images + 1.)/2.0*255.0 train_annotations = (train_annotations + 1.)/2.0*255.0 train_images_m = (train_images_m + 1.)/2.0*255.0 train_annotations_m = (train_annotations_m + 1.)/2.0*255.0 train_images_m[:,32:,:,:] = 0 train_annotations_m[:,32:,:,:] = 0 train_images = np.clip((train_images + train_images_m),0.0,255.0) train_annotations = np.clip((train_annotations + train_annotations_m),0.0,255.0) ''' train_images[train_images < 0.] = -1. train_annotations[train_annotations < 0.] = -1. train_images[train_images >= 0.] = 1.0 train_annotations[train_annotations >= 0.] = 1.0 ''' train_annotations_ = np.squeeze(train_annotations,axis = 3) train_images_ = train_images train_images = train_images/127.5 - 1.0 train_annotations = train_annotations/127.5 - 1.0 # for itr_ in range(FLAGS.batch_size): # utils.save_image(train_images_[itr_].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr_) ) # utils.save_image(train_annotations_[itr_].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr_) ) # train_images[:,x1:w1,y1:h1,:] = 0 # print(train_images) r_m, r_m2 = random_mask(64) #feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,mask:r_m, istrain:True } #train_images[:,50:100,50:100,:] =0 v = 0 # print(train_images) error_dec = np.random.normal(0.0,0.001,(FLAGS.batch_size,4,4,522)) error_dec_ = np.random.normal(0.0,0.001,(FLAGS.batch_size,1,1,8202)) # z_l_inv = z_ + np.random.normal(0.0,0.1) # feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_, e:error_dec, mask:r_m, istrain:True } # z_l = z_ + np.random.normal(0.0,0.001) # lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict) # z_l = z_ + np.random.normal(0.0,0.001) # print("Step: %d, lip_loss:%g" % (itr,lloss)) for p in range(20): z_ol = np.copy(z_) # z_l = z_ol + np.random.normal(0.0,0.001) # print("666666666666666666666666666666666666666") feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85, z: z_,e:error_dec, istrain:True } # lloss,_ = sess.run([lip_loss, train_lip ], feed_dict=feed_dict) # print("Step: %d, z_step: %d, lip_loss:%g" % (itr,p,lloss)) z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict) print("Step: %d, z_step: %d, Train_loss:%g" % (itr,p,z_loss)) # print(z_) g = sess.run([grads],feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001*v - 0.1*g[0][0] z_ += 0.001 * v_prev + (1+0.001)*v #z_ = np.clip(z_, -10.0, 10.0) ''' m = interp1d([-10.0,10.0],[-1.0,1.0]) print(np.max(z_)) print(np.min(z_)) z_ol_interp = m(z_ol) z_interp = m(z_) _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol_interp,z_new:z_interp,e_p:error_dec_,istrain:True,keep_probability: 0.85}) print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss)) ''' # _,z_pred_loss =sess.run([train_pred,zloss],feed_dict={image: train_images,mask:r_m,z:z_ol,z_new:z_,istrain:True,keep_probability: 0.85}) # print("Step: %d, z_step: %d, z_pred_loss:%g" % (itr,p,z_pred_loss)) # z_ = np.clip(z_, -1.0, 1.0) # print(v.shape) # print(z_.shape) feed_dict = {image: train_images, annotation: train_annotations, keep_probability:0.85,e:error_dec, z: z_, istrain:True } sess.run(train_op, feed_dict=feed_dict) if itr % 10 == 0: train_loss, summary_str = sess.run([loss, loss_summary], feed_dict=feed_dict) print("Step: %d, Train_loss:%g" % (itr, train_loss)) train_writer.add_summary(summary_str, itr) if itr % 500 == 0: valid_images, valid_annotations = validation_dataset_reader.next_batch(FLAGS.batch_size) # valid_annotations[valid_annotations < 0.] = -1. # valid_images[valid_images < 0.] = -1. # valid_annotations[valid_annotations >= 0.] = 1.0 # valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 valid_loss, summary_sva = sess.run([loss, loss_summary], feed_dict={image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z: z_,e:error_dec, istrain:False }) print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss)) # add validation loss to TensorBoard validation_writer.add_summary(summary_sva, itr) if itr % 3000 == 0: save_itr = save_itr + 3000 saver.save(sess, FLAGS.logs_dir + "model_fuse.ckpt", save_itr) elif FLAGS.mode == "visualize": valid_images, valid_annotations = train_dataset_reader.get_random_batch(FLAGS.batch_size) # valid_annotations[valid_annotations < 0.] = -1.0 # valid_images[valid_images < 0.] = -1.0 # valid_annotations[valid_annotations >= 0.] = 1.0 # valid_images[valid_images >= 0.] = 1.0 x1 = random.randint(0, 10) w1 = random.randint(30, 54) y1 = random.randint(0, 10) h1 = random.randint(30, 54) # valid_images[:,x1:w1,y1:h1,:] = 0 r_m, r_m2 = random_mask(64) # z_ = np.zeros(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10)) # z_ = np.reshape(signal.gaussian(200, std=1),(FLAGS.batch_size,1,1,10))-0.5 z_ = np.random.uniform(low=-1.0, high=1.0, size=(FLAGS.batch_size,1,1,10)) feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False } v= 0 m__ = interp1d([-10.0,10.0],[-1.0,1.0]) z_ = m__(z_) # feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m } for p in range(20): z_ol = np.copy(z_) # print("666666666666666666666666666666666666666") # print(z_) # feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z: z_, istrain:False,mask:r_m } # z_loss, summ = sess.run([loss,loss_summary], feed_dict=feed_dict) # print("z_step: %d, Train_loss:%g" % (p,z_loss)) # z_, z_pred_loss = sess.run(z_pred,zlossfeed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 1.0, z:z_ol, istrain:False,mask:r_m}) # print(z_) g = sess.run([grads],feed_dict=feed_dict) v_prev = np.copy(v) # print(g[0][0].shape) v = 0.001*v - 0.1*g[0][0] z_ = z_ol + 0.001 * v_prev + (1+0.001)*v # z_ = z_ol + 0.001 * v_prev + (1+0.001)*v # print("z_____________") # print(z__) # print("z_") # print(z_) # m__ = interp1d([-10.0,10.0],[-1.0,1.0]) # z_ol = m__(z_ol) # z_ = sess.run(z_pred,feed_dict = {image: valid_images, annotation: valid_annotations, keep_probability: 0.85, z:z_ol, istrain:False,mask:r_m}) # m_ = interp1d([-1.0,1.0],[-10.0,10.0]) # z_ = m_(z_) # z_ = np.clip(z_, -1.0, 1.0) # print(z_pred_loss) # m_ = interp1d([-1.0,1.0],[-10.0,10.0]) # z_ = m_(z_) pred = sess.run(logits, feed_dict={image: valid_images, annotation: valid_annotations,z:z_, istrain:False, keep_probability: 0.85}) valid_images = (valid_images +1.)/2.0*100.0 # predicted_patch = sess.run(mask) * pred # pred = valid_images_masked + predicted_patch pred_ = pred * 128.0 # pred = pred + 1./2.0*255 print(np.max(pred_)) print(np.min(pred_)) pred = np.reshape(np.concatenate((valid_images,pred_), axis=3),(-1,64,64,3)) valid_annotations_ = valid_annotations * 128.0 valid_annotations = np.reshape(np.concatenate((valid_images, valid_annotations_), axis=3),(-1,64,64,3)) valid_images_gray = np.squeeze(valid_images) # for itr in range(FLAGS.batch_size): # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)) # utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)) # utils.save_image(valid_images_masked[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr)+'_' + str(p) ) # utils.save_image(valid_annotations_[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr)+'_' + str(p) ) # utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="predz_" + str(5+itr)+'_' + str(p) ) # print("Saved image: %d" % itr) for itr in range(FLAGS.batch_size): utils.save_image(valid_images_gray[itr], FLAGS.logs_dir, name="inp_" + str(5+itr) ) utils.save_image(color.lab2rgb(pred[itr]), FLAGS.logs_dir, name="predz_" + str(5+itr) ) utils.save_image(color.lab2rgb(valid_annotations[itr]), FLAGS.logs_dir, name="gt_" + str(5+itr) )