def visualise_disentanglement(self, x, fixed_z_c): c = self.encode([x]) n_cs = c.shape[1] # self.c_dist.dim c = c[0] fixed_z, fixed_c = self.latent_dist.split_var(fixed_z_c[:1]) rimgs = [] if isinstance(self.c_dist, Gaussian): for target_c_index in range(n_cs): for ri in range(self.n_disentangle_samples): value = -1.0 + 2.0 / (self.n_disentangle_samples - 1.) * ri c_new = np.zeros((1, n_cs)) for i in range(n_cs): if (i == target_c_index): c_new[0][i] = value else: c_new[0][i] = c[i] z_c_new = np.concatenate([fixed_z, c_new], axis=1) rimgs.append(self.generate(z_c=z_c_new)) else: raise NotImplementedError rimgs = np.vstack(rimgs).reshape( [n_cs, self.n_disentangle_samples, -1]) rimgs = ((rimgs + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images(rimgs, os.path.join(self.dirs['samples'], 'disentanglement.png'), n_rows=n_cs, n_cols=self.n_disentangle_samples)
def visualise_reconstruction(self, X, fixed_z_c): fixed_z, fixed_c = self.latent_dist.split_var(fixed_z_c) X_r = self.reconstruct(X, z=fixed_z) X_r = ((X_r + 1.) * (255.99 / 2)).astype('int32') save_images( X_r.reshape([-1] + self.image_shape), os.path.join(self.dirs['samples'], 'samples_reconstructed.png'))
def visualise_reconstruction(self, X1,mk1,iteration): X_r1,X_r0= self.reconstruct(X1,mk1) #print(X_r0[3]) X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) X_r0 = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_r1, os.path.join(self.dirs['samples'], str(iteration)+'samples_reconstructed.png')) save_images(X_r0, os.path.join(self.dirs['samples'], str(iteration)+'reset0_reconstructed.png'))
def encodeImg(self,pathForSave,X1, mk1,k, is_training=False): X_r1,X_r0=self.session.run([self.x_out1,self.x_out_r0],feed_dict={self.x1: X1,self.mask: mk1, self.is_training: is_training}) X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) X_r0 = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_r1, os.path.join(pathForSave, 'iter'+str(k)+'_samples_reconstructed.png')) save_images(X_r0, os.path.join(pathForSave, 'iter'+str(k)+'_reset0_reconstructed.png'))
def getCodesAndImgs(self,pathForSave,X1,k, is_training=False): z1,X_r0=self.session.run([self.z1,self.x_out1],feed_dict={self.x1: X1,self.is_training: is_training}) ImageNorm0_1 = ((X_r0+1.)*(1.00/2)).astype('double').reshape([-1,self.image_shape[1],self.image_shape[2],self.image_shape[0]]) # for visual the first result to valide it effectiveness if k==1: X_save = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_save, os.path.join(pathForSave, 'iter'+str(k)+'_samples_reconstructed.png')) return z1,ImageNorm0_1
def visulize_rec_Origin(self,pathForSave,X1,X2,iteration): X_r1=self.session.run(self.x_out1 ,feed_dict={self.x1: X1,self.is_training: False}) #print(X_r0[3]) X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_r1, os.path.join(pathForSave, 'iter'+str(iteration)+'_samplesOrigin_reconstructed_X1.png')) X_r2=self.session.run(self.x_out1 ,feed_dict={self.x1: X2,self.is_training: False}) #print(X_r0[3]) X_r2 = ((X_r2+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_r2, os.path.join(pathForSave, 'iter'+str(iteration)+'_samplesOrigin_reconstructed_X2.png'))
def visulize_rec(self, X1, iteration): X_r1 = self.session.run(self.x_out1, feed_dict={ self.x1: X1, self.is_training: False }) # print(X_r0[3]) X_r1 = ((X_r1 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_r1, os.path.join(self.dirs['samples'], str(iteration) + 'samples_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): self.session.run(tf.global_variables_initializer()) # Fixed GT samples - save fixed_x, _ = next(self.train_iter) fixed_x = self.session.run(tf.constant(fixed_x)) save_images(fixed_x, os.path.join(self.dirs['samples'], 'samples_groundtruth.png')) start_iter = self.load() running_cost = 0. for iteration in range(start_iter, n_iters): start_time = time.time() _data, _ = next(self.train_iter) _, cost = self.session.run((self.optimizer, self.loss), feed_dict={self.x: _data, self.is_training:True}) running_cost += cost if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: t = time.time() dev_data, _ = next(self.dev_iter) dev_cost, dev_z_dist_info = self.session.run([self.loss, self.z_dist_info], feed_dict={self.x: dev_data, self.is_training:False}) n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters) avg_cost = running_cost / n_samples running_cost = 0. print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}".format(iteration, avg_cost, dev_cost)) if isinstance(self.z_dist, Gaussian): avg_dev_var = np.mean(dev_z_dist_info["stddev"]**2, axis=0) zss_str = "" for i,zss in enumerate(avg_dev_var): z_str = "z{0}={1:.2f}".format(i,zss) zss_str += z_str + ", " print("z variance:{0}".format(zss_str)) if self.vis_reconst: self.visualise_reconstruction(fixed_x) if self.vis_disent: self.visualise_disentanglement(fixed_x[0]) if np.any(np.isnan(avg_cost)): raise ValueError("NaN detected!") if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration)
def getVisualImgs(self, pathForSave, X1, X2, k, is_training=False): x2fg_x1bg_out, x1fg_x2bg_out = self.session.run( [self.x_out11, self.x_out22], feed_dict={ self.x1: X1, self.x2: X2, self.is_training: is_training }) X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] + self.image_shape) x2fg_x1bg_out_save = ( (x2fg_x1bg_out + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) x1fg_x2bg_out_save = ( (x1fg_x2bg_out + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_orig1_save, os.path.join(pathForSave, 'iter' + str(k) + '_origX1_img.png')) save_images( X_orig2_save, os.path.join(pathForSave, 'iter' + str(k) + '_origX2_img.png')) save_images( x2fg_x1bg_out_save, os.path.join(pathForSave, 'iter' + str(k) + 'x2fg_x1bg_out_img.png')) save_images( x1fg_x2bg_out_save, os.path.join(pathForSave, 'iter' + str(k) + 'x1fg_x2bg_out_img.png'))
def getVisualImgs(self, pathForSave, X1,X2,k, is_training=False): X_swap = self.session.run(self.x_swap,feed_dict={self.x1: X1,self.x2: X2, self.is_training: is_training}) X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_Swap_save = ((X_swap + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_orig1_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_img.png')) save_images(X_orig2_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_swap.png')) save_images(X_Swap_save, os.path.join(pathForSave, 'iter' + str(k) + '_swap_img.png'))
def visualise_reconstruction(self, X1, aux1,aux2,iteration): X_out1,mix_head_out,aux1_mix_head_out= self.reconstruct(X1, aux1,aux2) # print(X_out1.shape) X1 = ((X_out1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) mix_head_out = ((mix_head_out+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) aux1_mix_head_out = ((aux1_mix_head_out+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X1, os.path.join(self.dirs['samples'], str(iteration)+'samples_1_rec.png')) save_images(mix_head_out, os.path.join(self.dirs['samples'], str(iteration)+'X1bg_aux1head.png')) save_images(aux1_mix_head_out, os.path.join(self.dirs['samples'], str(iteration)+'X1head_aux1bg.png'))
def visualise_disentanglement(self, x): z = self.encode([x]) n_zs = z.shape[1] #self.latent_dist.dim z = z[0] rimgs = [] if isinstance(self.z_dist, Gaussian): for target_z_index in range(n_zs): for ri in range(self.n_disentangle_samples): value = -3.0 + 6.0 / (self.n_disentangle_samples-1.) * ri z_new = np.zeros((1, n_zs)) for i in range(n_zs): if (i == target_z_index): z_new[0][i] = value else: z_new[0][i] = z[i] rimgs.append(self.generate(z_mu=z_new)) else: raise NotImplementedError rimgs = np.vstack(rimgs).reshape([n_zs, self.n_disentangle_samples, -1]) #.transpose(1,0,2) rimgs = rimgs[[5,7,2,6,1,9,3]] #order of zs captured rimgs = ((rimgs+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(rimgs, os.path.join(self.dirs['samples'], 'disentanglement.png'), n_cols=n_zs, n_rows=self.n_disentangle_samples)
def getVisualImgs(self, pathForSave, X1, mk1, X2, mk2, X3, mk3, X4, mk4, k, is_training=False): X_out1, X_out2, X_r0, X_swap = self.session.run( [self.x_out1, self.x_out2, self.x_out11, self.x_hybrid3], feed_dict={ self.x1: X1, self.x2: X2, self.mk1: mk1, self.mk2: mk2, self.x3: X3, self.x4: X4, self.mk3: mk3, self.mk4: mk4, self.is_training: is_training }) X1_save = ((X_out1 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X2_save = ((X_out2 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_orig3_save = (X3 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_orig4_save = (X4 * 255.99).astype('int32').reshape([-1] + self.image_shape) X_reset0_save = ( (X_r0 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_Swap_save = ((X_swap + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_orig1_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_img1.png')) #save_images(X_orig2_save, os.path.join(pathForSave, 'iter'+str(k)+'_orig_img2.png')) #save_images(X_orig3_save, os.path.join(pathForSave, 'iter'+str(k)+'_orig_img3.png')) save_images( X_orig4_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_img4.png')) #save_images(X1_save, os.path.join(pathForSave, 'iter'+str(k)+'_reconst_img1.png')) #save_images(X2_save, os.path.join(pathForSave, 'iter'+str(k)+'_reconst_img2.png')) save_images( X_reset0_save, os.path.join(pathForSave, 'iter' + str(k) + '_reset0_img.png')) save_images( X_Swap_save, os.path.join(pathForSave, 'iter' + str(k) + '_swap_img.png'))
def visualise_reconstruction(self, X): X_r = self.reconstruct(X) X_r = ((X_r+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(X_r, os.path.join(self.dirs['samples'], 'samples_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): count = 0 self.session.run(tf.global_variables_initializer()) # Fixed GT samples - save fixed_x1, fixed_gt1, _ = next(self.train_iter1) fixed_x1 = self.session.run(tf.constant(fixed_x1)) save_images( fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png')) # fixed_x2, fixed_gt2, _ = next(self.train_iter2) fixed_x2 = self.session.run(tf.constant(fixed_x2)) save_images( fixed_x2, os.path.join(self.dirs['samples'], 'samples_2_groundtruth.png')) fixed_x3, fixed_mk3, _ = next(self.train_iter3) fixed_x3 = self.session.run(tf.constant(fixed_x3)) save_images( fixed_x3, os.path.join(self.dirs['samples'], 'samples_3_groundtruth.png')) start_iter = self.load() running_cost = 0. fixed_x4, fixed_label, _ = next(self.train_iter4) fixed_x4 = self.session.run(tf.constant(fixed_x4)) save_images( fixed_x4, os.path.join(self.dirs['samples'], 'samples_4_groundtruth.png')) logs = open('record_loss.txt', 'w') for iteration in range(start_iter, n_iters): _data1, _gt1, _ = next(self.train_iter1) _data2, _gt2, _ = next(self.train_iter2) _data3, _mask1, _ = next(self.train_iter3) _data4, _label, _ = next(self.train_iter4) _, cost = self.session.run( (self.optimizer, self.loss), feed_dict={ self.x1: _data1, self.x2: _data2, self.x3: _data3, self.x4: _data4, self.gt1: _gt1, self.gt2: _gt2, self.is_training: True }) running_cost += cost if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: dev_data1, dev_gt1, _ = next(self.dev_iter1) dev_data2, dev_gt2, _ = next(self.dev_iter2) dev_data3, dev_mask1, _ = next(self.dev_iter3) dev_data4, dev_label, _ = next(self.dev_iter4) dev_cost = self.session.run(self.loss, feed_dict={ self.x1: dev_data1, self.x2: dev_data2, self.x3: dev_data3, self.x4: dev_data4, self.gt1: dev_gt1, self.gt2: dev_gt2, self.is_training: False }) n_samples = 1. if ( iteration < start_iter + 4) else float(stats_iters) avg_cost = running_cost / n_samples running_cost = 0. print( "Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}" .format(iteration, avg_cost, dev_cost)) logs.writelines( "Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}\n" .format(iteration, avg_cost, dev_cost)) count = count + 1 if self.vis_reconst: self.visualise_reconstruction(fixed_x1, fixed_x2, fixed_gt1, fixed_gt2, fixed_x3, fixed_x4, iteration) if np.any(np.isnan(avg_cost)): raise ValueError("NaN detected!") # save checkpoint if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration) # for save loss logs.close()
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): # for save loss count=0 self.session.run(tf.global_variables_initializer()) # Fixed GT samples - save fixed_x1, fixed_mk1 ,_ = next(self.train_iter1) print("fixed_mk1=") print(fixed_mk1[0:4]) # print(fixed_label[0:4]) # replace mask #unitLength=3 #(need to changed when has larger unitLength) unitLength=int(self.latent_dim/self.latent_num) # generate zero representation and black image and gts0 img_zero,fixed_zero_mk,fixed_gts0=self.generateMaskZeroAndGts(self.batch_size,unitLength) # fixed_x1= self.session.run(tf.constant(fixed_x1)) save_images(fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png')) # start_iter = self.load() running_cost = 0. _gan_data=fixed_x1 logs=open('loss_records.txt','w') for iteration in range(start_iter, n_iters): start_time = time.time() _data1, _mask1,_ = next(self.train_iter1) _, cost = self.session.run((self.optimizer, self.loss),feed_dict={self.x1: _data1,self.is_training:True}) running_cost += cost if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: t = time.time() dev_data1, dev_mask1, dev_label1= next(self.dev_iter1) #dev_cost,dev_rec_loss,dev_reset0_loss,vector_loss,rec_zero_loss,class_loss= self.session.run([self.loss,self.rec_loss,self.reset0_loss,self.vector_loss,self.rec_zero_loss,self.class_loss],feed_dict={self.x1: dev_data1, self.mask: dev_mask1, self.vec_one:vector_one,self.img_black:img_zero,self.mask_zero:fixed_zero_mk,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt3:class_gt3,self.class_gt4:class_gt4,self.class_gt4:class_gt4,self.is_training:False}) dev_cost,dev_rec_loss= self.session.run([self.loss,self.rec_loss],feed_dict={self.x1: dev_data1,self.is_training:False}) n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters) avg_cost = running_cost / n_samples running_cost = 0. print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f})".format(iteration, avg_cost, dev_cost,dev_rec_loss)) logs.writelines("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f})\n".format( iteration, avg_cost, dev_cost, dev_rec_loss)) count=count+1 if self.vis_reconst: self.visulize_rec(fixed_x1,iteration) #self.visualise_reconstruction(img_zero,fixed_mk1,iteration) if np.any(np.isnan(avg_cost)): raise ValueError("NaN detected!") # save checkpoint if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration) _gan_data=_data1 logs.close()
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): # for save loss count=0 self.session.run(tf.global_variables_initializer()) # Fixed GT samples - save fixed_x1,fixed_mask_1,_= next(self.train_iter1) fixed_x2,fixed_mask_2,_= next(self.train_iter2) fixed_aux1,fixed_GT1,_= next(self.train_iter3) fixed_aux2,fixed_GT2,_= next(self.train_iter4) # print(fixed_x1.shape) # print(fixed_mask_1.shape) # print(fixed_x2.shape) # print(fixed_mask_2.shape) # print(fixed_aux1.shape) # print(fixed_GT1.shape) fixed_x1= self.session.run(tf.constant(fixed_x1)) fixed_mask_1= self.session.run(tf.constant(fixed_mask_1)) # print("fixed_mask_1.shape") # print(fixed_mask_1.shape) #fixed_x1_2 =fixed_x1.reshape([64,32,32,3]) #fixed_x1 = ((fixed_x1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png')) ##save_images(fixed_mask_1, os.path.join(self.dirs['samples'], 'mask_1_groundtruth.png')) fixed_aux1= self.session.run(tf.constant(fixed_aux1)) fixed_aux2= self.session.run(tf.constant(fixed_aux2)) #fixed_aux1 = ((fixed_aux1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) #fixed_aux2 = ((fixed_aux2+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape) save_images(fixed_aux1, os.path.join(self.dirs['samples'], 'aux_1_groundtruth.png')) save_images(fixed_aux2, os.path.join(self.dirs['samples'], 'aux_2_groundtruth.png')) # start_iter = self.load() running_cost = 0. class_gt0, class_gt1,class_gt2,class_gt3,class_gt4,class_gt5, class_gt6, class_gt7, class_gt8,class_gt9,class_gt10 = self.generateClassificationLabel(self.batch_size) _gan_data=fixed_x1 logs=open('loss_records.txt','w') for iteration in range(start_iter, n_iters): start_time = time.time() _data1,_mask1, _ = next(self.train_iter1) _aux_label, _, _ = next(self.train_iter2) _aux1,_gt1, _ = next(self.train_iter3) _aux2,_gt2, _ = next(self.train_iter4) _, cost = self.session.run((self.optimizer, self.loss),feed_dict={self.x1:_data1,self.aux_class_gt:_aux_label,self.aux1_mask:_mask1,self.aux1:_aux1,self.aux2:_aux2,self.aux_GT1:_gt1,self.aux_GT2:_gt2,self.is_training:True,self.class_gt0:class_gt0,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt3:class_gt3, self.class_gt4:class_gt4, self.class_gt5:class_gt5, self.class_gt6:class_gt6,self.class_gt7:class_gt7, self.class_gt8:class_gt8, self.class_gt9:class_gt9, self.class_gt10:class_gt10}) running_cost += cost if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: t = time.time() dev_data1,dev_mask1, _ = next(self.dev_iter1) dev_aux1,dev_gt1, _ = next(self.dev_iter3) dev_aux2,dev_gt2, _ = next(self.dev_iter4) dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss= \ self.session.run([self.loss,self.rec_img_loss1,self.rec_aux1_loss2,self.rec_aux2_loss3,self.rec_aux1_swap_loss4,self.rec_aux2_swap_loss5,self.rec_dual_loss6,self.head_loss,self.class_loss,self.fuzzy_class_loss,self.aux1_class_loss,self.aux2_class_loss,self.fuzzy_bg_class_loss,self.aux1_bg_class_loss,self.aux2_bg_class_loss], feed_dict={self.x1:dev_data1,self.aux1_mask:dev_mask1,self.aux1:dev_aux1,self.aux2:dev_aux2,self.aux_GT1:dev_gt1,self.aux_GT2:dev_gt2,self.is_training:False, self.class_gt0: class_gt0, self.class_gt1: class_gt1,self.class_gt2: class_gt2,self.class_gt3:class_gt3,self.aux_class_gt:_aux_label,self.class_gt4:class_gt4, self.class_gt5:class_gt5, self.class_gt6:class_gt6,self.class_gt7:class_gt7, self.class_gt8:class_gt8, self.class_gt9:class_gt9, self.class_gt10:class_gt10}) n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters) avg_cost = running_cost / n_samples running_cost = 0. print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(img1_loss:{3:.1f},aux1_loss2:{4:.1f},aux2_loss3:{5:.1f},aux1_swap_loss:{6:.1f},aux2_swap_loss:{7:.1f},dual_swap_loss:{8:.1f},head loss:{9:.1f},class loss:{10:.1f}(fuzzy_class:{11:.1f},aux1_class_loss:{12:.1f},aux2_class_loss:{13:.1f},fuzzy_bg_class_loss:{14:.1f},aux1_bg_class_loss:{15:.1f},aux2_bg_class_loss:{16:.1f}))". format(iteration, avg_cost, dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss)) logs.writelines("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(img1_loss:{3:.1f},aux1_loss2:{4:.1f},aux2_loss3:{5:.1f},aux1_swap_loss:{6:.1f},aux2_swap_loss:{7:.1f},dual_swap_loss:{8:.1f},head loss:{9:.1f},class loss:{10:.1f}(fuzzy_class:{11:.1f},aux1_class_loss:{12:.1f},aux2_class_loss:{13:.1f},fuzzy_bg_class_loss:{14:.1f},aux1_bg_class_loss:{15:.1f},aux2_bg_class_loss:{16:.1f}))\n". format(iteration, avg_cost, dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss)) count=count+1 if self.vis_reconst: self.visualise_reconstruction(fixed_x1,fixed_aux1,fixed_aux2,iteration) #self.visualise_reconstruction(img_zero,fixed_mk1,iteration) if np.any(np.isnan(avg_cost)): raise ValueError("NaN detected!") # save checkpoint if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration) _gan_data=_data1 logs.close()
def visualise_reconstruction(self, X1, X2, gt1, gt2, X3, X4, iteration): X_r1, X_r2, X_r11, X_r22, X_r33, X_r44 = self.reconstruct( X1, X2, gt1, gt2, X3, X4) X_r1 = ((X_r1 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r2 = ((X_r2 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r11 = ((X_r11 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r22 = ((X_r22 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r33 = ((X_r33 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r44 = ((X_r44 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_r1, os.path.join(self.dirs['samples'], str(iteration) + 'samples_1_reconstructed.png')) save_images( X_r2, os.path.join(self.dirs['samples'], str(iteration) + 'samples_2_reconstructed.png')) save_images( X_r11, os.path.join(self.dirs['samples'], str(iteration) + 'samples_11_reconstructed.png')) save_images( X_r22, os.path.join(self.dirs['samples'], str(iteration) + 'samples_22_reconstructed.png')) save_images( X_r33, os.path.join(self.dirs['samples'], str(iteration) + 'samples_33_reconstructed.png')) save_images( X_r44, os.path.join(self.dirs['samples'], str(iteration) + 'samples_44_reconstructed.png'))
def encodeImg(self, pathForSave, ratio, X1, mk1, X2, mk2, k, is_training=False): X_r1, X_r2, X_r11, X_r22, X_r1twice, X_r2twice = self.session.run( [ self.x_out3, self.x_out4, self.x_hybrid3, self.x_hybrid4, self.x_out33, self.x_out44 ], feed_dict={ self.x3: X1, self.x4: X2, self.mk3: mk1, self.mk4: mk2, self.is_training: is_training }) X_r1 = ((X_r1 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r2 = ((X_r2 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r11 = ((X_r11 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r22 = ((X_r22 + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_r1, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_3_reconstructed.png')) save_images( X_r2, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_4_reconstructed.png')) save_images( X_r11, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_hrd3_reconstructed.png')) save_images( X_r22, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_hrd4_reconstructed.png')) X_r1twice = ((X_r1twice + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) X_r2twice = ((X_r2twice + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape) save_images( X_r1twice, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_3_twice_reconstructed.png')) save_images( X_r2twice, os.path.join( pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) + '_samples_4_twice_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): self.session.run(tf.global_variables_initializer()) # Fixed GT samples fixed_x, _ = next(self.train_iter) fixed_x = self.session.run(tf.constant(fixed_x)) save_images( fixed_x, os.path.join(self.dirs['samples'], 'samples_groundtruth.png')) # Fixed prior sample, for generating samples fixed_z_c = self.session.run( self.latent_dist.sample_prior(self.batch_size)) log_vars = [x for _, x in self.log_vars] log_keys = [x for x, _ in self.log_vars] running_log_vals = [0.] * len(self.log_vars) start_iter = self.load() for iteration in range(start_iter, n_iters): start_time = time.time() # Train generator if iteration > start_iter: _ = self.session.run(self.gen_opt, feed_dict={ self.x: _data, self.is_training: True }) # Train critic if self.mode == 'wgan-gp': disc_iters = self.critic_iters else: disc_iters = 1 for i in range(disc_iters): _data, _ = next(self.train_iter) log_vals = self.session.run([self.disc_opt] + log_vars, feed_dict={ self.x: _data, self.is_training: True })[1:] running_log_vals = [ running + batch for running, batch in zip(running_log_vals, log_vals) ] if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: t = time.time() dev_data, _ = next(self.dev_iter) dev_log_vals = self.session.run( log_vars + [self.q_c_given_x_real_dist_info], feed_dict={ self.x: dev_data, self.is_training: False }) dev_c_dist_info = dev_log_vals[-1] dev_log_vals = dev_log_vals[:-1] n_samples = 1. if ( iteration < start_iter + 4) else float(stats_iters) avg_log_vals = [ r_log_val / n_samples for r_log_val in running_log_vals ] running_log_vals = [0.] * len(self.log_vars) tr_log_line = " | ".join( "{0}: {1:.2f}".format(k, v) for k, v in zip(log_keys, avg_log_vals)) dev_log_line = " | ".join( "{0}: {1:.2f}".format(k, v) for k, v in zip(log_keys, dev_log_vals)) print("Iteration:{0} \nTrain: {1} \nDev : {2}".format( iteration, tr_log_line, dev_log_line)) if isinstance(self.c_dist, Gaussian) and not self.c_dist._fix_std: avg_dev_var = np.mean(dev_c_dist_info["stddev"]**2, axis=0) cs_str = "" for i, cs in enumerate(avg_dev_var): c_str = "c{0}={1:.2f}".format(i, cs) cs_str += c_str + ", " print("c variance:{0}".format(cs_str)) fixed_samples = self.generate( z_c=fixed_z_c).reshape([-1] + self.image_shape) save_images( fixed_samples, os.path.join(self.dirs['samples'], 'samples_generated.png')) if self.vis_reconst: self.visualise_reconstruction(fixed_x, fixed_z_c) if self.vis_disent: self.visualise_disentanglement(fixed_x[0], fixed_z_c) if np.any(np.isnan(avg_log_vals)): raise ValueError("NaN detected!") if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration)
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval): # for save loss logArray = np.zeros((1000, 4)) count = 0 self.session.run(tf.global_variables_initializer()) # Fixed GT samples - save fixed_x1, fixed_mk1, _ = next(self.train_iter1) print("fixed_mk1=") print(fixed_mk1[0:4]) # replace mask unitLength = 1 #(need to changed when has larger unitLength) # get classification gt and label class_gt0, class_gt1, class_gt2, vector_one = self.generateClassificationLabelandVecOne( self.batch_size) # generate zero representation and white image img_zero, fixed_zero_mk = self.generateMaskZero( self.batch_size, unitLength) # fixed_x1 = self.session.run(tf.constant(fixed_x1)) save_images( fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png')) # fixed_img_zero = self.session.run(tf.constant(img_zero * 0.0)) save_images( fixed_img_zero, os.path.join(self.dirs['samples'], 'samples_white_groundtruth.png')) start_iter = self.load() running_cost = 0. GAN_runing_gen_cost = 0. GAN_runing_dis_cost = 0. _gan_data = fixed_x1 for iteration in range(start_iter, n_iters): start_time = time.time() _data1, _mask1, _ = next(self.train_iter1) _, cost = self.session.run( (self.optimizer, self.loss), feed_dict={ self.x1: _data1, self.mask: _mask1, self.vec_one: vector_one, self.img_white: img_zero, self.mask_zero: fixed_zero_mk, self.class_gt0: class_gt0, self.class_gt1: class_gt1, self.class_gt2: class_gt2, self.is_training: True }) running_cost += cost #==generator training _, _disc_gene_cost = self.session.run( (self.gen_train_optimizer, self.gen_cost), feed_dict={ self.x1: _data1, self.x_gan: _gan_data, self.mask: _mask1, self.vec_one: vector_one, self.img_white: img_zero, self.mask_zero: fixed_zero_mk, self.class_gt0: class_gt0, self.class_gt1: class_gt1, self.class_gt2: class_gt2, self.is_training: True }) GAN_runing_gen_cost += _disc_gene_cost #==Discriminator training for i in range(CRITIC_ITERS): _, _disc_disc_cost = self.session.run( (self.disc_train_optimizer, self.disc_cost), feed_dict={ self.x1: _data1, self.x_gan: _gan_data, self.mask: _mask1, self.vec_one: vector_one, self.img_white: img_zero, self.mask_zero: fixed_zero_mk, self.class_gt0: class_gt0, self.class_gt1: class_gt1, self.class_gt2: class_gt2, self.is_training: True }) GAN_runing_dis_cost += _disc_disc_cost GAN_runing_dis_cost = GAN_runing_dis_cost / CRITIC_ITERS if iteration % n_iters_per_epoch == 1: print("Epoch: {0}".format(iteration // n_iters_per_epoch)) # Print avg stats and dev set stats if (iteration < start_iter + 4) or iteration % stats_iters == 0: t = time.time() dev_data1, dev_mask1, _ = next(self.dev_iter1) #dev_cost,dev_rec_loss,dev_reset0_loss,vector_loss,rec_zero_loss,class_loss= self.session.run([self.loss,self.rec_loss,self.reset0_loss,self.vector_loss,self.rec_zero_loss,self.class_loss],feed_dict={self.x1: dev_data1, self.mask: dev_mask1, self.vec_one:vector_one,self.img_white:img_zero,self.mask_zero:fixed_zero_mk,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt4:class_gt4,self.is_training:False}) dev_cost, dev_rec_loss, dev_reset0_loss, rec_zero_loss, class_loss, gen_cost, disc_cost = self.session.run( [ self.loss, self.rec_loss, self.reset0_loss, self.rec_zero_loss, self.class_loss, self.gen_cost, self.disc_cost ], feed_dict={ self.x1: dev_data1, self.x_gan: _gan_data, self.mask: dev_mask1, self.vec_one: vector_one, self.img_white: img_zero, self.mask_zero: fixed_zero_mk, self.class_gt0: class_gt0, self.class_gt1: class_gt1, self.class_gt2: class_gt2, self.is_training: False }) n_samples = 1. if ( iteration < start_iter + 4) else float(stats_iters) avg_cost = running_cost / n_samples avg_GAN_runing_gen_cost = GAN_runing_gen_cost / n_samples avg_GAN_runing_dis_cost = GAN_runing_dis_cost / n_samples running_cost = 0. GAN_runing_gen_cost = 0. GAN_runing_dis_cost = 0. print( "Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f},reset0_loss:{4:.1f},rec_zero_loss:{5:.1f},class_loss:{6:.1f})" .format(iteration, avg_cost, dev_cost, dev_rec_loss, dev_reset0_loss, rec_zero_loss, class_loss)) print( "Iteration:{0} \t| Train gen_cost:{1:.1f} disc_cost:{2:.1f}\t| Dev gen_cost:{3:.1f} disc_cost:{4:.1f}" .format(iteration, avg_GAN_runing_gen_cost, avg_GAN_runing_dis_cost, gen_cost, disc_cost)) # for save loss logArray[count, 0] = iteration // n_iters_per_epoch logArray[count, 1] = iteration logArray[count, 2] = avg_cost logArray[count, 3] = dev_cost count = count + 1 if self.vis_reconst: self.visualise_reconstruction(fixed_x1, fixed_mk1, iteration) #self.visualise_reconstruction(img_zero,fixed_mk1,iteration) if np.any(np.isnan(avg_cost)): raise ValueError("NaN detected!") # save checkpoint if (iteration > start_iter) and iteration % (ckpt_interval) == 0: self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration) _gan_data = _data1 # for save loss np.save('logArray.npy', logArray)