def visualise_disentanglement(self, x, fixed_z_c):
c = self.encode([x])
n_cs = c.shape[1] # self.c_dist.dim
c = c[0]
fixed_z, fixed_c = self.latent_dist.split_var(fixed_z_c[:1])
rimgs = []
if isinstance(self.c_dist, Gaussian):
for target_c_index in range(n_cs):
for ri in range(self.n_disentangle_samples):
value = -1.0 + 2.0 / (self.n_disentangle_samples - 1.) * ri
c_new = np.zeros((1, n_cs))
for i in range(n_cs):
if (i == target_c_index):
c_new[0][i] = value
else:
c_new[0][i] = c[i]
z_c_new = np.concatenate([fixed_z, c_new], axis=1)
rimgs.append(self.generate(z_c=z_c_new))
else:
raise NotImplementedError
rimgs = np.vstack(rimgs).reshape(
[n_cs, self.n_disentangle_samples, -1])
rimgs = ((rimgs + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(rimgs,
os.path.join(self.dirs['samples'], 'disentanglement.png'),
n_rows=n_cs,
n_cols=self.n_disentangle_samples)
def visualise_reconstruction(self, X, fixed_z_c):
fixed_z, fixed_c = self.latent_dist.split_var(fixed_z_c)
X_r = self.reconstruct(X, z=fixed_z)
X_r = ((X_r + 1.) * (255.99 / 2)).astype('int32')
save_images(
X_r.reshape([-1] + self.image_shape),
os.path.join(self.dirs['samples'], 'samples_reconstructed.png'))
def visualise_reconstruction(self, X1,mk1,iteration):
X_r1,X_r0= self.reconstruct(X1,mk1)
#print(X_r0[3])
X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
X_r0 = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_r1, os.path.join(self.dirs['samples'], str(iteration)+'samples_reconstructed.png'))
save_images(X_r0, os.path.join(self.dirs['samples'], str(iteration)+'reset0_reconstructed.png'))
def encodeImg(self,pathForSave,X1, mk1,k, is_training=False):
X_r1,X_r0=self.session.run([self.x_out1,self.x_out_r0],feed_dict={self.x1: X1,self.mask: mk1, self.is_training: is_training})
X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
X_r0 = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_r1, os.path.join(pathForSave, 'iter'+str(k)+'_samples_reconstructed.png'))
save_images(X_r0, os.path.join(pathForSave, 'iter'+str(k)+'_reset0_reconstructed.png'))
def getCodesAndImgs(self,pathForSave,X1,k, is_training=False):
z1,X_r0=self.session.run([self.z1,self.x_out1],feed_dict={self.x1: X1,self.is_training: is_training})
ImageNorm0_1 = ((X_r0+1.)*(1.00/2)).astype('double').reshape([-1,self.image_shape[1],self.image_shape[2],self.image_shape[0]])
# for visual the first result to valide it effectiveness
if k==1:
X_save = ((X_r0+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_save, os.path.join(pathForSave, 'iter'+str(k)+'_samples_reconstructed.png'))
return z1,ImageNorm0_1
def visulize_rec_Origin(self,pathForSave,X1,X2,iteration):
X_r1=self.session.run(self.x_out1 ,feed_dict={self.x1: X1,self.is_training: False})
#print(X_r0[3])
X_r1 = ((X_r1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_r1, os.path.join(pathForSave, 'iter'+str(iteration)+'_samplesOrigin_reconstructed_X1.png'))
X_r2=self.session.run(self.x_out1 ,feed_dict={self.x1: X2,self.is_training: False})
#print(X_r0[3])
X_r2 = ((X_r2+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_r2, os.path.join(pathForSave, 'iter'+str(iteration)+'_samplesOrigin_reconstructed_X2.png'))
def visulize_rec(self, X1, iteration):
X_r1 = self.session.run(self.x_out1,
feed_dict={
self.x1: X1,
self.is_training: False
})
# print(X_r0[3])
X_r1 = ((X_r1 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(
X_r1,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
self.session.run(tf.global_variables_initializer())
# Fixed GT samples - save
fixed_x, _ = next(self.train_iter)
fixed_x = self.session.run(tf.constant(fixed_x))
save_images(fixed_x, os.path.join(self.dirs['samples'], 'samples_groundtruth.png'))
start_iter = self.load()
running_cost = 0.
for iteration in range(start_iter, n_iters):
start_time = time.time()
_data, _ = next(self.train_iter)
_, cost = self.session.run((self.optimizer, self.loss), feed_dict={self.x: _data, self.is_training:True})
running_cost += cost
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
t = time.time()
dev_data, _ = next(self.dev_iter)
dev_cost, dev_z_dist_info = self.session.run([self.loss, self.z_dist_info],
feed_dict={self.x: dev_data, self.is_training:False})
n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters)
avg_cost = running_cost / n_samples
running_cost = 0.
print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}".format(iteration, avg_cost, dev_cost))
if isinstance(self.z_dist, Gaussian):
avg_dev_var = np.mean(dev_z_dist_info["stddev"]**2, axis=0)
zss_str = ""
for i,zss in enumerate(avg_dev_var):
z_str = "z{0}={1:.2f}".format(i,zss)
zss_str += z_str + ", "
print("z variance:{0}".format(zss_str))
if self.vis_reconst:
self.visualise_reconstruction(fixed_x)
if self.vis_disent:
self.visualise_disentanglement(fixed_x[0])
if np.any(np.isnan(avg_cost)):
raise ValueError("NaN detected!")
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration)
def getVisualImgs(self, pathForSave, X1, X2, k, is_training=False):
x2fg_x1bg_out, x1fg_x2bg_out = self.session.run(
[self.x_out11, self.x_out22],
feed_dict={
self.x1: X1,
self.x2: X2,
self.is_training: is_training
})
X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
x2fg_x1bg_out_save = (
(x2fg_x1bg_out + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
x1fg_x2bg_out_save = (
(x1fg_x2bg_out + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(
X_orig1_save,
os.path.join(pathForSave, 'iter' + str(k) + '_origX1_img.png'))
save_images(
X_orig2_save,
os.path.join(pathForSave, 'iter' + str(k) + '_origX2_img.png'))
save_images(
x2fg_x1bg_out_save,
os.path.join(pathForSave,
'iter' + str(k) + 'x2fg_x1bg_out_img.png'))
save_images(
x1fg_x2bg_out_save,
os.path.join(pathForSave,
'iter' + str(k) + 'x1fg_x2bg_out_img.png'))
def getVisualImgs(self, pathForSave, X1,X2,k, is_training=False):
X_swap = self.session.run(self.x_swap,feed_dict={self.x1: X1,self.x2: X2, self.is_training: is_training})
X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] + self.image_shape)
X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] + self.image_shape)
X_Swap_save = ((X_swap + 1.) * (255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_orig1_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_img.png'))
save_images(X_orig2_save, os.path.join(pathForSave, 'iter' + str(k) + '_orig_swap.png'))
save_images(X_Swap_save, os.path.join(pathForSave, 'iter' + str(k) + '_swap_img.png'))
def visualise_reconstruction(self, X1, aux1,aux2,iteration):
X_out1,mix_head_out,aux1_mix_head_out= self.reconstruct(X1, aux1,aux2)
# print(X_out1.shape)
X1 = ((X_out1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
mix_head_out = ((mix_head_out+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
aux1_mix_head_out = ((aux1_mix_head_out+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X1, os.path.join(self.dirs['samples'], str(iteration)+'samples_1_rec.png'))
save_images(mix_head_out, os.path.join(self.dirs['samples'], str(iteration)+'X1bg_aux1head.png'))
save_images(aux1_mix_head_out, os.path.join(self.dirs['samples'], str(iteration)+'X1head_aux1bg.png'))
def visualise_disentanglement(self, x):
z = self.encode([x])
n_zs = z.shape[1] #self.latent_dist.dim
z = z[0]
rimgs = []
if isinstance(self.z_dist, Gaussian):
for target_z_index in range(n_zs):
for ri in range(self.n_disentangle_samples):
value = -3.0 + 6.0 / (self.n_disentangle_samples-1.) * ri
z_new = np.zeros((1, n_zs))
for i in range(n_zs):
if (i == target_z_index):
z_new[0][i] = value
else:
z_new[0][i] = z[i]
rimgs.append(self.generate(z_mu=z_new))
else:
raise NotImplementedError
rimgs = np.vstack(rimgs).reshape([n_zs, self.n_disentangle_samples, -1]) #.transpose(1,0,2)
rimgs = rimgs[[5,7,2,6,1,9,3]] #order of zs captured
rimgs = ((rimgs+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(rimgs, os.path.join(self.dirs['samples'], 'disentanglement.png'),
n_cols=n_zs, n_rows=self.n_disentangle_samples)
def getVisualImgs(self,
pathForSave,
X1,
mk1,
X2,
mk2,
X3,
mk3,
X4,
mk4,
k,
is_training=False):
X_out1, X_out2, X_r0, X_swap = self.session.run(
[self.x_out1, self.x_out2, self.x_out11, self.x_hybrid3],
feed_dict={
self.x1: X1,
self.x2: X2,
self.mk1: mk1,
self.mk2: mk2,
self.x3: X3,
self.x4: X4,
self.mk3: mk3,
self.mk4: mk4,
self.is_training: is_training
})
X1_save = ((X_out1 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] +
self.image_shape)
X2_save = ((X_out2 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] +
self.image_shape)
X_orig1_save = (X1 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
X_orig2_save = (X2 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
X_orig3_save = (X3 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
X_orig4_save = (X4 * 255.99).astype('int32').reshape([-1] +
self.image_shape)
X_reset0_save = (
(X_r0 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_Swap_save = ((X_swap + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] +
self.image_shape)
save_images(
X_orig1_save,
os.path.join(pathForSave, 'iter' + str(k) + '_orig_img1.png'))
#save_images(X_orig2_save, os.path.join(pathForSave, 'iter'+str(k)+'_orig_img2.png'))
#save_images(X_orig3_save, os.path.join(pathForSave, 'iter'+str(k)+'_orig_img3.png'))
save_images(
X_orig4_save,
os.path.join(pathForSave, 'iter' + str(k) + '_orig_img4.png'))
#save_images(X1_save, os.path.join(pathForSave, 'iter'+str(k)+'_reconst_img1.png'))
#save_images(X2_save, os.path.join(pathForSave, 'iter'+str(k)+'_reconst_img2.png'))
save_images(
X_reset0_save,
os.path.join(pathForSave, 'iter' + str(k) + '_reset0_img.png'))
save_images(
X_Swap_save,
os.path.join(pathForSave, 'iter' + str(k) + '_swap_img.png'))
def visualise_reconstruction(self, X):
X_r = self.reconstruct(X)
X_r = ((X_r+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(X_r, os.path.join(self.dirs['samples'], 'samples_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
count = 0
self.session.run(tf.global_variables_initializer())
# Fixed GT samples - save
fixed_x1, fixed_gt1, _ = next(self.train_iter1)
fixed_x1 = self.session.run(tf.constant(fixed_x1))
save_images(
fixed_x1,
os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png'))
#
fixed_x2, fixed_gt2, _ = next(self.train_iter2)
fixed_x2 = self.session.run(tf.constant(fixed_x2))
save_images(
fixed_x2,
os.path.join(self.dirs['samples'], 'samples_2_groundtruth.png'))
fixed_x3, fixed_mk3, _ = next(self.train_iter3)
fixed_x3 = self.session.run(tf.constant(fixed_x3))
save_images(
fixed_x3,
os.path.join(self.dirs['samples'], 'samples_3_groundtruth.png'))
start_iter = self.load()
running_cost = 0.
fixed_x4, fixed_label, _ = next(self.train_iter4)
fixed_x4 = self.session.run(tf.constant(fixed_x4))
save_images(
fixed_x4,
os.path.join(self.dirs['samples'], 'samples_4_groundtruth.png'))
logs = open('record_loss.txt', 'w')
for iteration in range(start_iter, n_iters):
_data1, _gt1, _ = next(self.train_iter1)
_data2, _gt2, _ = next(self.train_iter2)
_data3, _mask1, _ = next(self.train_iter3)
_data4, _label, _ = next(self.train_iter4)
_, cost = self.session.run(
(self.optimizer, self.loss),
feed_dict={
self.x1: _data1,
self.x2: _data2,
self.x3: _data3,
self.x4: _data4,
self.gt1: _gt1,
self.gt2: _gt2,
self.is_training: True
})
running_cost += cost
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
dev_data1, dev_gt1, _ = next(self.dev_iter1)
dev_data2, dev_gt2, _ = next(self.dev_iter2)
dev_data3, dev_mask1, _ = next(self.dev_iter3)
dev_data4, dev_label, _ = next(self.dev_iter4)
dev_cost = self.session.run(self.loss,
feed_dict={
self.x1: dev_data1,
self.x2: dev_data2,
self.x3: dev_data3,
self.x4: dev_data4,
self.gt1: dev_gt1,
self.gt2: dev_gt2,
self.is_training: False
})
n_samples = 1. if (
iteration < start_iter + 4) else float(stats_iters)
avg_cost = running_cost / n_samples
running_cost = 0.
print(
"Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}"
.format(iteration, avg_cost, dev_cost))
logs.writelines(
"Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}\n"
.format(iteration, avg_cost, dev_cost))
count = count + 1
if self.vis_reconst:
self.visualise_reconstruction(fixed_x1, fixed_x2,
fixed_gt1, fixed_gt2,
fixed_x3, fixed_x4,
iteration)
if np.any(np.isnan(avg_cost)):
raise ValueError("NaN detected!")
# save checkpoint
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session,
os.path.join(self.dirs['ckpt'], self.exp_name),
global_step=iteration)
# for save loss
logs.close()
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
# for save loss
count=0
self.session.run(tf.global_variables_initializer())
# Fixed GT samples - save
fixed_x1, fixed_mk1 ,_ = next(self.train_iter1)
print("fixed_mk1=")
print(fixed_mk1[0:4])
# print(fixed_label[0:4])
# replace mask
#unitLength=3 #(need to changed when has larger unitLength)
unitLength=int(self.latent_dim/self.latent_num)
# generate zero representation and black image and gts0
img_zero,fixed_zero_mk,fixed_gts0=self.generateMaskZeroAndGts(self.batch_size,unitLength)
#
fixed_x1= self.session.run(tf.constant(fixed_x1))
save_images(fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png'))
#
start_iter = self.load()
running_cost = 0.
_gan_data=fixed_x1
logs=open('loss_records.txt','w')
for iteration in range(start_iter, n_iters):
start_time = time.time()
_data1, _mask1,_ = next(self.train_iter1)
_, cost = self.session.run((self.optimizer, self.loss),feed_dict={self.x1: _data1,self.is_training:True})
running_cost += cost
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
t = time.time()
dev_data1, dev_mask1, dev_label1= next(self.dev_iter1)
#dev_cost,dev_rec_loss,dev_reset0_loss,vector_loss,rec_zero_loss,class_loss= self.session.run([self.loss,self.rec_loss,self.reset0_loss,self.vector_loss,self.rec_zero_loss,self.class_loss],feed_dict={self.x1: dev_data1, self.mask: dev_mask1, self.vec_one:vector_one,self.img_black:img_zero,self.mask_zero:fixed_zero_mk,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt3:class_gt3,self.class_gt4:class_gt4,self.class_gt4:class_gt4,self.is_training:False})
dev_cost,dev_rec_loss= self.session.run([self.loss,self.rec_loss],feed_dict={self.x1: dev_data1,self.is_training:False})
n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters)
avg_cost = running_cost / n_samples
running_cost = 0.
print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f})".format(iteration, avg_cost, dev_cost,dev_rec_loss))
logs.writelines("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f})\n".format(
iteration, avg_cost, dev_cost, dev_rec_loss))
count=count+1
if self.vis_reconst:
self.visulize_rec(fixed_x1,iteration)
#self.visualise_reconstruction(img_zero,fixed_mk1,iteration)
if np.any(np.isnan(avg_cost)):
raise ValueError("NaN detected!")
# save checkpoint
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration)
_gan_data=_data1
logs.close()
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
# for save loss
count=0
self.session.run(tf.global_variables_initializer())
# Fixed GT samples - save
fixed_x1,fixed_mask_1,_= next(self.train_iter1)
fixed_x2,fixed_mask_2,_= next(self.train_iter2)
fixed_aux1,fixed_GT1,_= next(self.train_iter3)
fixed_aux2,fixed_GT2,_= next(self.train_iter4)
# print(fixed_x1.shape)
# print(fixed_mask_1.shape)
# print(fixed_x2.shape)
# print(fixed_mask_2.shape)
# print(fixed_aux1.shape)
# print(fixed_GT1.shape)
fixed_x1= self.session.run(tf.constant(fixed_x1))
fixed_mask_1= self.session.run(tf.constant(fixed_mask_1))
# print("fixed_mask_1.shape")
# print(fixed_mask_1.shape)
#fixed_x1_2 =fixed_x1.reshape([64,32,32,3])
#fixed_x1 = ((fixed_x1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(fixed_x1, os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png'))
##save_images(fixed_mask_1, os.path.join(self.dirs['samples'], 'mask_1_groundtruth.png'))
fixed_aux1= self.session.run(tf.constant(fixed_aux1))
fixed_aux2= self.session.run(tf.constant(fixed_aux2))
#fixed_aux1 = ((fixed_aux1+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
#fixed_aux2 = ((fixed_aux2+1.)*(255.99/2)).astype('int32').reshape([-1] + self.image_shape)
save_images(fixed_aux1, os.path.join(self.dirs['samples'], 'aux_1_groundtruth.png'))
save_images(fixed_aux2, os.path.join(self.dirs['samples'], 'aux_2_groundtruth.png'))
#
start_iter = self.load()
running_cost = 0.
class_gt0, class_gt1,class_gt2,class_gt3,class_gt4,class_gt5, class_gt6, class_gt7, class_gt8,class_gt9,class_gt10 = self.generateClassificationLabel(self.batch_size)
_gan_data=fixed_x1
logs=open('loss_records.txt','w')
for iteration in range(start_iter, n_iters):
start_time = time.time()
_data1,_mask1, _ = next(self.train_iter1)
_aux_label, _, _ = next(self.train_iter2)
_aux1,_gt1, _ = next(self.train_iter3)
_aux2,_gt2, _ = next(self.train_iter4)
_, cost = self.session.run((self.optimizer, self.loss),feed_dict={self.x1:_data1,self.aux_class_gt:_aux_label,self.aux1_mask:_mask1,self.aux1:_aux1,self.aux2:_aux2,self.aux_GT1:_gt1,self.aux_GT2:_gt2,self.is_training:True,self.class_gt0:class_gt0,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt3:class_gt3,
self.class_gt4:class_gt4, self.class_gt5:class_gt5, self.class_gt6:class_gt6,self.class_gt7:class_gt7, self.class_gt8:class_gt8, self.class_gt9:class_gt9, self.class_gt10:class_gt10})
running_cost += cost
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
t = time.time()
dev_data1,dev_mask1, _ = next(self.dev_iter1)
dev_aux1,dev_gt1, _ = next(self.dev_iter3)
dev_aux2,dev_gt2, _ = next(self.dev_iter4)
dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss= \
self.session.run([self.loss,self.rec_img_loss1,self.rec_aux1_loss2,self.rec_aux2_loss3,self.rec_aux1_swap_loss4,self.rec_aux2_swap_loss5,self.rec_dual_loss6,self.head_loss,self.class_loss,self.fuzzy_class_loss,self.aux1_class_loss,self.aux2_class_loss,self.fuzzy_bg_class_loss,self.aux1_bg_class_loss,self.aux2_bg_class_loss],
feed_dict={self.x1:dev_data1,self.aux1_mask:dev_mask1,self.aux1:dev_aux1,self.aux2:dev_aux2,self.aux_GT1:dev_gt1,self.aux_GT2:dev_gt2,self.is_training:False,
self.class_gt0: class_gt0, self.class_gt1: class_gt1,self.class_gt2: class_gt2,self.class_gt3:class_gt3,self.aux_class_gt:_aux_label,self.class_gt4:class_gt4, self.class_gt5:class_gt5, self.class_gt6:class_gt6,self.class_gt7:class_gt7, self.class_gt8:class_gt8, self.class_gt9:class_gt9, self.class_gt10:class_gt10})
n_samples = 1. if (iteration < start_iter + 4) else float(stats_iters)
avg_cost = running_cost / n_samples
running_cost = 0.
print("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(img1_loss:{3:.1f},aux1_loss2:{4:.1f},aux2_loss3:{5:.1f},aux1_swap_loss:{6:.1f},aux2_swap_loss:{7:.1f},dual_swap_loss:{8:.1f},head loss:{9:.1f},class loss:{10:.1f}(fuzzy_class:{11:.1f},aux1_class_loss:{12:.1f},aux2_class_loss:{13:.1f},fuzzy_bg_class_loss:{14:.1f},aux1_bg_class_loss:{15:.1f},aux2_bg_class_loss:{16:.1f}))".
format(iteration, avg_cost, dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss))
logs.writelines("Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(img1_loss:{3:.1f},aux1_loss2:{4:.1f},aux2_loss3:{5:.1f},aux1_swap_loss:{6:.1f},aux2_swap_loss:{7:.1f},dual_swap_loss:{8:.1f},head loss:{9:.1f},class loss:{10:.1f}(fuzzy_class:{11:.1f},aux1_class_loss:{12:.1f},aux2_class_loss:{13:.1f},fuzzy_bg_class_loss:{14:.1f},aux1_bg_class_loss:{15:.1f},aux2_bg_class_loss:{16:.1f}))\n".
format(iteration, avg_cost, dev_loss,dev_rec_img_loss1,dev_rec_aux1_loss2,dev_rec_aux2_loss3,dev_rec_aux1_swap_loss4,dev_rec_aux2_swap_loss5,dev_rec_dual_loss6,head_loss,class_loss,fuzzy_class,aux1_class_loss,aux2_class_loss,fuzzy_bg_class_loss,aux1_bg_class_loss,aux2_bg_class_loss))
count=count+1
if self.vis_reconst:
self.visualise_reconstruction(fixed_x1,fixed_aux1,fixed_aux2,iteration)
#self.visualise_reconstruction(img_zero,fixed_mk1,iteration)
if np.any(np.isnan(avg_cost)):
raise ValueError("NaN detected!")
# save checkpoint
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session, os.path.join(self.dirs['ckpt'], self.exp_name), global_step=iteration)
_gan_data=_data1
logs.close()
def visualise_reconstruction(self, X1, X2, gt1, gt2, X3, X4, iteration):
X_r1, X_r2, X_r11, X_r22, X_r33, X_r44 = self.reconstruct(
X1, X2, gt1, gt2, X3, X4)
X_r1 = ((X_r1 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r2 = ((X_r2 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r11 = ((X_r11 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r22 = ((X_r22 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r33 = ((X_r33 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r44 = ((X_r44 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(
X_r1,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_1_reconstructed.png'))
save_images(
X_r2,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_2_reconstructed.png'))
save_images(
X_r11,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_11_reconstructed.png'))
save_images(
X_r22,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_22_reconstructed.png'))
save_images(
X_r33,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_33_reconstructed.png'))
save_images(
X_r44,
os.path.join(self.dirs['samples'],
str(iteration) + 'samples_44_reconstructed.png'))
def encodeImg(self,
pathForSave,
ratio,
X1,
mk1,
X2,
mk2,
k,
is_training=False):
X_r1, X_r2, X_r11, X_r22, X_r1twice, X_r2twice = self.session.run(
[
self.x_out3, self.x_out4, self.x_hybrid3, self.x_hybrid4,
self.x_out33, self.x_out44
],
feed_dict={
self.x3: X1,
self.x4: X2,
self.mk3: mk1,
self.mk4: mk2,
self.is_training: is_training
})
X_r1 = ((X_r1 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r2 = ((X_r2 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r11 = ((X_r11 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
X_r22 = ((X_r22 + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] + self.image_shape)
save_images(
X_r1,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_3_reconstructed.png'))
save_images(
X_r2,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_4_reconstructed.png'))
save_images(
X_r11,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_hrd3_reconstructed.png'))
save_images(
X_r22,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_hrd4_reconstructed.png'))
X_r1twice = ((X_r1twice + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] +
self.image_shape)
X_r2twice = ((X_r2twice + 1.) *
(255.99 / 2)).astype('int32').reshape([-1] +
self.image_shape)
save_images(
X_r1twice,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_3_twice_reconstructed.png'))
save_images(
X_r2twice,
os.path.join(
pathForSave, 'ratio=' + str(ratio) + '_iter' + str(k) +
'_samples_4_twice_reconstructed.png'))
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
self.session.run(tf.global_variables_initializer())
# Fixed GT samples
fixed_x, _ = next(self.train_iter)
fixed_x = self.session.run(tf.constant(fixed_x))
save_images(
fixed_x,
os.path.join(self.dirs['samples'], 'samples_groundtruth.png'))
# Fixed prior sample, for generating samples
fixed_z_c = self.session.run(
self.latent_dist.sample_prior(self.batch_size))
log_vars = [x for _, x in self.log_vars]
log_keys = [x for x, _ in self.log_vars]
running_log_vals = [0.] * len(self.log_vars)
start_iter = self.load()
for iteration in range(start_iter, n_iters):
start_time = time.time()
# Train generator
if iteration > start_iter:
_ = self.session.run(self.gen_opt,
feed_dict={
self.x: _data,
self.is_training: True
})
# Train critic
if self.mode == 'wgan-gp':
disc_iters = self.critic_iters
else:
disc_iters = 1
for i in range(disc_iters):
_data, _ = next(self.train_iter)
log_vals = self.session.run([self.disc_opt] + log_vars,
feed_dict={
self.x: _data,
self.is_training: True
})[1:]
running_log_vals = [
running + batch
for running, batch in zip(running_log_vals, log_vals)
]
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
t = time.time()
dev_data, _ = next(self.dev_iter)
dev_log_vals = self.session.run(
log_vars + [self.q_c_given_x_real_dist_info],
feed_dict={
self.x: dev_data,
self.is_training: False
})
dev_c_dist_info = dev_log_vals[-1]
dev_log_vals = dev_log_vals[:-1]
n_samples = 1. if (
iteration < start_iter + 4) else float(stats_iters)
avg_log_vals = [
r_log_val / n_samples for r_log_val in running_log_vals
]
running_log_vals = [0.] * len(self.log_vars)
tr_log_line = " | ".join(
"{0}: {1:.2f}".format(k, v)
for k, v in zip(log_keys, avg_log_vals))
dev_log_line = " | ".join(
"{0}: {1:.2f}".format(k, v)
for k, v in zip(log_keys, dev_log_vals))
print("Iteration:{0} \nTrain: {1} \nDev : {2}".format(
iteration, tr_log_line, dev_log_line))
if isinstance(self.c_dist,
Gaussian) and not self.c_dist._fix_std:
avg_dev_var = np.mean(dev_c_dist_info["stddev"]**2, axis=0)
cs_str = ""
for i, cs in enumerate(avg_dev_var):
c_str = "c{0}={1:.2f}".format(i, cs)
cs_str += c_str + ", "
print("c variance:{0}".format(cs_str))
fixed_samples = self.generate(
z_c=fixed_z_c).reshape([-1] + self.image_shape)
save_images(
fixed_samples,
os.path.join(self.dirs['samples'],
'samples_generated.png'))
if self.vis_reconst:
self.visualise_reconstruction(fixed_x, fixed_z_c)
if self.vis_disent:
self.visualise_disentanglement(fixed_x[0], fixed_z_c)
if np.any(np.isnan(avg_log_vals)):
raise ValueError("NaN detected!")
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session,
os.path.join(self.dirs['ckpt'], self.exp_name),
global_step=iteration)
def train(self, n_iters, n_iters_per_epoch, stats_iters, ckpt_interval):
# for save loss
logArray = np.zeros((1000, 4))
count = 0
self.session.run(tf.global_variables_initializer())
# Fixed GT samples - save
fixed_x1, fixed_mk1, _ = next(self.train_iter1)
print("fixed_mk1=")
print(fixed_mk1[0:4])
# replace mask
unitLength = 1 #(need to changed when has larger unitLength)
# get classification gt and label
class_gt0, class_gt1, class_gt2, vector_one = self.generateClassificationLabelandVecOne(
self.batch_size)
# generate zero representation and white image
img_zero, fixed_zero_mk = self.generateMaskZero(
self.batch_size, unitLength)
#
fixed_x1 = self.session.run(tf.constant(fixed_x1))
save_images(
fixed_x1,
os.path.join(self.dirs['samples'], 'samples_1_groundtruth.png'))
#
fixed_img_zero = self.session.run(tf.constant(img_zero * 0.0))
save_images(
fixed_img_zero,
os.path.join(self.dirs['samples'],
'samples_white_groundtruth.png'))
start_iter = self.load()
running_cost = 0.
GAN_runing_gen_cost = 0.
GAN_runing_dis_cost = 0.
_gan_data = fixed_x1
for iteration in range(start_iter, n_iters):
start_time = time.time()
_data1, _mask1, _ = next(self.train_iter1)
_, cost = self.session.run(
(self.optimizer, self.loss),
feed_dict={
self.x1: _data1,
self.mask: _mask1,
self.vec_one: vector_one,
self.img_white: img_zero,
self.mask_zero: fixed_zero_mk,
self.class_gt0: class_gt0,
self.class_gt1: class_gt1,
self.class_gt2: class_gt2,
self.is_training: True
})
running_cost += cost
#==generator training
_, _disc_gene_cost = self.session.run(
(self.gen_train_optimizer, self.gen_cost),
feed_dict={
self.x1: _data1,
self.x_gan: _gan_data,
self.mask: _mask1,
self.vec_one: vector_one,
self.img_white: img_zero,
self.mask_zero: fixed_zero_mk,
self.class_gt0: class_gt0,
self.class_gt1: class_gt1,
self.class_gt2: class_gt2,
self.is_training: True
})
GAN_runing_gen_cost += _disc_gene_cost
#==Discriminator training
for i in range(CRITIC_ITERS):
_, _disc_disc_cost = self.session.run(
(self.disc_train_optimizer, self.disc_cost),
feed_dict={
self.x1: _data1,
self.x_gan: _gan_data,
self.mask: _mask1,
self.vec_one: vector_one,
self.img_white: img_zero,
self.mask_zero: fixed_zero_mk,
self.class_gt0: class_gt0,
self.class_gt1: class_gt1,
self.class_gt2: class_gt2,
self.is_training: True
})
GAN_runing_dis_cost += _disc_disc_cost
GAN_runing_dis_cost = GAN_runing_dis_cost / CRITIC_ITERS
if iteration % n_iters_per_epoch == 1:
print("Epoch: {0}".format(iteration // n_iters_per_epoch))
# Print avg stats and dev set stats
if (iteration < start_iter + 4) or iteration % stats_iters == 0:
t = time.time()
dev_data1, dev_mask1, _ = next(self.dev_iter1)
#dev_cost,dev_rec_loss,dev_reset0_loss,vector_loss,rec_zero_loss,class_loss= self.session.run([self.loss,self.rec_loss,self.reset0_loss,self.vector_loss,self.rec_zero_loss,self.class_loss],feed_dict={self.x1: dev_data1, self.mask: dev_mask1, self.vec_one:vector_one,self.img_white:img_zero,self.mask_zero:fixed_zero_mk,self.class_gt1:class_gt1,self.class_gt2:class_gt2,self.class_gt4:class_gt4,self.is_training:False})
dev_cost, dev_rec_loss, dev_reset0_loss, rec_zero_loss, class_loss, gen_cost, disc_cost = self.session.run(
[
self.loss, self.rec_loss, self.reset0_loss,
self.rec_zero_loss, self.class_loss, self.gen_cost,
self.disc_cost
],
feed_dict={
self.x1: dev_data1,
self.x_gan: _gan_data,
self.mask: dev_mask1,
self.vec_one: vector_one,
self.img_white: img_zero,
self.mask_zero: fixed_zero_mk,
self.class_gt0: class_gt0,
self.class_gt1: class_gt1,
self.class_gt2: class_gt2,
self.is_training: False
})
n_samples = 1. if (
iteration < start_iter + 4) else float(stats_iters)
avg_cost = running_cost / n_samples
avg_GAN_runing_gen_cost = GAN_runing_gen_cost / n_samples
avg_GAN_runing_dis_cost = GAN_runing_dis_cost / n_samples
running_cost = 0.
GAN_runing_gen_cost = 0.
GAN_runing_dis_cost = 0.
print(
"Iteration:{0} \t| Train cost:{1:.1f} \t| Dev cost: {2:.1f}(reconstr_loss:{3:.1f},reset0_loss:{4:.1f},rec_zero_loss:{5:.1f},class_loss:{6:.1f})"
.format(iteration, avg_cost, dev_cost, dev_rec_loss,
dev_reset0_loss, rec_zero_loss, class_loss))
print(
"Iteration:{0} \t| Train gen_cost:{1:.1f} disc_cost:{2:.1f}\t| Dev gen_cost:{3:.1f} disc_cost:{4:.1f}"
.format(iteration, avg_GAN_runing_gen_cost,
avg_GAN_runing_dis_cost, gen_cost, disc_cost))
# for save loss
logArray[count, 0] = iteration // n_iters_per_epoch
logArray[count, 1] = iteration
logArray[count, 2] = avg_cost
logArray[count, 3] = dev_cost
count = count + 1
if self.vis_reconst:
self.visualise_reconstruction(fixed_x1, fixed_mk1,
iteration)
#self.visualise_reconstruction(img_zero,fixed_mk1,iteration)
if np.any(np.isnan(avg_cost)):
raise ValueError("NaN detected!")
# save checkpoint
if (iteration > start_iter) and iteration % (ckpt_interval) == 0:
self.saver.save(self.session,
os.path.join(self.dirs['ckpt'], self.exp_name),
global_step=iteration)
_gan_data = _data1
# for save loss
np.save('logArray.npy', logArray)
