def sample_from_z(self, config, z):
counter = 0
if config.rotate_azimuth:
low = cfg['azi_low']
high = cfg['azi_high']
step = STEP
elif config.rotate_elevation:
low = cfg['ele_low']
high = cfg['ele_high']
step = STEP
else:
low = 0
high = 90
step = 30
for i in range(low, high, step):
if config.rotate_azimuth:
sample_view = np.tile(
np.array([
i * math.pi / 180.0, 0 * math.pi / 180.0, 1.0, 0, 0, 0
]), (cfg['batch_size'], 1))
elif config.rotate_azimuth:
sample_view = np.tile(
np.array([
270 * math.pi / 180.0, (90 - i) * math.pi / 180.0, 1.0,
0, 0, 0
]), (cfg['batch_size'], 1))
else:
sample_view = self.gen_view_func(
cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=False,
with_scale=to_bool(str(cfg['with_translation'])))
feed_eval = {self.z: z, self.view_in: sample_view}
samples = self.sess.run(self.G, feed_dict=feed_eval)
ren_img = np.clip(255 * samples, 0, 255).astype(np.uint8)
try:
cv2.imwrite(
os.path.join(self.sample_dir,
"{0}_samples_{1}.jpg".format(counter, i)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
except:
cv2.imwrite(
os.path.join(self.sample_dir,
"{0}_samples_{1}.jpg".format(counter, i)),
ren_img[0])
def sample_HoloGAN(self, config):
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
return
SAMPLE_DIR = os.path.join(OUTPUT_DIR, "samples")
if not os.path.exists(SAMPLE_DIR):
os.makedirs(SAMPLE_DIR)
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
if config.rotate_azimuth:
low = cfg['azi_low']
high = cfg['azi_high']
step = 10
elif config.rotate_elevation:
low = cfg['ele_low']
high = cfg['ele_high']
step = 5
else:
low = 0
high = 10
step = 1
for i in range(low, high, step):
if config.rotate_azimuth:
sample_view = np.tile(
np.array([i * math.pi / 180.0, 0 * math.pi / 180.0, 1.0, 0, 0, 0]), (cfg['batch_size'], 1))
elif config.rotate_azimuth:
sample_view = np.tile(
np.array([270 * math.pi / 180.0, (90 - i) * math.pi / 180.0, 1.0, 0, 0, 0]), (cfg['batch_size'], 1))
else:
sample_view = self.gen_view_func(cfg['batch_size'],
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=False,
with_scale=to_bool(str(cfg['with_translation'])))
feed_eval = {self.z: sample_z,
self.view_in: sample_view}
samples = self.sess.run(self.G, feed_dict=feed_eval)
ren_img = inverse_transform(samples)
ren_img = np.clip(255 * ren_img, 0, 255).astype(np.uint8)
try:
scipy.misc.imsave(
os.path.join(SAMPLE_DIR, "{0}_samples_{1}.png".format(counter, i)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
except:
scipy.misc.imsave(
os.path.join(SAMPLE_DIR, "{0}_samples_{1}.png".format(counter, i)),
ren_img[0])
def generate_images(self, config):
count = 0
could_load, checkpoint_counter = self.load()
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
return
if not os.path.exists(self.sample_dir):
os.makedirs(self.sample_dir)
for i in range(100):
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
sample_view = self.gen_view_func(
cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
feed_eval = {self.z: sample_z, self.view_in: sample_view}
samples = self.sess.run(self.G, feed_dict=feed_eval)
ren_img = np.clip(255 * samples, 0, 255).astype(np.uint8)
for img in ren_img:
cv2.imwrite(os.path.join(GEN_DIR, "{0}.jpg".format(count)),
img)
count = count + 1
print('Image saved: ', count)
def train_HoloGAN(self, config):
self.d_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
self.g_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
d_optim = tf.train.AdamOptimizer(cfg['d_eta'], beta1=cfg['beta1'], beta2=cfg['beta2']).minimize(self.d_loss, var_list=self.d_vars)
g_optim = tf.train.AdamOptimizer(cfg['g_eta'], beta1=cfg['beta1'], beta2=cfg['beta2']).minimize(self.g_loss, var_list=self.g_vars)
tf.global_variables_initializer().run()
shutil.copyfile(sys.argv[1], os.path.join(LOGDIR, 'config.json'))
self.g_sum = merge_summary([self.d_loss_fake_sum, self.g_loss_sum])
self.d_sum = merge_summary([self.d_loss_real_sum, self.d_loss_sum])
self.writer = SummaryWriter(os.path.join(LOGDIR, str(dt.now())), self.sess.graph)
num_objs = 1
# Sample noise Z and view parameters to test during training
sample_z_bg = self.sampling_Z(cfg['z_dim_bg'], num_objs=1, type=str(cfg['sample_z']))
sample_z_fg = self.sampling_Z(cfg['z_dim_fg'], num_objs=num_objs, type=str(cfg['sample_z']))
sample_view_bg = self.gen_view_func(cfg['batch_size'] * 1,
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
sample_view_fg = self.gen_view_func(cfg['batch_size'] * num_objs,
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
# sample_view_bg = tf.reshape(sample_view_bg, (cfg['batch_size'], 1, *sample_view_bg.shape[1:]))
# sample_view_fg = tf.reshape(sample_view_fg, (cfg['batch_size'], num_objs, *sample_view_fg.shape[1:]))
sample_files = self.data[0:cfg['batch_size']]
if config.dataset == "cats" or config.dataset == "cars":
sample_images = [get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop) for sample_file in sample_files]
else:
sample_images = [get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop) for sample_file in sample_files]
## TEMP
batch_images = sample_images
# batch_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
# batch_view = self.gen_view_func(cfg['batch_size'],
# cfg['ele_low'], cfg['ele_high'],
# cfg['azi_low'], cfg['azi_high'],
# cfg['scale_low'], cfg['scale_high'],
# cfg['x_low'], cfg['x_high'],
# cfg['y_low'], cfg['y_high'],
# cfg['z_low'], cfg['z_high'],
# with_translation=False,
# with_scale=to_bool(str(cfg['with_translation'])))
batch_z_bg = self.sampling_Z(cfg['z_dim_bg'], num_objs=1, type=str(cfg['sample_z']))
batch_z_fg = self.sampling_Z(cfg['z_dim_fg'], num_objs=num_objs, type=str(cfg['sample_z']))
batch_view_bg = self.gen_view_func(cfg['batch_size'] * 1,
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
batch_view_fg = self.gen_view_func(cfg['batch_size'] * num_objs,
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
# batch_view_bg = tf.reshape(batch_view_bg, (cfg['batch_size'], 1, *batch_view_bg.shape[1:]))
# batch_view_fg = tf.reshape(batch_view_fg, (cfg['batch_size'], num_objs, *batch_view_fg.shape[1:]))
counter = 0
start_time = time.time()
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
self.data = glob.glob(os.path.join(IMAGE_PATH, self.input_fname_pattern))
d_lr = cfg['d_eta']
g_lr = cfg['g_eta']
for epoch in range(cfg['max_epochs']):
d_lr = d_lr if epoch < cfg['epoch_step'] else d_lr * (cfg['max_epochs'] - epoch) / (cfg['max_epochs'] - cfg['epoch_step'])
g_lr = g_lr if epoch < cfg['epoch_step'] else g_lr * (cfg['max_epochs'] - epoch) / (cfg['max_epochs'] - cfg['epoch_step'])
random.shuffle(self.data)
batch_idxs = min(len(self.data), config.train_size) // cfg['batch_size']
for idx in range(0, batch_idxs):
batch_files = self.data[idx * cfg['batch_size']:(idx + 1) * cfg['batch_size']]
# if config.dataset == "cats" or config.dataset == "cars":
# batch_images = [get_image(batch_file,
# input_height=self.input_height,
# input_width=self.input_width,
# resize_height=self.output_height,
# resize_width=self.output_width,
# crop=self.crop) for batch_file in batch_files]
# else:
# batch_images = [get_image(batch_file,
# input_height=self.input_height,
# input_width=self.input_width,
# resize_height=self.output_height,
# resize_width=self.output_width,
# crop=self.crop) for batch_file in batch_files]
# batch_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
# batch_view = self.gen_view_func(cfg['batch_size'],
# cfg['ele_low'], cfg['ele_high'],
# cfg['azi_low'], cfg['azi_high'],
# cfg['scale_low'], cfg['scale_high'],
# cfg['x_low'], cfg['x_high'],
# cfg['y_low'], cfg['y_high'],
# cfg['z_low'], cfg['z_high'],
# with_translation=False,
# with_scale=to_bool(str(cfg['with_translation'])))
feed = {self.inputs: batch_images,
self.z_bg: batch_z_bg,
self.z_fg: batch_z_fg,
# self.z: batch_z,
self.view_in_bg: batch_view_bg,
self.view_in_fg: batch_view_fg,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr}
# Update D network
_, summary_str = self.sess.run([d_optim, self.d_sum],feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Update G network
_, summary_str = self.sess.run([g_optim, self.g_sum], feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Run g_optim twice
_, summary_str = self.sess.run([g_optim, self.g_sum], feed_dict=feed)
self.writer.add_summary(summary_str, counter)
errD_fake = self.d_loss_fake.eval(feed)
errD_real = self.d_loss_real.eval(feed)
errG = self.g_loss.eval(feed)
# errQ = self.q_loss.eval(feed)
errQ = 0.
print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f, q_loss: %.8f" \
% (epoch, idx, batch_idxs,
time.time() - start_time, errD_fake + errD_real, errG, errQ))
if np.mod(counter, 100) == 0:
# self.save(LOGDIR, counter)
feed_eval = {self.inputs: sample_images,
self.z_bg: sample_z_bg,
self.z_fg: sample_z_fg,
# self.z: batch_z,
self.view_in_bg: sample_view_bg,
self.view_in_fg: sample_view_fg,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr}
samples, d_loss, g_loss = self.sess.run(
[self.G, self.d_loss, self.g_loss],
feed_dict=feed_eval)
print('generated samples')
print(samples)
ren_img = inverse_transform(samples)
ren_img = np.clip(255 * ren_img, 0, 255).astype(np.uint8)
# real_img = inverse_transform(np.array(sample_images))
# real_img = np.clip(255 * real_img, 0, 255).astype(np.uint8)
try:
self.writer.add_summary(tf.summary.image("{0}_GAN.png".format(counter), ren_img).eval(), counter)
imsave(
os.path.join(OUTPUT_DIR, "{0}_GAN.png".format(counter)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
# imsave(
# os.path.join(OUTPUT_DIR, "{0}_reals_GAN.png".format(counter)),
# merge(real_img, [cfg['batch_size'] // 4, 4]))
# print("[Sample] d_loss: %.8f, g_loss: %.8f" % (d_loss, g_loss))
except:
self.writer.add_summary(tf.summary.image("{0}_GAN.png".format(counter), ren_img[:1]).eval(), counter)
imsave(
os.path.join(OUTPUT_DIR, "{0}_GAN.png".format(counter)),
ren_img[0])
# imsave(
# os.path.join(OUTPUT_DIR, "{0}_reals_GAN.png".format(counter)),
# real_img[0])
# print("[Sample] d_loss: %.8f, g_loss: %.8f" % (d_loss, g_loss))
counter += 1
return
self.save(LOGDIR, counter)
def train_HoloGAN(self, config):
self.d_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
self.g_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
d_optim = tf.train.AdamOptimizer(cfg['d_eta'], beta1=cfg['beta1'], beta2=cfg['beta2']).minimize(self.d_loss, var_list=self.d_vars)
g_optim = tf.train.AdamOptimizer(cfg['g_eta'], beta1=cfg['beta1'], beta2=cfg['beta2']).minimize(self.g_loss, var_list=self.g_vars)
tf.global_variables_initializer().run()
shutil.copyfile(sys.argv[1], os.path.join(LOGDIR, 'config.json'))
self.g_sum = merge_summary([self.d_loss_fake_sum, self.g_loss_sum])
self.d_sum = merge_summary([self.d_loss_real_sum, self.d_loss_sum])
self.writer = SummaryWriter(LOGDIR, self.sess.graph)
# Sample noise Z and view parameters to test during training
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
sample_view = self.gen_view_func(cfg['batch_size'],
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=False,
with_scale=to_bool(str(cfg['with_translation'])))
sample_files = self.data[0:cfg['batch_size']]
if config.dataset in ['cats', 'cars']:
sample_images = [get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False)
for sample_file in sample_files]
else:
sample_images = [get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=True)
for sample_file in sample_files]
counter = 1
start_time = time.time()
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(' [*] Load SUCCESS')
else:
print(' [!] Load failed...')
self.data = glob.glob(os.path.join(IMAGE_PATH, self.input_fname_pattern))
d_lr = cfg['d_eta']
g_lr = cfg['g_eta']
for epoch in range(cfg['max_epochs']):
d_lr = d_lr if epoch < cfg['epoch_step'] else d_lr * (cfg['max_epochs'] - epoch) / (cfg['max_epochs'] - cfg['epoch_step'])
g_lr = g_lr if epoch < cfg['epoch_step'] else g_lr * (cfg['max_epochs'] - epoch) / (cfg['max_epochs'] - cfg['epoch_step'])
random.shuffle(self.data)
batch_idxs = min(len(self.data), config.train_size) // cfg['batch_size']
for idx in range(0, batch_idxs):
batch_files = self.data[idx * cfg['batch_size']:(idx + 1) * cfg['batch_size']]
if config.dataset == 'cats' or config.dataset == 'cars':
batch_images = [get_image(batch_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False)
for batch_file in batch_files]
else:
batch_images = [get_image(batch_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop)
for batch_file in batch_files]
batch_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
batch_view = self.gen_view_func(cfg['batch_size'],
cfg['ele_low'], cfg['ele_high'],
cfg['azi_low'], cfg['azi_high'],
cfg['scale_low'], cfg['scale_high'],
cfg['x_low'], cfg['x_high'],
cfg['y_low'], cfg['y_high'],
cfg['z_low'], cfg['z_high'],
with_translation=False,
with_scale=to_bool(str(cfg['with_translation'])))
feed = {self.inputs: batch_images,
self.z: batch_z,
self.view_in: batch_view,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr}
# Update D network
_, summary_str = self.sess.run([d_optim, self.d_sum], feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Update G network
_, summary_str = self.sess.run([g_optim, self.g_sum], feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Run g_optim twice
_, summary_str = self.sess.run([g_optim, self.g_sum], feed_dict=feed)
self.writer.add_summary(summary_str, counter)
errD_fake = self.d_loss_fake.eval(feed)
errD_real = self.d_loss_real.eval(feed)
errG = self.g_loss.eval(feed)
errQ = self.q_loss.eval(feed)
counter += 1
print('Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f, q_loss: %.8f' \
% (epoch, idx, batch_idxs, time.time() - start_time, errD_fake + errD_real, errG, errQ))
if np.mod(counter, 1000) == 1:
self.save(LOGDIR, counter)
feed_eval = {self.inputs: sample_images,
self.z: sample_z,
self.view_in: sample_view,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr}
samples, d_loss, g_loss = self.sess.run([self.G, self.d_loss, self.g_loss], feed_dict=feed_eval)
ren_img = inverse_transform(samples)
ren_img = np.clip(255 * ren_img, 0, 255).astype(np.uint8)
try:
scipy.misc.imsave(
os.path.join(OUTPUT_DIR, '{0}_GAN.png'.format(counter)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
print('[Sample] d_loss: %.8f, g_loss: %.8f' % (d_loss, g_loss))
except:
scipy.misc.imsave(
os.path.join(OUTPUT_DIR, '{0}_GAN.png'.format(counter)),
ren_img[0])
print('[Sample] d_loss: %.8f, g_loss: %.8f' % (d_loss, g_loss))
def sample_HoloGAN(self):
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
return
SAMPLE_DIR = os.path.join(OUTPUT_DIR, "samples")
if not os.path.exists(SAMPLE_DIR):
os.makedirs(SAMPLE_DIR)
# sample_z = self.sampling_Z(cfg.z_dim, str(cfg.sample_z))
if cfg.rotate_azimuth:
low = cfg.azi_low
high = cfg.azi_high
step = 10
elif cfg.rotate_elevation:
low = cfg.ele_low
high = cfg.ele_high
step = 5
else:
low = 0
high = 10
step = 1
for i in range(low, high, step):
if cfg.rotate_azimuth:
sample_view = np.tile(
np.array([
i * math.pi / 180.0, 0 * math.pi / 180.0, 1.0, 0, 0, 0
]), (cfg.batch_size, 1))
elif cfg.rotate_elevation:
sample_view = np.tile(
np.array([
270 * math.pi / 180.0, (90 - i) * math.pi / 180.0, 1.0,
0, 0, 0
]), (cfg.batch_size, 1))
else:
sample_view = self.gen_view_func(
cfg.batch_size,
cfg.ele_low,
cfg.ele_high,
cfg.azi_low,
cfg.azi_high,
cfg.scale_low,
cfg.scale_high,
cfg.x_low,
cfg.x_high,
cfg.y_low,
cfg.y_high,
cfg.z_low,
cfg.z_high,
with_translation=False,
with_scale=to_bool(str(cfg.with_translation)))
# Use global vectors
feed_eval = {self.z: SAMPLE_Z, self.view_in: SAMPLE_VIEW}
samples = self.sess.run(self.G, feed_dict=feed_eval)
ren_img = inverse_transform(samples)
ren_img = np.clip(255 * ren_img, 0, 255).astype(np.uint8)
grid_x = GRID_X
if ADD_EMBEDDING:
ren_img = np.concatenate([self.images, ren_img], axis=0)
grid_x += 1
Image.fromarray(merge(ren_img, [grid_x, GRID_Y]).astype(
np.uint8)).save(
os.path.join(SAMPLE_DIR,
"{0}_samples_{1}.png".format(counter, i)))
def train_HoloGAN(self):
self.d_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
self.g_lr_in = tf.placeholder(tf.float32, None, name='d_eta')
d_optim = tf.train_on_loader.AdamOptimizer(cfg.d_eta,
beta1=cfg.beta1,
beta2=cfg.beta2).minimize(
self.d_loss,
var_list=self.d_vars)
g_optim = tf.train_on_loader.AdamOptimizer(cfg.g_eta,
beta1=cfg.beta1,
beta2=cfg.beta2).minimize(
self.g_loss,
var_list=self.g_vars)
tf.global_variables_initializer().run()
self.g_sum = merge_summary([self.d_loss_fake_sum, self.g_loss_sum])
self.d_sum = merge_summary([self.d_loss_real_sum, self.d_loss_sum])
self.writer = SummaryWriter(LOGDIR, self.sess.graph)
# Sample noise Z and view parameters to test during training
# sample_z = self.sampling_Z(cfg.z_dim - cfg.emb_dim, str(cfg.sample_z))
# sample_view = self.gen_view_func(cfg.batch_size,
# cfg.ele_low, cfg.ele_high,
# cfg.azi_low, cfg.azi_high,
# cfg.scale_low, cfg.scale_high,
# cfg.x_low, cfg.x_high,
# cfg.y_low, cfg.y_high,
# cfg.z_low, cfg.z_high,
# with_translation=False,
# with_scale=to_bool(str(cfg.with_translation)))
sample_files = self.data[0:cfg.batch_size]
if cfg.dataset == "cats" or cfg.dataset == "cars":
sample_images = [
get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False) for sample_file in sample_files
]
else:
sample_images = [
get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop) for sample_file in sample_files
]
counter = 1
start_time = time.time()
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
self.data = glob.glob(
os.path.join(IMAGE_PATH, self.input_fname_pattern))
d_lr = cfg.d_eta
g_lr = cfg.g_eta
for epoch in range(cfg.max_epochs):
d_lr = d_lr if epoch < cfg.epoch_step else d_lr * (
cfg.max_epochs - epoch) / (cfg.max_epochs - cfg.epoch_step)
g_lr = g_lr if epoch < cfg.epoch_step else g_lr * (
cfg.max_epochs - epoch) / (cfg.max_epochs - cfg.epoch_step)
random.shuffle(self.data)
batch_idxs = min(len(self.data), cfg.train_size) // cfg.batch_size
for idx in tqdm(range(0, batch_idxs)):
batch_files = self.data[idx * cfg.batch_size:(idx + 1) *
cfg.batch_size]
if cfg.dataset == "cats" or cfg.dataset == "cars":
batch_images = [
get_image(batch_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False) for batch_file in batch_files
]
else:
batch_images = [
get_image(batch_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop) for batch_file in batch_files
]
if ADD_EMBEDDING:
batch_z = self.sampling_Z(cfg.z_dim - cfg.emb_dim,
str(cfg.sample_z))
else:
batch_z = self.sampling_Z(cfg.z_dim, str(cfg.sample_z))
# batch_z = self.sampling_Z(cfg.z_dim, str(cfg.sample_z))
# batch_emb = torch.stack([self.emb_transforms(Image.open(file).convert('RGB')) for file in batch_files])
# batch_emb = self.embedder(batch_emb.to(torch.float32).cuda()).cpu().detach().numpy()
if ADD_EMBEDDING:
batch_emb = [
self.emb_transforms(Image.open(file).convert('RGB'))
for file in batch_files
]
if len(batch_emb) != cfg.batch_size:
batch_emb = batch_emb * cfg.batch_size
batch_emb = batch_emb[:cfg.batch_size]
batch_emb = torch.stack(batch_emb).numpy()
batch_emb = self.sess_emb.run(self.emb_output,
{self.emb_input: batch_emb})
batch_z = np.hstack((batch_z, batch_emb))
batch_view = self.gen_view_func(cfg.batch_size,
cfg.ele_low,
cfg.ele_high,
cfg.azi_low,
cfg.azi_high,
cfg.scale_low,
cfg.scale_high,
cfg.x_low,
cfg.x_high,
cfg.y_low,
cfg.y_high,
cfg.z_low,
cfg.z_high,
with_translation=False,
with_scale=to_bool(
str(cfg.with_translation)))
# control_weight = [
# op for op in tf.get_default_graph().get_operations()
# if op.name == 'layer3.20.conv1.weight']
# control_weight = control_weight[0].values()[0].eval(session=self.sess)
# print(np.mean(control_weight), np.std(control_weight))
feed = {
self.inputs: batch_images,
self.z: batch_z,
self.view_in: batch_view,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr
}
# Update D network
_, summary_str = self.sess.run([d_optim, self.d_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Update G network
_, summary_str = self.sess.run([g_optim, self.g_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Run g_optim twice
_, summary_str = self.sess.run([g_optim, self.g_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
errD_fake = self.d_loss_fake.eval(feed)
errD_real = self.d_loss_real.eval(feed)
errG = self.g_loss.eval(feed)
errQ = self.q_loss.eval(feed)
if ADD_EMBEDDING:
errE = self.e_loss.eval(feed)
else:
errE = 0
counter += 1
print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f, q_loss: %.8f, e_loss: %.8f" \
% (epoch, idx, batch_idxs,
time.time() - start_time, errD_fake + errD_real, errG, errQ, errE))
if np.mod(counter, cfg.refresh_rate) == 1:
assert 'log' in str(LOGDIR)
shutil.rmtree(LOGDIR, ignore_errors=True)
shutil.copyfile(sys.argv[1],
os.path.join(LOGDIR, 'config.py'))
self.save(LOGDIR, counter)
# Use global vectors
feed_eval = {
self.inputs: sample_images,
self.z: SAMPLE_Z,
self.view_in: SAMPLE_VIEW,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr
}
samples, d_loss, g_loss = self.sess.run(
[self.G, self.d_loss, self.g_loss],
feed_dict=feed_eval)
ren_img = inverse_transform(samples)
ren_img = np.clip(255 * ren_img, 0, 255).astype(np.uint8)
grid_x = GRID_X
if ADD_EMBEDDING:
ren_img = np.concatenate([self.images, ren_img],
axis=0)
grid_x += 1
# try:
Image.fromarray(
merge(ren_img,
[grid_x, GRID_Y]).astype(np.uint8)).save(
os.path.join(OUTPUT_DIR,
"{0}_GAN.png".format(counter)))
print("[Sample] d_loss: %.8f, g_loss: %.8f" %
(d_loss, g_loss))
np.concatenate([
generate_random_rotation_translation(1,
cfg.ele_low,
cfg.ele_high,
angle,
angle + 1,
cfg.scale_low,
cfg.scale_high,
cfg.x_low,
cfg.x_high,
cfg.y_low,
cfg.y_high,
cfg.z_low,
cfg.z_high,
with_translation=False,
with_scale=to_bool(
str(cfg.with_translation)))
for angle in np.linspace(cfg.azi_low, cfg.azi_high, GRID_X)
]), [GRID_Y, 1])
SAMPLE_VIEW = SAMPLE_VIEW.reshape((GRID_Y, GRID_X, -1)).transpose(
(1, 0, 2)).reshape((GRID_X * GRID_Y, -1))
assert SAMPLE_VIEW.shape == (GRID_X * GRID_Y, 6)
ADD_EMBEDDING = True
if ADD_EMBEDDING:
cfg.z_dim += cfg.emb_dim
class HoloGAN(object):
def __init__(self,
sess,
emb_graph,
def sample_HoloGAN(self, config):
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
count = 0
could_load, checkpoint_counter = self.load()
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
return
if not os.path.exists(self.sample_dir):
os.makedirs(self.sample_dir)
if config.rotate_azimuth:
low = cfg['azi_low']
high = cfg['azi_high']
step = STEP
elif config.rotate_elevation:
low = cfg['ele_low']
high = cfg['ele_high']
step = STEP
else:
low = 0
high = 10
step = 1
for i in range(low, high, step):
if config.rotate_azimuth:
sample_view = np.tile(
np.array([
i * math.pi / 180.0, 0 * math.pi / 180.0, 1.0, 0, 0, 0
]), (cfg['batch_size'], 1))
#print(sample_view)
elif config.rotate_elevation:
sample_view = np.tile(
np.array([
270 * math.pi / 180.0, (90 - i) * math.pi / 180.0, 1.0,
0, 0, 0
]), (cfg['batch_size'], 1))
else:
sample_view = self.gen_view_func(
cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=to_bool(str(cfg['with_translation'])),
with_scale=to_bool(str(cfg['with_scale'])))
feed_eval = {self.z: sample_z, self.view_in: sample_view}
samples = self.sess.run(self.G, feed_dict=feed_eval)
ren_img = np.clip(255 * samples, 0, 255).astype(np.uint8)
"""
cv2.imshow('image',ren_img[0])
cv2.waitKey(0)
raise Exception('å')
"""
if count < 10:
count_str = 'a' + str(count)
elif count > 99:
count_str = 'c' + str(count)
else:
count_str = 'b' + str(count)
try:
cv2.imwrite(
os.path.join(self.sample_dir, "{0}.jpg".format(count_str)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
except:
cv2.imwrite(
os.path.join(self.sample_dir, "{0}.jpg".format(count_str)),
ren_img[0])
count = count + 1
def train_HoloGAN(self, config):
self.d_lr_in = tf.compat.v1.placeholder(tf.float32, None, name='d_eta')
self.g_lr_in = tf.compat.v1.placeholder(tf.float32, None, name='d_eta')
d_optim = tf.train.AdamOptimizer(cfg['d_eta'],
beta1=cfg['beta1'],
beta2=cfg['beta2']).minimize(
self.d_loss, var_list=self.d_vars)
g_optim = tf.train.AdamOptimizer(cfg['g_eta'],
beta1=cfg['beta1'],
beta2=cfg['beta2']).minimize(
self.g_loss, var_list=self.g_vars)
tf.global_variables_initializer().run()
shutil.copyfile(sys.argv[1], os.path.join(self.log_dir, 'config.json'))
self.g_sum = merge_summary([self.d_loss_fake_sum, self.g_loss_sum])
self.d_sum = merge_summary([self.d_loss_real_sum, self.d_loss_sum])
self.writer = SummaryWriter(self.log_dir, self.sess.graph)
# Sample noise Z and view parameters to test during training
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
sample_view = self.gen_view_func(cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=False,
with_scale=to_bool(
str(cfg['with_translation'])))
sample_files = self.data[0:cfg['batch_size']]
sample_images = [
get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False) for sample_file in sample_files
]
counter = 1
start_time = time.time()
could_load, checkpoint_counter = self.load()
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
d_lr = cfg['d_eta']
g_lr = cfg['g_eta']
for epoch in range(cfg['max_epochs']):
d_lr = d_lr if epoch < cfg['epoch_step'] else d_lr * (
cfg['max_epochs'] - epoch) / (cfg['max_epochs'] -
cfg['epoch_step'])
g_lr = g_lr if epoch < cfg['epoch_step'] else g_lr * (
cfg['max_epochs'] - epoch) / (cfg['max_epochs'] -
cfg['epoch_step'])
random.shuffle(self.data)
batch_idxs = min(len(self.data),
config.train_size) // cfg['batch_size']
batch_idxs = int(batch_idxs)
for idx in range(0, batch_idxs):
batch_files = self.data[idx * cfg['batch_size']:(idx + 1) *
cfg['batch_size']]
batch_images = [
get_image(batch_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=self.crop) for batch_file in batch_files
]
"""
img = np.clip(255 * batch_images[0], 0, 255).astype(np.uint8)
cv2.imshow('image',img)
cv2.waitKey(0)
raise Exception('å')
"""
batch_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
batch_view = self.gen_view_func(
cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=False,
with_scale=to_bool(str(cfg['with_translation'])))
feed = {
self.inputs: batch_images,
self.z: batch_z,
self.view_in: batch_view,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr
}
# Update D network
_, summary_str = self.sess.run([d_optim, self.d_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Update G network
_, summary_str = self.sess.run([g_optim, self.g_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
# Run g_optim twice
_, summary_str = self.sess.run([g_optim, self.g_sum],
feed_dict=feed)
self.writer.add_summary(summary_str, counter)
errD_fake = self.d_loss_fake.eval(feed)
errD_real = self.d_loss_real.eval(feed)
errG = self.g_loss.eval(feed)
errQ = self.q_loss.eval(feed)
counter += 1
print(
"Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f, q_loss: %.8f"
% (epoch, idx, batch_idxs, time.time() - start_time,
errD_fake + errD_real, errG, errQ))
if np.mod(counter, SAVE_STEP) == 1:
self.save(counter)
feed_eval = {
self.inputs: sample_images,
self.z: sample_z,
self.view_in: sample_view,
self.d_lr_in: d_lr,
self.g_lr_in: g_lr
}
samples, d_loss, g_loss = self.sess.run(
[self.G, self.d_loss, self.g_loss],
feed_dict=feed_eval)
ren_img = np.clip(255 * samples, 0, 255).astype(np.uint8)
try:
cv2.imwrite(
os.path.join(self.img_dir,
"{0}_GAN.jpg".format(counter)),
merge(ren_img, [cfg['batch_size'] // 4, 4]))
print("[Sample] d_loss: %.8f, g_loss: %.8f" %
(d_loss, g_loss))
except:
cv2.imwrite(
os.path.join(self.img_dir,
"{0}_GAN.jpg".format(counter)),
ren_img[0])
print("[Sample] d_loss: %.8f, g_loss: %.8f" %
(d_loss, g_loss))
def train_z_map(self, config):
sample_z = self.sampling_Z(cfg['z_dim'], str(cfg['sample_z']))
sample_view = self.gen_view_func(cfg['batch_size'],
cfg['ele_low'],
cfg['ele_high'],
cfg['azi_low'],
cfg['azi_high'],
cfg['scale_low'],
cfg['scale_high'],
cfg['x_low'],
cfg['x_high'],
cfg['y_low'],
cfg['y_high'],
cfg['z_low'],
cfg['z_high'],
with_translation=False,
with_scale=to_bool(
str(cfg['with_translation'])))
sample_file = self.z_map_image[0]
sample_img = get_image(sample_file,
input_height=self.input_height,
input_width=self.input_width,
resize_height=self.output_height,
resize_width=self.output_width,
crop=False)
sample_img = sample_img.reshape(1, 64, 64, 3).astype('float32')
vars = tf.trainable_variables()
z_var = [var for var in vars if 'z_weight' in var.name]
#print('z var: ', self.sess.run(z_var))
#raise Exception('hej')
mae = tf.keras.losses.MeanAbsoluteError(reduction="sum")
target_image_difference = mae(sample_img[0], self.G)
regularizer = tf.abs(tf.norm(z_var) - np.sqrt(cfg['z_dim']))
regularizer = tf.cast(regularizer, dtype=tf.float32)
z_map_loss = target_image_difference + regularizer
z_lr = tf.compat.v1.placeholder(tf.float32, None, name='z_lr')
optimizer = tf.train.AdamOptimizer(learning_rate=z_lr,
name="z_map_optimizer").minimize(
z_map_loss, var_list=z_var)
tf.global_variables_initializer().run()
could_load, checkpoint_counter = self.load()
if could_load:
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
return
num_optimization_steps = 500
losses = []
print('START')
feed = {
self.view_in: sample_view,
self.z: sample_z,
self.inputs: sample_img
}
original_img = self.sess.run(self.G, feed_dict=feed)
original_img = np.clip(255 * original_img, 0, 255).astype(np.uint8)
losses = []
lr = 0.001
for step in range(num_optimization_steps):
feed_z_map = {
self.view_in: sample_view,
self.z: sample_z,
z_lr: lr
}
_, loss = self.sess.run([optimizer, z_map_loss],
feed_dict=feed_z_map)
print('loss: ', loss)
losses.append(loss)
if loss < 500:
lr = 0.0005
print()
feed = {self.view_in: sample_view, self.z: sample_z}
reconstructed_img = self.sess.run(self.G, feed_dict=feed)
reconstructed_img = np.clip(255 * reconstructed_img, 0,
255).astype(np.uint8)
sample_img = np.clip(255 * sample_img, 0, 255).astype(np.uint8)
start = original_img[0]
target = sample_img[0]
result = reconstructed_img[0]
image = np.concatenate([start, target, result], axis=1)
print(os.path.join(self.img_dir, "result.jpg"))
cv2.imwrite(os.path.join(self.sample_dir, "result.jpg"), image)
plt.plot(losses)
plt.xlabel('steps')
plt.ylabel('loss')
plt.ylim(ymin=0)
plt.xlim(xmin=0)
plt.title("Loss per iteration for 50 samples of cars")
#plt.show()
if str.lower(str(cfg["sample_from_z"])) == "true":
self.sample_from_z(config, sample_z)
