def prepare_img_for_captioning(img_batch, bottom, rows=None):
# Show at most 8 images per row
batch_size = img_batch.shape[0]
n = min(8, batch_size)
img_shape = img_batch[0].shape
if rows is None:
rows = batch_size // n
# Leave top row empty for the caption text
text_row = np.tile(np.ones(img_shape) * 255, reps=(1, n, 1))
top_row = text_row
# Fill in the super image row by row
img_batch = denormalize_images(img_batch)
super_img = top_row
for rown in range(rows):
if batch_size > rown * n:
row = []
for i in range(rown * n, (rown + 1) * n):
row.append(img_batch[i])
row = np.concatenate(row, axis=1)
super_img = np.concatenate([super_img, row], axis=0)
if bottom:
super_img = np.concatenate([super_img, text_row], axis=0)
super_img = super_img.astype(np.uint8)
return super_img
def evaluate_inception(self):
incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE
logits, _ = load_inception_inference(
self.sess, self.classes, incep_batch_size,
self.cfg.EVAL.INCEP_CHECKPOINT_DIR)
pred_op = tf.nn.softmax(logits)
z = tf.placeholder(tf.float32, [self.bs, self.model.z_dim], name='z')
cond = tf.placeholder(tf.float32, [self.bs] + [self.model.embed_dim],
name='cond')
eval_gen, _, _ = self.model.generator(z,
cond,
reuse=False,
is_training=False)
saver = tf.train.Saver(tf.global_variables('g_net'))
could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR)
if could_load:
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
raise RuntimeError(
'Could not load the checkpoints of the generator')
print('Generating x...')
size = self.cfg.EVAL.SIZE
n_batches = size // self.bs
w, h, c = self.model.image_dims[0], self.model.image_dims[
1], self.model.image_dims[2]
samples = np.zeros((n_batches * self.bs, w, h, c))
for i in range(n_batches):
print("\rGenerating batch %d/%d" % (i + 1, n_batches),
end="",
flush=True)
sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim))
_, _, embed, _, _ = self.dataset.test.next_batch(self.bs,
4,
embeddings=True)
start = i * self.bs
end = start + self.bs
gen_batch = self.sess.run(eval_gen,
feed_dict={
z: sample_z,
cond: embed
})
samples[start:end] = denormalize_images(gen_batch)
print('\nComputing inception score...')
mean, std = inception_score.get_inception_score(samples,
self.sess,
incep_batch_size,
10,
pred_op,
verbose=True)
print('Inception Score | mean:', "%.2f" % mean, 'std:', "%.2f" % std)
def evaluate_fid(self):
incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE
_, layers = load_inception_inference(self.sess, 20, incep_batch_size,
self.cfg.EVAL.INCEP_CHECKPOINT_DIR)
pool3 = layers['PreLogits']
act_op = tf.reshape(pool3, shape=[incep_batch_size, -1])
if not os.path.exists(self.cfg.EVAL.ACT_STAT_PATH):
print('Computing activation statistics for real x')
fid.compute_and_save_activation_statistics(self.cfg.EVAL.R_IMG_PATH, self.sess, incep_batch_size, act_op,
self.cfg.EVAL.ACT_STAT_PATH, verbose=True)
print('Loading activation statistics for the real x')
stats = np.load(self.cfg.EVAL.ACT_STAT_PATH)
mu_real = stats['mu']
sigma_real = stats['sigma']
z = tf.placeholder(tf.float32, [self.bs, self.model.z_dim], name='real_images')
cond = tf.placeholder(tf.float32, [self.bs] + [self.model.embed_dim], name='cond')
eval_gen, _, _ = self.model.generator(z, cond, reuse=False)
saver = tf.train.Saver(tf.global_variables('g_net'))
could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR)
if could_load:
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
raise RuntimeError('Could not load the checkpoints of the generator')
print('Generating x...')
fid_size = self.cfg.EVAL.SIZE
n_batches = fid_size // self.bs
w, h, c = self.model.image_dims[0], self.model.image_dims[1], self.model.image_dims[2]
samples = np.zeros((n_batches * self.bs, w, h, c))
for i in range(n_batches):
start = i * self.bs
end = start + self.bs
sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim))
images, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 4, embeddings=True)
samples[start: end] = denormalize_images(self.sess.run(eval_gen, feed_dict={z: sample_z, cond: embed}))
print('Computing activation statistics for generated x...')
mu_gen, sigma_gen = fid.calculate_activation_statistics(samples, self.sess, incep_batch_size, act_op,
verbose=True)
print("calculate FID:", end=" ", flush=True)
try:
FID = fid.calculate_frechet_distance(mu_gen, sigma_gen, mu_real, sigma_real)
except Exception as e:
print(e)
FID = 500
print(FID)
def evaluate_inception(self):
incep_batch_size = self.cfg.EVAL.INCEP_BATCH_SIZE
logits, _ = load_inception_inference(self.sess, self.cfg.EVAL.NUM_CLASSES, incep_batch_size,
self.cfg.EVAL.INCEP_CHECKPOINT_DIR)
pred_op = tf.nn.softmax(logits)
z = tf.placeholder(tf.float32, [self.bs, self.model.stagei.z_dim], name='z')
cond = tf.placeholder(tf.float32, [self.bs] + [self.model.stagei.embed_dim], name='cond')
stagei_gen, _, _ = self.model.stagei.generator(z, cond, reuse=False, is_training=False)
eval_gen, _, _ = self.model.generator(stagei_gen, cond, reuse=False, is_training=False)
self.Retrieval.eval(self.bs)
saver = tf.train.Saver(tf.global_variables('g_net')+tf.global_variables('vf_')+tf.global_variables('sf_')+
tf.global_variables('att'))
could_load, _ = load(saver, self.sess, self.model.stagei.cfg.CHECKPOINT_DIR)
if could_load:
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
raise RuntimeError('Could not load the checkpoints of stage I')
saver = tf.train.Saver(tf.global_variables('stageII_g_net'))
could_load, _ = load(saver, self.sess, self.cfg.CHECKPOINT_DIR)
if could_load:
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
raise RuntimeError('Could not load the checkpoints of stage II')
print('Generating batches...')
size = self.cfg.EVAL.SIZE
n_batches = size // self.bs
all_preds = []
for i in range(n_batches):
print("\rGenerating batch %d/%d" % (i + 1, n_batches), end="", flush=True)
sample_z = np.random.normal(0, 1, size=(self.bs, self.model.z_dim))
# _, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 4, embeddings=True)
_, _, embed, _, _ = self.dataset.test.next_batch(self.bs, 1, embeddings=True)
im_feats, sent_feats, labels = self.test_data_loader.get_batch(i, self.bs, phase = 'incep')
# Generate a batch and scale it up for inception
sent_emb = self.sess.run(self.Retrieval.sent_embed_tensor,
feed_dict={
self.Retrieval.image_placeholder_test: im_feats,
self.Retrieval.sent_placeholder_test: sent_feats,
})
gen_batch = self.sess.run(eval_gen, feed_dict={z: sample_z, cond: sent_emb})
samples = denormalize_images(gen_batch)
incep_samples = np.empty((self.bs, 299, 299, 3))
for sample_idx in range(self.bs):
incep_samples[sample_idx] = prep_incep_img(samples[sample_idx])
# Run prediction for current batch
pred = self.sess.run(pred_op, feed_dict={'inputs:0': incep_samples})
all_preds.append(pred)
# Get rid of the first dimension
all_preds = np.concatenate(all_preds, 0)
print('\nComputing inception score...')
mean, std = inception_score.get_inception_from_predictions(all_preds, 10)
print('Inception Score | mean:', "%.2f" % mean, 'std:', "%.2f" % std)