def test(self, run):
with tf.Session() as sess:
self.model.restore(sess, "ckpts/%s" % run)
sess.run(self.iter_init_op)
set_comparison = u.SetComparison()
num_imgs = 0
xent_losses = []
debug_img = None # created on first call
while True:
try:
if debug_img is None:
# fetch imgs as well to create debug_img
imgs, true_bitmaps, predicted_bitmaps, xent_loss = sess.run(
[
self.test_imgs, self.test_xys_bitmaps,
self.model.output, self.model.xent_loss
])
# choose a random element from batch
idx = random.randint(0, true_bitmaps.shape[0] - 1)
debug_img = u.debug_img(imgs[idx], true_bitmaps[idx],
predicted_bitmaps[idx])
else:
true_bitmaps, predicted_bitmaps, xent_loss = sess.run([
self.test_xys_bitmaps, self.model.output,
self.model.xent_loss
])
xent_losses.append(xent_loss)
iterator_batch_size = true_bitmaps.shape[0]
num_imgs += iterator_batch_size
for idx in range(iterator_batch_size):
# this is dumb; should do against label db!
true_centroids = u.centroids_of_connected_components(
true_bitmaps[idx])
predicted_centroids = u.centroids_of_connected_components(
predicted_bitmaps[idx])
tp, fn, fp = set_comparison.compare_sets(
true_centroids, predicted_centroids)
except tf.errors.OutOfRangeError:
# end of iterator
break
precision, recall, f1 = set_comparison.precision_recall_f1()
return {
"num_imgs": num_imgs,
"debug_img": debug_img, # for tensorboard
"precision": precision,
"recall": recall,
"f1": f1,
"xent": np.mean(xent_losses)
}
def pr_stats(run, image_dir, label_db, connected_components_threshold):
# TODO: a bunch of this can go back into one off init in a class
_train_opts, model = m.restore_model(run)
label_db = LabelDB(label_db_file=label_db)
set_comparison = u.SetComparison()
# use 4 images for debug
debug_imgs = []
for idx, filename in enumerate(sorted(os.listdir(image_dir))):
# load next image
# TODO: this block used in various places, refactor
img = np.array(Image.open(image_dir + "/" +
filename)) # uint8 0->255 (H, W)
img = img.astype(np.float32)
img = (img / 127.5) - 1.0 # -1.0 -> 1.0 # see data.py
# run through model
prediction = expit(model.predict(np.expand_dims(img, 0))[0])
if len(debug_imgs) < 4:
debug_imgs.append(u.side_by_side(rgb=img, bitmap=prediction))
# calc [(x,y), ...] centroids
predicted_centroids = u.centroids_of_connected_components(
prediction, rescale=2.0, threshold=connected_components_threshold)
# compare to true labels
true_centroids = label_db.get_labels(filename)
true_centroids = [(y, x) for (x, y) in true_centroids] # sigh...
tp, fn, fp = set_comparison.compare_sets(true_centroids,
predicted_centroids)
precision, recall, f1 = set_comparison.precision_recall_f1()
return {
"debug_imgs": debug_imgs,
"precision": precision,
"recall": recall,
"f1": f1
}
def dump_images(prefix):
# run from imgs -> bitmap and stitch them together...
img_collage = Image.new('RGB', (17 * 8, 17 * 8), (0, 0, 0))
bitmap_collage = Image.new('RGB', (9 * 8, 9 * 8), (255, 255, 255))
centroids_collage = Image.new('RGB', (9 * 8, 9 * 8), (255, 255, 255))
ims, bs = sess.run([imgs, model.output])
for x in range(8):
for y in range(8):
i = (x * 8) + y
img_collage.paste(u.zero_centered_array_to_pil_image(ims[i]),
(17 * x, 17 * y))
output_bitmap = u.bitmap_to_pil_image(bs[i])
bitmap_collage.paste(output_bitmap, (9 * x, 9 * y))
centroids = u.centroids_of_connected_components(bs[i])
centroid_bitmap = u.bitmap_from_centroids(centroids, h=8, w=8)
centroid_bitmap = u.bitmap_to_single_channel_pil_image(
centroid_bitmap)
centroids_collage.paste(centroid_bitmap, (9 * x, 9 * y))
img_collage.save("images/ra/%s_imgs.png" % prefix)
bitmap_collage.save("images/ra/%s_bitmaps.png" % prefix)
centroids_collage.save("images/ra/%s_centroids.png" % prefix)
imgs = random.sample(imgs, opts.num)
for idx, filename in enumerate(sorted(imgs)):
# load next image (and add dummy batch dimension)
img = np.array(Image.open(opts.image_dir+"/"+filename)) # uint8 0->255
img = img.astype(np.float32)
img = (img / 127.5) - 1.0 # -1.0 -> 1.0 # see data.py
try:
# run single image through model
prediction = sess.run(model.output, feed_dict={model.imgs: [img]})[0]
# calc [(x,y), ...] centroids
centroids = u.centroids_of_connected_components(prediction,
rescale=2.0,
threshold=opts.connected_components_threshold)
print("\t".join(map(str, [idx, filename, len(centroids)])))
# export some debug image (if requested)
if opts.export_pngs != '':
if opts.export_pngs == 'predictions':
debug_img = u.side_by_side(rgb=img, bitmap=prediction)
elif opts.export_pngs == 'centroids':
debug_img = u.red_dots(rgb=img, centroids=centroids)
else:
raise Exception("unknown --export-pngs option")
debug_img.save("%s/%s.png" % (export_dir, filename))
# set new labels (if requested)
])
train_summaries_writer.add_summary(u.explicit_summaries({"xent": xl}),
step)
debug_img_summary = u.pil_image_to_tf_summary(u.debug_img(i, bm, o))
train_summaries_writer.add_summary(debug_img_summary, step)
train_summaries_writer.flush()
# ... test
i, bm, o, xl, step = sess.run([
test_imgs, test_xys_bitmaps, test_model.output, test_model.xent_loss,
global_step
])
set_comparison = u.SetComparison()
for idx in range(bm.shape[0]):
true_centroids = u.centroids_of_connected_components(bm[idx])
predicted_centroids = u.centroids_of_connected_components(o[idx])
set_comparison.compare_sets(true_centroids, predicted_centroids)
precision, recall, f1 = set_comparison.precision_recall_f1()
tag_values = {
"xent": xl,
"precision": precision,
"recall": recall,
"f1": f1
}
test_summaries_writer.add_summary(u.explicit_summaries(tag_values), step)
debug_img_summary = u.pil_image_to_tf_summary(u.debug_img(i, bm, o))
test_summaries_writer.add_summary(debug_img_summary, step)
test_summaries_writer.flush()
# load next image (and add dummy batch dimension)
img = np.array(Image.open(opts.image_dir + "/" + filename)) # unit8 0->255
img = img.astype(np.float32)
img = (img / 127.5) - 1.0 # -1.0 -> 1.0 # see data.py
try:
# run single image through model
s = time.time()
prediction = sess.run(model_output,
feed_dict={imgs_placeholder: [img]})[0]
prediction_time = time.time() - s
prediction_times.append(prediction_time)
# calc [(x,y), ...] centroids
s = time.time()
centroids = u.centroids_of_connected_components(prediction,
rescale=2.0)
centroid_calc_time = time.time() - s
centroid_calc_times.append(centroid_calc_time)
print("\t".join(
map(str, [
idx, filename,
len(centroids), prediction_time, centroid_calc_time
])))
# export some debug image (if requested)
if opts.export_pngs != '':
if opts.export_pngs == 'predictions':
debug_img = u.side_by_side(rgb=img, bitmap=prediction)
elif opts.export_pngs == 'centroids':
debug_img = u.red_dots(rgb=img, centroids=centroids)
for idx, filename in enumerate(sorted(imgs)):
# load next image
img = np.array(Image.open(opts.image_dir + "/" +
filename)) # uint8 0->255 (H, W)
img = img.astype(np.float32)
img = (img / 127.5) - 1.0 # -1.0 -> 1.0 # see data.py
# run through model (adding / removing dummy batch)
# recall: output from model is logits so we need to expit
# TODO: do this in batch !!
prediction = expit(model.predict(np.expand_dims(img, 0))[0])
# calc [(x,y), ...] centroids
centroids = u.centroids_of_connected_components(
prediction,
rescale=2.0,
threshold=train_opts['connected_components_threshold'])
print("\t".join(map(str, [idx, filename, len(centroids)])))
# export some debug image (if requested)
if opts.export_pngs != '':
if opts.export_pngs == 'predictions':
debug_img = u.side_by_side(rgb=img, bitmap=prediction)
elif opts.export_pngs == 'centroids':
debug_img = u.red_dots(rgb=img, centroids=centroids)
else:
raise Exception("unknown --export-pngs option")
debug_img.save("%s/%s.png" % (export_dir, filename))
# set new labels (if requested)
if db: