def create_queues(hypes, phase):
"""Create Queues."""
arch = hypes['arch']
dtypes = [tf.float32, tf.int32]
shape_known = hypes['jitter']['reseize_image'] \
or hypes['jitter']['crop_patch']
if shape_known:
if hypes['jitter']['crop_patch']:
height = hypes['jitter']['patch_height']
width = hypes['jitter']['patch_width']
else:
height = hypes['jitter']['image_height']
width = hypes['jitter']['image_width']
channel = hypes['arch']['num_channels']
num_classes = cutils.get_num_classes(hypes)
shapes = [[height, width, channel],
[height, width, num_classes]]
else:
shapes = None
capacity = 50
q = tf.FIFOQueue(capacity=50, dtypes=dtypes, shapes=shapes)
tf.summary.scalar("queue/%s/fraction_of_%d_full" %
(q.name + "_" + phase, capacity),
math_ops.cast(q.size(), tf.float32) * (1. / capacity))
return q
def eval_image(hypes, gt_image, cnn_image):
flat_gt_image = gt_image.flatten()
flat_cnn_image = cnn_image.flatten()
confusion_matrix = metrics.confusion_matrix(flat_gt_image, flat_cnn_image, range(cutils.get_num_classes(hypes)))
return confusion_matrix
def _add_softmax(hypes, logits):
num_classes = cutils.get_num_classes(hypes)
with tf.name_scope('decoder'):
logits = tf.reshape(logits, (-1, num_classes))
epsilon = tf.constant(value=hypes['solver']['epsilon'])
# logits = logits + epsilon
softmax = tf.nn.softmax(logits)
return softmax
def _compute_f1(hypes, labels, softmax, epsilon):
labels = tf.to_float(
tf.reshape(labels, (-1, cutils.get_num_classes(hypes))))[:, 1]
logits = softmax[:, 1]
true_positive = tf.reduce_sum(labels * logits)
false_positive = tf.reduce_sum((1 - labels) * logits)
recall = true_positive / tf.reduce_sum(labels)
precision = true_positive / (true_positive + false_positive + epsilon)
score = 2 * recall * precision / (precision + recall)
f1_score = 1 - 2 * recall * precision / (precision + recall)
return f1_score
def loss(hypes, decoded_logits, labels):
"""Calculate the loss from the logits and the labels.
Args:
logits: Logits tensor, float - [batch_size, NUM_CLASSES].
labels: Labels tensor, int32 - [batch_size].
Returns:
loss: Loss tensor of type float.
"""
logits = decoded_logits['logits']
with tf.name_scope('loss'):
num_classes = cutils.get_num_classes(hypes)
logits = tf.reshape(logits, (-1, num_classes))
shape = [logits.get_shape()[0], num_classes]
epsilon = tf.constant(value=hypes['solver']['epsilon'])
# logits = logits + epsilon
labels = tf.to_float(tf.reshape(labels, (-1, num_classes)))
softmax = tf.nn.softmax(logits) + epsilon
if hypes['loss'] == 'xentropy':
cross_entropy_mean = _compute_cross_entropy_mean(
hypes, labels, softmax)
elif hypes['loss'] == 'softF1':
cross_entropy_mean = _compute_f1(hypes, labels, softmax, epsilon)
elif hypes['loss'] == 'softIU':
cross_entropy_mean = _compute_soft_ui(hypes, labels, softmax,
epsilon)
reg_loss_col = tf.GraphKeys.REGULARIZATION_LOSSES
weight_loss = tf.add_n(tf.get_collection(reg_loss_col),
name='reg_loss')
total_loss = cross_entropy_mean + weight_loss
losses = {}
losses['total_loss'] = total_loss
losses['xentropy'] = cross_entropy_mean
losses['weight_loss'] = weight_loss
return losses
def inference(hypes, images, train=True):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
train: whether the network is used for train of inference
Returns:
softmax_linear: Output tensor with the computed logits.
"""
vgg16_npy_path = os.path.join(hypes['dirs']['data_dir'], 'weights',
"vgg16.npy")
vgg_fcn = fcn8_vgg.FCN8VGG(vgg16_npy_path=vgg16_npy_path)
vgg_fcn.wd = hypes['wd']
vgg_fcn.build(images,
train=train,
num_classes=cutils.get_num_classes(hypes),
random_init_fc8=True)
logits = {}
logits['images'] = images
if hypes['arch']['fcn_in'] == 'pool5':
logits['fcn_in'] = vgg_fcn.pool5
elif hypes['arch']['fcn_in'] == 'fc7':
logits['fcn_in'] = vgg_fcn.fc7
else:
raise NotImplementedError
logits['feed2'] = vgg_fcn.pool4
logits['feed4'] = vgg_fcn.pool3
logits['fcn_logits'] = vgg_fcn.upscore32
logits['deep_feat'] = vgg_fcn.pool5
logits['early_feat'] = vgg_fcn.conv4_3
return logits
def inputs(hypes, q, phase):
"""Generate Inputs images."""
if phase == 'val':
image, label = q.dequeue()
image = tf.expand_dims(image, 0)
label = tf.expand_dims(label, 0)
return image, label
shape_known = hypes['jitter']['reseize_image'] \
or hypes['jitter']['crop_patch']
if not shape_known:
image, label = q.dequeue()
num_classes = cutils.get_num_classes(hypes)
label.set_shape([None, None, num_classes])
image.set_shape([None, None, 3])
image = tf.expand_dims(image, 0)
label = tf.expand_dims(label, 0)
if hypes['solver']['batch_size'] > 1:
logging.error("Using a batch_size of {} with unknown shape."
.format(hypes['solver']['batch_size']))
logging.error("Set batch_size to 1 or use `reseize_image` "
"or `crop_patch` to obtain a defined shape")
raise ValueError
else:
image, label = q.dequeue_many(hypes['solver']['batch_size'])
image = _processe_image(hypes, image)
# Display the training images in the visualizer.
tensor_name = image.op.name
tf.summary.image(tensor_name + '/image', image)
road = tf.expand_dims(tf.to_float(label[:, :, :, 0]), 3)
tf.summary.image(tensor_name + '/gt_image', road)
return image, label
def create_test_output(hypes, sess, image_pl, softmax, data_file):
# data_dir = hypes['dirs']['data_dir']
# data_file = os.path.join(data_dir, test_file)
image_dir = os.path.dirname(data_file)
logdir_prediction = "test_images_prediction/"
logging.info(
"Images will be written to {}/test_images_{{prediction, rg}}".format(
logdir_prediction))
logdir_prediction = os.path.join(hypes['dirs']['output_dir'],
logdir_prediction)
if not os.path.exists(logdir_prediction):
os.mkdir(logdir_prediction)
num_classes = cutils.get_num_classes(hypes)
color_dict = cutils.get_output_color_dict(hypes)
total_confusion_matrix = np.zeros([num_classes, num_classes], int)
name_classes = cutils.get_name_classes(hypes)
with open(data_file) as file:
for i, datum in enumerate(file):
t = time.time()
image_file = datum.rstrip()
if len(image_file.split(" ")) > 1:
image_file, gt_file = image_file.split(" ")
gt_file = os.path.join(image_dir, gt_file)
gt_image = scp.misc.imread(gt_file, mode='RGB')
image_file = os.path.join(image_dir, image_file)
image = scp.misc.imread(image_file)
shape = image.shape
feed_dict = {image_pl: image}
output = sess.run([softmax['softmax']], feed_dict=feed_dict)
output_im = output[0].argmax(axis=1).reshape(shape[0], shape[1])
# Saving RB Plot
name = os.path.basename(image_file)
new_name = name.split('.')[0] + '_prediction.png'
prediction_image = utils.overlay_segmentation(
image, output_im, color_dict)
save_file = os.path.join(logdir_prediction, new_name)
scp.misc.imsave(save_file, prediction_image)
elapsed = time.time() - t
print("elapsed time: " + str(elapsed))
if 'gt_image' in locals():
confusion_matrix = kitti_eval.eval_image(
hypes, cutils.get_gt_image_index(gt_image, hypes),
output_im)
total_confusion_matrix += confusion_matrix
for j in range(num_classes):
gray_scale_file_name = name.split(
'.')[0] + '_' + name_classes[j] + '_grayscale.png'
save_file = os.path.join(logdir_prediction,
gray_scale_file_name)
output_prob_class = np.around(
output[0][:, j].reshape(shape[0], shape[1]) * 255)
scp.misc.imsave(save_file, output_prob_class)
if 'gt_image' in locals():
normalized_total_confusion_matrix = total_confusion_matrix.astype(
'float') / total_confusion_matrix.sum(axis=1)[:, np.newaxis]
normalized_total_confusion_matrix[np.isnan(
normalized_total_confusion_matrix)] = 0
classes_result = {
"confusion_matrix": total_confusion_matrix,
"normalized_confusion_matrix":
normalized_total_confusion_matrix
}
for i in range(num_classes):
classes_result[
name_classes[i]] = kitti_eval.obtain_class_result(
total_confusion_matrix, i)
eval_result = {"classes_result": classes_result}
print(eval_result)
def evaluate(hypes, sess, image_pl, inf_out):
softmax = inf_out['softmax']
data_dir = hypes['dirs']['data_dir']
color_dict = cutils.get_output_color_dict(hypes)
num_classes = cutils.get_num_classes(hypes)
eval_dict = {}
for phase in ['train', 'val']:
total_confusion_matrix = np.zeros([num_classes, num_classes], int)
data_file = hypes['data']['{}_file'.format(phase)]
data_file = os.path.join(data_dir, data_file)
image_dir = os.path.dirname(data_file)
image_list = []
with open(data_file) as file:
for i, datum in enumerate(file):
datum = datum.rstrip()
image_file, gt_file = datum.split(" ")
image_file = os.path.join(image_dir, image_file)
gt_file = os.path.join(image_dir, gt_file)
image = scp.misc.imread(image_file, mode='RGB')
gt_image = scp.misc.imread(gt_file, mode='RGB')
if hypes['jitter']['fix_shape']:
shape = image.shape
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
assert(image_height >= shape[0])
assert(image_width >= shape[1])
offset_x = (image_height - shape[0])//2
offset_y = (image_width - shape[1])//2
new_image = np.zeros([image_height, image_width, 3])
new_image[offset_x:offset_x+shape[0],
offset_y:offset_y+shape[1]] = image
input_image = new_image
elif hypes['jitter']['reseize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image, gt_image = resize_label_image(image, gt_image,
image_height,
image_width)
input_image = image
else:
input_image = image
shape = input_image.shape
feed_dict = {image_pl: input_image}
output = sess.run([softmax], feed_dict=feed_dict)
output_im = output[0].argmax(axis=1).reshape(shape[0], shape[1])
output_prob = output[0].max(axis=1).reshape(shape[0], shape[1])
if hypes['jitter']['fix_shape']:
gt_shape = gt_image.shape
output_im = output_im[offset_x:offset_x+gt_shape[0],
offset_y:offset_y+gt_shape[1]]
output_prob = output_prob[offset_x:offset_x + gt_shape[0],
offset_y:offset_y + gt_shape[1]]
if phase == 'val':
# Saving RB Plot
ov_image = seg.make_overlay(image, output_prob)
name = os.path.basename(image_file)
image_list.append((name, ov_image))
name2 = name.split('.')[0] + '_prediction.png'
prediction_image = utils.overlay_segmentation(image, output_im, color_dict)
image_list.append((name2, prediction_image))
confusion_matrix = eval_image(hypes, cutils.get_gt_image_index(gt_image, hypes), output_im)
total_confusion_matrix += confusion_matrix
classes_result = {"confusion_matrix": total_confusion_matrix,
"normalized_confusion_matrix": normalize_confusion_matrix(total_confusion_matrix)}
name_classes = cutils.get_name_classes(hypes)
for i in range(num_classes):
classes_result[name_classes[i]] = obtain_class_result(total_confusion_matrix, i)
eval_dict[phase] = {"classes_result": classes_result}
if phase == 'val':
start_time = time.time()
for i in xrange(10):
sess.run([softmax], feed_dict=feed_dict)
dt = (time.time() - start_time)/10
eval_list = []
for phase in ['train', 'val']:
print(phase)
print(eval_dict[phase]["classes_result"]["confusion_matrix"])
print(eval_dict[phase]["classes_result"]["normalized_confusion_matrix"])
for class_name in name_classes:
eval_list.append(('[{} {}] Recall'.format(phase, class_name), 100 * eval_dict[phase]["classes_result"][class_name]["recall"]))
eval_list.append(('[{} {}] Precision'.format(phase, class_name), 100 * eval_dict[phase]["classes_result"][class_name]["precision"]))
eval_list.append(('[{} {}] TNR'.format(phase, class_name), 100 * eval_dict[phase]["classes_result"][class_name]["TNR"]))
eval_list.append(('[{} {}] Accuracy'.format(phase, class_name), 100 * eval_dict[phase]["classes_result"][class_name]["accuracy"]))
eval_list.append(('[{} {}] F1'.format(phase, class_name), 100 * eval_dict[phase]["classes_result"][class_name]["F1"]))
eval_list.append(('Speed (msec)', 1000*dt))
eval_list.append(('Speed (fps)', 1/dt))
return eval_list, image_list