def run_detection(img):
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
print("starting KittiSeg inference")
print("la1")
image_pl = tf.placeholder(tf.float32)
print("la21")
image = tf.expand_dims(image_pl, 0)
print("la")
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules, image=image)
print("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
print("Weights loaded successfully.")
print("Starting inference")
# Load and resize input image
image = img
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
logging.info("--> Done with detection")
return green_image
def inference(image):
# Load and resize input image
# image = scp.misc.imread(input_image)
# if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
# print(image_height, image_width)
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
# rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
return green_image
def run_eval(load_out, output_folder, data_file):
meta_hypes, subhypes, submodules, decoded_logits, sess, image_pl = load_out
assert(len(meta_hypes['model_list']) == 3)
# inf_out['pred_boxes_new'], inf_out['pred_confidences']
seg_softmax = decoded_logits['segmentation']['softmax']
pred_boxes_new = decoded_logits['detection']['pred_boxes_new']
pred_confidences = decoded_logits['detection']['pred_confidences']
road_softmax = decoded_logits['road']['softmax'][0]
eval_list = [seg_softmax, pred_boxes_new, pred_confidences, road_softmax]
def my_process(image):
return process_image(subhypes, image)
if FLAGS.speed_test:
eval_runtime(sess, subhypes, image_pl, eval_list, data_file)
exit(0)
test_constant_input(subhypes)
test_segmentation_input(subhypes)
import utils.train_utils as dec_utils
gen = _output_generator(sess, eval_list, image_pl, data_file, my_process)
for image_file, output in gen:
image = scp.misc.imread(image_file)
image = process_image(subhypes, image)
shape = image.shape
seg_softmax, pred_boxes_new, pred_confidences, road_softmax = output
# Create Segmentation Overlay
shape = image.shape
seg_softmax = seg_softmax[:, 1].reshape(shape[0], shape[1])
hard = seg_softmax > 0.5
overlay_image = utils.fast_overlay(image, hard)
# Draw Detection Boxes
new_img, rects = dec_utils.add_rectangles(
subhypes['detection'], [overlay_image], pred_confidences,
pred_boxes_new, show_removed=False,
use_stitching=True, rnn_len=subhypes['detection']['rnn_len'],
min_conf=0.50, tau=subhypes['detection']['tau'])
# Draw road classification
highway = (np.argmax(output[0][0]) == 0)
new_img = road_draw(new_img, highway)
# Save image file
im_name = os.path.basename(image_file)
new_im_file = os.path.join(output_folder, im_name)
im_name = os.path.basename(image_file)
new_im_file = os.path.join(output_folder, im_name)
scp.misc.imsave(new_im_file, new_img)
logging.info("Plotting file: {}".format(new_im_file))
eval_runtime(sess, subhypes, image_pl, eval_list, data_file)
exit(0)
def create_test_output(hypes, sess, image_pl, softmax):
data_dir = hypes['dirs']['data_dir']
data_file = os.path.join(data_dir, test_file)
image_dir = os.path.dirname(data_file)
logdir = "test_images/"
logdir_rb = "test_images_rb/"
logdir_green = "test_images_green/"
logging.info(
"Images will be written to {}/test_images_{{green, rg}}".format(
logdir))
logdir = os.path.join(hypes['dirs']['output_dir'], logdir)
logdir_rb = os.path.join(hypes['dirs']['output_dir'], logdir_rb)
logdir_green = os.path.join(hypes['dirs']['output_dir'], logdir_green)
if not os.path.exists(logdir):
os.mkdir(logdir)
if not os.path.exists(logdir_rb):
os.mkdir(logdir_rb)
if not os.path.exists(logdir_green):
os.mkdir(logdir_green)
image_list = []
with open(data_file) as file:
for i, image_file in enumerate(file):
image_file = image_file.rstrip()
image_file = os.path.join(image_dir, image_file)
image = scp.misc.imread(image_file, mode='RGB')
shape = image.shape
print(shape)
feed_dict = {image_pl: image}
output = sess.run([softmax['softmax']], feed_dict=feed_dict)
output_im = output[0][:, 1].reshape(shape[0], shape[1])
ov_image = seg.make_overlay(image, output_im)
hard = output_im > 0.5
green_image = utils.fast_overlay(image, hard)
name = os.path.basename(image_file)
# new_name = name.split('_')[0] + "_crack_" + name.split('_')[1]
new_name = 'crack_' + name
save_file = os.path.join(logdir, new_name)
logging.info("Writing file: %s", save_file)
scp.misc.imsave(save_file, output_im)
save_file = os.path.join(logdir_rb, new_name)
scp.misc.imsave(save_file, ov_image)
save_file = os.path.join(logdir_green, new_name)
scp.misc.imsave(save_file, green_image)
def create_test_output(hypes, sess, image_pl, softmax):
data_dir = hypes['dirs']['data_dir']
data_file = os.path.join(data_dir, test_file)
image_dir = os.path.dirname(data_file)
logdir = "test_images/"
logdir_rb = "test_images_rb/"
logdir_green = "test_images_green/"
logging.info("Images will be written to {}/test_images_{{green, rg}}"
.format(logdir))
logdir = os.path.join(hypes['dirs']['output_dir'], logdir)
logdir_rb = os.path.join(hypes['dirs']['output_dir'], logdir_rb)
logdir_green = os.path.join(hypes['dirs']['output_dir'], logdir_green)
if not os.path.exists(logdir):
os.mkdir(logdir)
if not os.path.exists(logdir_rb):
os.mkdir(logdir_rb)
if not os.path.exists(logdir_green):
os.mkdir(logdir_green)
image_list = []
with open(data_file) as file:
for i, image_file in enumerate(file):
image_file = image_file.rstrip()
image_file = os.path.join(image_dir, image_file)
image = scp.misc.imread(image_file)
shape = image.shape
feed_dict = {image_pl: image}
output = sess.run([softmax['softmax']], feed_dict=feed_dict)
output_im = output[0][:, 1].reshape(shape[0], shape[1])
ov_image = seg.make_overlay(image, output_im)
hard = output_im > 0.5
green_image = utils.fast_overlay(image, hard)
name = os.path.basename(image_file)
new_name = name.split('_')[0] + "_road_" + name.split('_')[1]
save_file = os.path.join(logdir, new_name)
logging.info("Writing file: %s", save_file)
scp.misc.imsave(save_file, output_im)
save_file = os.path.join(logdir_rb, new_name)
scp.misc.imsave(save_file, ov_image)
save_file = os.path.join(logdir_green, new_name)
scp.misc.imsave(save_file, green_image)
def evaluate(hypes, sess, image_pl, inf_out):
hand_out_layer = inf_out['output']
local_out_layer = inf_out['output2']
recog_layer = inf_out['recog']
data_dir = hypes['dirs']['data_dir']
eval_dict = {}
background_color = np.array(hypes['data']['background_color'])
hand_color1 = np.array(hypes['data']['hand_color1'])
hand_color2 = np.array(hypes['data']['hand_color2'])
hand_color3 = np.array(hypes['data']['hand_color3'])
hand_color4 = np.array(hypes['data']['hand_color4'])
hand_color5 = np.array(hypes['data']['hand_color5'])
heatmap_color = np.array(hypes['data']['heatmap_color'])
# for phase in ['train', 'val']:
for phase in ['val']:
data_file = hypes['data']['{}_file'.format(phase)]
data_file = os.path.join(data_dir, data_file)
image_dir = os.path.dirname(data_file)
thresh = np.array(range(0, 256)) / 255.0
total_fp = np.zeros(thresh.shape)
total_fn = np.zeros(thresh.shape)
total_posnum = 0
total_negnum = 0
image_list = []
gt_class_list = []
pred_class_list = []
with open(data_file) as file:
for i, datum in enumerate(file):
datum = datum.rstrip()
image_file, local_gt_file = datum.split("\t")
if "BigObjects" in local_gt_file:
hand_gt_file = local_gt_file.replace(
"/BigObjects/", "/Masks/", 1)
else:
hand_gt_file = local_gt_file.replace(
"/Objects/", "/Masks/", 1)
if "GTEA" in hand_gt_file:
hand_gt_file = hand_gt_file.replace(".jpg", ".png", 1)
image_file = os.path.join(image_dir, image_file)
hand_gt_file = os.path.join(image_dir, hand_gt_file)
local_gt_file = os.path.join(image_dir, local_gt_file)
# image = scp.misc.imread(image_file, mode='RGB')
# gt_image = scp.misc.imread(gt_file, mode='RGB')
image = cv2.imread(image_file, cv2.IMREAD_COLOR)
hand_gt_image = cv2.imread(hand_gt_file, cv2.IMREAD_COLOR)
local_gt_image = cv2.imread(local_gt_file, cv2.IMREAD_COLOR)
gt_class = get_gt_class(hypes, data_dir, image_file)
if hypes['jitter']['fix_shape']:
shape = image.shape
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
assert (image_height >= shape[0])
assert (image_width >= shape[1])
offset_x = (image_height - shape[0]) // 2
offset_y = (image_width - shape[1]) // 2
new_image = np.zeros([image_height, image_width, 3])
new_image[offset_x:offset_x + shape[0],
offset_y:offset_y + shape[1]] = image
input_image = new_image
elif hypes['jitter']['resize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image, hand_gt_image, local_gt_image = resize_label_image(
image, hand_gt_image, local_gt_image, image_height,
image_width)
input_image = image
else:
input_image = image
"""
1st output channel: background (non-hand)
2nd output channel: hand
3rd output channel: object-center
"""
# background
gt_bg = np.all(hand_gt_image == background_color, axis=2)
# hand
gt_hand1 = np.all(hand_gt_image == hand_color1, axis=2)
gt_hand2 = np.all(hand_gt_image == hand_color2, axis=2)
gt_hand3 = np.all(hand_gt_image == hand_color3, axis=2)
gt_hand4 = np.all(hand_gt_image == hand_color4, axis=2)
gt_hand5 = np.all(hand_gt_image == hand_color5, axis=2)
# non-object
gt_no_obj = np.all(local_gt_image == background_color, axis=2)
# object location heatmap
gt_obj = np.any(local_gt_image != background_color, axis=2)
assert (gt_obj.shape == gt_bg.shape)
shape = gt_bg.shape
gt_bg = gt_bg.reshape(shape[0], shape[1], 1)
# hand masking
gt_hand1 = gt_hand1.reshape(shape[0], shape[1], 1)
gt_hand2 = gt_hand2.reshape(shape[0], shape[1], 1)
gt_hand3 = gt_hand3.reshape(shape[0], shape[1], 1)
gt_hand4 = gt_hand4.reshape(shape[0], shape[1], 1)
gt_hand5 = gt_hand5.reshape(shape[0], shape[1], 1)
# hands concatenation
gt_hand = gt_hand1 | gt_hand2 | gt_hand3 | gt_hand4 | gt_hand5
# object center masking
gt_obj = gt_obj.reshape(shape[0], shape[1], 1)
# gt_image concatenation
gt_image = np.concatenate((gt_bg, gt_hand, gt_obj), axis=2)
shape = input_image.shape
feed_dict = {image_pl: input_image}
hand, local, pred_class = sess.run(
[hand_out_layer, local_out_layer, recog_layer],
feed_dict=feed_dict)
# getting a hand estimation
output_hand = hand[:, 1].reshape(shape[0], shape[1])
# getting an object localization
output_local = local[:, 1].reshape(shape[0], shape[1])
# add gt class and predicted class to the lists
gt_class_list.append(gt_class)
pred_class_list.append(pred_class)
if hypes['jitter']['fix_shape']:
gt_shape = gt_image.shape
output_hand = output_hand[offset_x:offset_x + gt_shape[0],
offset_y:offset_y + gt_shape[1]]
output_local = output_local[offset_x:offset_x +
gt_shape[0],
offset_y:offset_y +
gt_shape[1]]
if phase == 'val':
# Saving RB Plot
ov_hand_image = seg.make_overlay(image, output_hand)
ov_local_image = seg.make_overlay(image, output_local)
name = os.path.basename(image_file)
name_hand = name.split('.')[0] + '_hand.png'
name_local = name.split('.')[0] + '_local.png'
image_list.append((name_hand, ov_hand_image))
image_list.append((name_local, ov_local_image))
name2_hand = name_hand.split('.')[0] + '_blue.png'
name2_local = name_local.split('.')[0] + '_red.png'
hard_hand = output_hand > 0.5
hard_local = output_local > 0.5
hand_image = utils.fast_overlay(image, hard_hand, \
color=[0, 0, 255, 127])
local_image = utils.fast_overlay(image, hard_local, \
color=[255, 0, 0, 127])
image_list.append((name2_hand, hand_image))
image_list.append((name2_local, local_image))
FN, FP, posNum, negNum = eval_image(
hypes, local_gt_image, output_local)
total_fp += FP
total_fn += FN
total_posnum += posNum
total_negnum += negNum
eval_dict[phase] = pxEval_maximizeFMeasure(total_posnum,
total_negnum,
total_fn,
total_fp,
thresh=thresh)
# calculate precision, recall, fscore, (and support) of classification
ret = precision_recall_fscore_support(gt_class_list,
pred_class_list,
average='macro')
eval_dict[phase]['RecogPrec'] = ret[0]
eval_dict[phase]['RecogRec'] = ret[1]
eval_dict[phase]['RecogF1'] = ret[2]
if phase == 'val':
start_time = time.time()
for i in range(10):
sess.run([hand_out_layer, local_out_layer],
feed_dict=feed_dict)
dt = (time.time() - start_time) / 10
eval_list = []
# for phase in ['train', 'val']:
for phase in ['val']:
eval_list.append(('[{}] Recognition F1'.format(phase),
100 * eval_dict[phase]['RecogF1']))
eval_list.append(('[{}] Recognition Precision'.format(phase),
100 * eval_dict[phase]['RecogPrec']))
eval_list.append(('[{}] Recognition Recall'.format(phase),
100 * eval_dict[phase]['RecogRec']))
eval_list.append(
('[{}] MaxF1'.format(phase), 100 * eval_dict[phase]['MaxF']))
eval_list.append(('[{}] BestThresh'.format(phase),
100 * eval_dict[phase]['BestThresh']))
eval_list.append(('[{}] Average Precision'.format(phase),
100 * eval_dict[phase]['AvgPrec']))
eval_list.append(('Speed (msec)', 1000 * dt))
eval_list.append(('Speed (fps)', 1 / dt))
return eval_list, image_list
def model_path():
tv_utils.set_gpus_to_use()
# if FLAGS.input_image is None:
# logging.error("No input_image was given.")
# logging.info(
# "Usage: python demo.py --input_image data/test.png "
# "[--output_image output_image] [--logdir /path/to/weights] "
# "[--gpus GPUs_to_use] ")
# exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules, image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
cv2.namedWindow("visual_image", flags=cv2.WINDOW_FREERATIO)
while True:
frame = queue.get()
cv2.imshow("image", frame)
#cv2.waitKey(1)
image = frame
# logging.info("Starting inference using {} as input".format(input_image))
# Load and resize input image
#image = scp.misc.imread(input_image)
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
cv2.imshow("visual_image", green_image)
cv2.waitKey(1)
def main(_):
tv_utils.set_gpus_to_use()
if FLAGS.input is None:
logging.error("No input was given.")
logging.info(
"Usage: python demo.py --input data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'MultiNet')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
logging.info("Loading model from: {}".format(logdir))
# Loads the model from rundir
load_out = load_united_model(logdir)
# Create list of relevant tensors to evaluate
meta_hypes, subhypes, submodules, decoded_logits, sess, image_pl = load_out
seg_softmax = decoded_logits['segmentation']['softmax']
pred_boxes_new = decoded_logits['detection']['pred_boxes_new']
pred_confidences = decoded_logits['detection']['pred_confidences']
if len(meta_hypes['model_list']) == 3:
road_softmax = decoded_logits['road']['softmax'][0]
else:
road_softmax = None
eval_list = [seg_softmax, pred_boxes_new, pred_confidences, road_softmax]
# Run some tests on the hypes
test_constant_input(subhypes)
test_segmentation_input(subhypes)
# Load and reseize Image
image_file = FLAGS.input
image = scp.misc.imread(image_file)
hypes_road = subhypes['road']
shape = image.shape
image_height = hypes_road['jitter']['image_height']
image_width = hypes_road['jitter']['image_width']
assert (image_height >= shape[0])
assert (image_width >= shape[1])
image = scp.misc.imresize(image, (image_height, image_width, 3),
interp='cubic')
import utils.train_utils as dec_utils
# Run KittiSeg model on image
feed_dict = {image_pl: image}
output = sess.run(eval_list, feed_dict=feed_dict)
seg_softmax, pred_boxes_new, pred_confidences, road_softmax = output
# Create Segmentation Overlay
shape = image.shape
seg_softmax = seg_softmax[:, 1].reshape(shape[0], shape[1])
hard = seg_softmax > 0.5
overlay_image = tv_utils.fast_overlay(image, hard)
# Draw Detection Boxes
new_img, rects = dec_utils.add_rectangles(
subhypes['detection'], [overlay_image],
pred_confidences,
pred_boxes_new,
show_removed=False,
use_stitching=True,
rnn_len=subhypes['detection']['rnn_len'],
min_conf=0.50,
tau=subhypes['detection']['tau'])
# Draw road classification
highway = (np.argmax(road_softmax) == 1)
new_img = road_draw(new_img, highway)
logging.info("")
# Printing some more output information
threshold = 0.5
accepted_predictions = []
# removing predictions <= threshold
for rect in rects:
if rect.score >= threshold:
accepted_predictions.append(rect)
print('')
logging.info("{} Cars detected".format(len(accepted_predictions)))
# Printing coordinates of predicted rects.
for i, rect in enumerate(accepted_predictions):
logging.info("")
logging.info("Coordinates of Box {}".format(i))
logging.info(" x1: {}".format(rect.x1))
logging.info(" x2: {}".format(rect.x2))
logging.info(" y1: {}".format(rect.y1))
logging.info(" y2: {}".format(rect.y2))
logging.info(" Confidence: {}".format(rect.score))
if len(meta_hypes['model_list']) == 3:
logging.info("Raw Classification Softmax outputs are: {}".format(
output[0][0]))
# Save output image file
if FLAGS.output is None:
output_base_name = FLAGS.input
out_image_name = output_base_name.split('.')[0] + '_out.png'
else:
out_image_name = FLAGS.output
scp.misc.imsave(out_image_name, new_img)
logging.info("")
logging.info("Output image has been saved to: {}".format(
os.path.realpath(out_image_name)))
logging.info("")
logging.warning("Do NOT use this Code to evaluate multiple images.")
logging.warning("Demo.py is **very slow** and designed "
"to be a tutorial to show how the MultiNet works.")
logging.warning("")
logging.warning("Please see this comment, if you like to apply demo.py to"
" multiple images see:")
logging.warning("https://github.com/MarvinTeichmann/KittiBox/"
"issues/15#issuecomment-301800058")
exit(0)
def main(_):
tv_utils.set_gpus_to_use()
if FLAGS.input_image is None:
logging.error("No input_image was given.")
logging.info(
"Usage: python demo.py --input_image data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules, image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
input_image = FLAGS.input_image
logging.info("Starting inference using {} as input".format(input_image))
# Load and resize input image
image = scp.misc.imread(input_image)
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
print("pixe_value")
print(output_image[0][1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
print("predic_val")
suoyin = np.where(street_prediction == True)
chang = len(suoyin[0])
test = np.zeros((chang, 2), dtype=np.int)
for tmp0 in range(chang):
test[tmp0][0] = suoyin[0][tmp0]
test[tmp0][1] = suoyin[1][tmp0]
print(test[0].shape)
print(suoyin[0].shape)
print(len(suoyin[0]))
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
# Save output images to disk.
if FLAGS.output_image is None:
output_base_name = input_image
else:
output_base_name = FLAGS.output_image
raw_image_name = output_base_name.split('.')[0] + '_raw.png'
rb_image_name = output_base_name.split('.')[0] + '_rb.png'
green_image_name = output_base_name.split('.')[0] + '_green.png'
#scp.misc.imsave(raw_image_name, output_image)
#scp.misc.imsave(rb_image_name, rb_image)
scp.misc.imsave(green_image_name, green_image)
logging.info("")
logging.info("Raw output image has been saved to: {}".format(
os.path.realpath(raw_image_name)))
logging.info("Red-Blue overlay of confs have been saved to: {}".format(
os.path.realpath(rb_image_name)))
logging.info("Green plot of predictions have been saved to: {}".format(
os.path.realpath(green_image_name)))
def main(_):
tv_utils.set_gpus_to_use()
if FLAGS.input_image is None:
logging.error("No input_image was given.")
logging.info(
"Usage: python demo.py --input_image data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
exit(1)
if FLAGS.logdir is None:
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
#create tf graph and build net
with tf.Graph().as_default():
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
prediction = core.build_inference_graph(hypes, modules, image=image)
logging.info("Graph build successfully.")
sess = tf.Session()
saver = tf.train.Saver()
core.load_weights(logdir, sess, saver)
input_image = FLAGS.input_image
#logging.info("start inference using {} as input".format(input_image))
cap = cv2.VideoCapture(
0) #live camera and change as video path if you have save video
while True:
#image_bgr = cv2.imread(input_image)
#b, g, r = cv2.split(image_bgr) # get b,g,r
#image = cv2.merge([r, g, b]) # switch it to rgb
ret, image = cap.read()
if hypes['jitter']['reseize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
inerp='cubic')
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
#reshape
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
rb_image = seg.make_overlay(image, output_image)
threshold = 0.5
street_prediction = output_image > threshold
green_image = tv_utils.fast_overlay(image, street_prediction)
cv2.imshow('ori', green_image) #live camera: imshow or video:save
if cv2.waitKey(25) & 0xFF == ord(
'q'): #live camera imshow, delete if occur error in video show
cv2.destroyAllWindows()
break
def evaluate(hypes, sess, image_pl, inf_out):
softmax = inf_out['softmax']
data_dir = hypes['dirs']['data_dir']
eval_dict = {}
for phase in ['train', 'val']:
data_file = hypes['data']['{}_file'.format(phase)]
data_file = os.path.join(data_dir, data_file)
image_dir = os.path.dirname(data_file)
thresh = np.array(range(0, 256))/255.0
total_fp = np.zeros(thresh.shape)
total_fn = np.zeros(thresh.shape)
total_posnum = 0
total_negnum = 0
image_list = []
with open(data_file) as file:
for i, datum in enumerate(file):
datum = datum.rstrip()
image_file, gt_file = datum.split(" ")
image_file = os.path.join(image_dir, image_file)
gt_file = os.path.join(image_dir, gt_file)
image = scp.misc.imread(image_file, mode='RGB')
gt_image = scp.misc.imread(gt_file, mode='RGB')
if hypes['jitter']['fix_shape']:
shape = image.shape
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
assert(image_height >= shape[0])
assert(image_width >= shape[1])
offset_x = (image_height - shape[0])//2
offset_y = (image_width - shape[1])//2
new_image = np.zeros([image_height, image_width, 3])
new_image[offset_x:offset_x+shape[0],
offset_y:offset_y+shape[1]] = image
input_image = new_image
elif hypes['jitter']['reseize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
gt_image_old = gt_image
image, gt_image = resize_label_image(image, gt_image,
image_height,
image_width)
input_image = image
else:
input_image = image
shape = input_image.shape
feed_dict = {image_pl: input_image}
output = sess.run([softmax], feed_dict=feed_dict)
output_im = output[0][:, 1].reshape(shape[0], shape[1])
if hypes['jitter']['fix_shape']:
gt_shape = gt_image.shape
output_im = output_im[offset_x:offset_x+gt_shape[0],
offset_y:offset_y+gt_shape[1]]
if phase == 'val':
# Saving RB Plot
ov_image = seg.make_overlay(image, output_im)
name = os.path.basename(image_file)
image_list.append((name, ov_image))
name2 = name.split('.')[0] + '_green.png'
hard = output_im > 0.5
green_image = utils.fast_overlay(image, hard)
image_list.append((name2, green_image))
FN, FP, posNum, negNum = eval_image(hypes,
gt_image, output_im)
total_fp += FP
total_fn += FN
total_posnum += posNum
total_negnum += negNum
eval_dict[phase] = seg.pxEval_maximizeFMeasure(
total_posnum, total_negnum, total_fn, total_fp, thresh=thresh)
if phase == 'val':
start_time = time.time()
for i in xrange(10):
sess.run([softmax], feed_dict=feed_dict)
dt = (time.time() - start_time)/10
eval_list = []
for phase in ['train', 'val']:
eval_list.append(('[{}] MaxF1'.format(phase),
100*eval_dict[phase]['MaxF']))
eval_list.append(('[{}] BestThresh'.format(phase),
100*eval_dict[phase]['BestThresh']))
eval_list.append(('[{}] Average Precision'.format(phase),
100*eval_dict[phase]['AvgPrec']))
eval_list.append(('Speed (msec)', 1000*dt))
eval_list.append(('Speed (fps)', 1/dt))
return eval_list, image_list
def main(_):
tv_utils.set_gpus_to_use()
if (FLAGS.input_image is None) and (FLAGS.output_image is None):
logging.error("No input_image or output_image were given.")
logging.info(
"Usage: python demo_modified.py --input_image data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules, image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
# Load and resize input image
input_image = FLAGS.input_image
input_images = open(input_image, 'r').read().splitlines()
# Run KittiSeg model on image
n = 0
for list_images in input_images:
if (n % 3) == 0:
image = scp.misc.imread(list_images)
name_split = list_images.split('/')
image_name = name_split[len(name_split) - 1].split('.')[0]
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
feed = {image_pl: image}
softmax = prediction['softmax']
#time of net foward
start_time = timeit.default_timer()
output = sess.run([softmax], feed_dict=feed)
elapsed = timeit.default_timer() - start_time
print(elapsed)
#exit(1)
#feed = {image_pl: image}
#softmax = prediction['softmax']
#output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.01
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
# Save output images to disk.
output_base_name = FLAGS.output_image
raw_image_name = output_base_name + image_name + '_raw.png'
#rb_image_name = output_base_name + image_name + '_rb.png'
green_image_name = output_base_name + image_name + '_green.png'
scp.misc.imsave(raw_image_name, output_image)
#scp.misc.imsave(rb_image_name, rb_image)
scp.misc.imsave(green_image_name, green_image)
logging.info("")
logging.info("Raw output image has been saved to: {}".format(
os.path.realpath(raw_image_name)))
#logging.info("Red-Blue overlay of confs have been saved to: {}".format(
# os.path.realpath(rb_image_name)))
logging.info(
"Green plot of predictions have been saved to: {}".format(
os.path.realpath(green_image_name)))
n = n + 1
def main(_):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.7
tv_utils.set_gpus_to_use()
if FLAGS.input_image is None:
logging.error("No input_image was given.")
logging.info(
"Usage: python demo.py --input_image data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
num_classes = 5
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules,
image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session(config=config)
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
input_image = FLAGS.input_image
logging.info("Starting inference using {} as input".format(input_image))
#Dealing with video segmentation with frame by frame
#TODO: build a commandline
loaded_video = skvideo.io.vread('t1.avi')[:100]
writer = skvideo.io.FFmpegWriter("outputvideo.avi")
for image in loaded_video:
# Load and resize input image
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image, size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run(softmax, feed_dict=feed)
print(len(output), type(output), output.shape)
# Reshape output from flat vector to 2D Image
output = np.transpose(output)
shape = image.shape
output = output.reshape(num_classes, shape[0], shape[1])
output_image = output[0].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
street_predictions = output > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
green_images = []
rb_images = []
output_images = []
for c in range(0,5):
green_images.append(tv_utils.fast_overlay(image, street_predictions[c]))
rb_images.append(seg.make_overlay(image, output[c]))
output_images.append(output[c].reshape(shape[0], shape[1]))
# Save output images to disk.
if FLAGS.output_image is None:
output_base_name = input_image
else:
output_base_name = FLAGS.output_image
#Name and save the the red blue segmentation for first 5 frames as png to debug.
green_image_names = []
rb_image_names = []
raw_image_names = []
for c in range(1,6):
green_image_names.append(output_base_name.split('.')[0] + str(c) + '_green.png')
rb_image_names.append(output_base_name.split('.')[0] + str(c) + '_rb.png')
raw_image_names.append(output_base_name.split('.')[0] + str(c) + '_raw.png')
for c in range(0,5):
print(green_image_names[c], green_images[c].shape)
scp.misc.imsave(raw_image_names[c], output_images[c])
scp.misc.imsave(rb_image_names[c], rb_images[c])
scp.misc.imsave(green_image_names[c], green_images[c])
#Output the green masked video as a file and show it to screen
writer.writeFrame(green_images[4])
cv2.imshow('frame', green_images[4])
#user can press p to quit during processing.
if cv2.waitKey(1) & 0xFF == ord('q'):
break
writer.close()
def evaluate(hypes, sess, image_pl, inf_out):
out_layer = inf_out['output']
recog_layer = inf_out['recog']
data_dir = hypes['dirs']['data_dir']
eval_dict = {}
# for phase in ['train', 'val']:
for phase in ['val']:
data_file = hypes['data']['{}_file'.format(phase)]
data_file = os.path.join(data_dir, data_file)
image_dir = os.path.dirname(data_file)
thresh = np.array(range(0, 256)) / 255.0
total_fp = np.zeros(thresh.shape)
total_fn = np.zeros(thresh.shape)
total_posnum = 0
total_negnum = 0
image_list = []
gt_class_list = []
pred_class_list = []
with open(data_file) as file:
for i, datum in enumerate(file):
datum = datum.rstrip()
image_file, gt_file = datum.split("\t")
image_file = os.path.join(image_dir, image_file)
gt_file = os.path.join(image_dir, gt_file)
# image = scp.misc.imread(image_file, mode='RGB')
# gt_image = scp.misc.imread(gt_file, mode='RGB')
image = cv2.imread(image_file, cv2.IMREAD_COLOR)
gt_image = cv2.imread(gt_file, cv2.IMREAD_COLOR)
gt_class = get_gt_class(hypes, data_dir, image_file)
if hypes['jitter']['fix_shape']:
shape = image.shape
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
assert (image_height >= shape[0])
assert (image_width >= shape[1])
offset_x = (image_height - shape[0]) // 2
offset_y = (image_width - shape[1]) // 2
new_image = np.zeros([image_height, image_width, 3])
new_image[offset_x:offset_x + shape[0],
offset_y:offset_y + shape[1]] = image
input_image = new_image
elif hypes['jitter']['resize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
gt_image_old = gt_image
image, gt_image = resize_label_image(
image, gt_image, image_height, image_width)
input_image = image
else:
input_image = image
shape = input_image.shape
feed_dict = {image_pl: input_image}
output = sess.run([out_layer, recog_layer],
feed_dict=feed_dict)
# getting an inference of object of interest location
output_im = output[0][:, 1].reshape(shape[0], shape[1])
pred_class = output[1]
# add gt class and predicted class to the lists
gt_class_list.append(gt_class)
pred_class_list.append(pred_class)
if hypes['jitter']['fix_shape']:
gt_shape = gt_image.shape
output_im = output_im[offset_x:offset_x + gt_shape[0],
offset_y:offset_y + gt_shape[1]]
if phase == 'val':
# Saving RB Plot
ov_image = seg.make_overlay(image, output_im)
name = os.path.basename(image_file)
image_list.append((name, ov_image))
name2 = name.split('.')[0] + '_blue.png'
hard = output_im > 0.5
blue_image = utils.fast_overlay(image, hard, \
color=[0, 0, 255, 127])
image_list.append((name2, blue_image))
FN, FP, posNum, negNum = eval_image(
hypes, gt_image, output_im)
total_fp += FP
total_fn += FN
total_posnum += posNum
total_negnum += negNum
eval_dict[phase] = pxEval_maximizeFMeasure(total_posnum,
total_negnum,
total_fn,
total_fp,
thresh=thresh)
# calculate precision, recall, fscore, (and support) of classification
ret = precision_recall_fscore_support(gt_class_list,
pred_class_list,
average='macro')
eval_dict[phase]['RecogPrec'] = ret[0]
eval_dict[phase]['RecogRec'] = ret[1]
eval_dict[phase]['RecogF1'] = ret[2]
if phase == 'val':
start_time = time.time()
for i in range(10):
sess.run([out_layer], feed_dict=feed_dict)
dt = (time.time() - start_time) / 10
eval_list = []
if hypes['arch']['output'] != "regress":
# for phase in ['train', 'val']:
for phase in ['val']:
eval_list.append(('[{}] Recognition F1'.format(phase),
100 * eval_dict[phase]['RecogF1']))
eval_list.append(('[{}] Recognition Precision'.format(phase),
100 * eval_dict[phase]['RecogPrec']))
eval_list.append(('[{}] Recognition Recall'.format(phase),
100 * eval_dict[phase]['RecogRec']))
eval_list.append(
('[{}] MaxF1'.format(phase), 100 * eval_dict[phase]['MaxF']))
eval_list.append(('[{}] BestThresh'.format(phase),
100 * eval_dict[phase]['BestThresh']))
eval_list.append(('[{}] Average Precision'.format(phase),
100 * eval_dict[phase]['AvgPrec']))
eval_list.append(('Speed (msec)', 1000 * dt))
eval_list.append(('Speed (fps)', 1 / dt))
return eval_list, image_list
def main(_):
tv_utils.set_gpus_to_use()
# if FLAGS.input_image is None:
# logging.error("No input_image was given.")
# logging.info(
# "Usage: python demo.py --input_image data/test.png "
# "[--output_image output_image] [--logdir /path/to/weights] "
# "[--gpus GPUs_to_use] ")
# exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules, image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
dataset = kitti_object(
os.path.join(ROOT_DIR, 'free-space/dataset/KITTI/object'))
point_pub = rospy.Publisher('cloud', PointCloud2, queue_size=50)
# picture_pub = rospy.Publisher("kitti_image",newImage,queue_size=50)
rospy.init_node('point-cloud', anonymous=True)
# h = std_msgs.msg.Header()
# h.frame_id="base_link"
# h.stamp=rospy.Time.now()
#rate = rospy.Rate(10)
#point_msg=PointCloud2()
video_dir = '/home/user/Data/lrx_work/free-space/kitti.avi'
fps = 10
num = 4541
img_size = (1241, 376)
fourcc = 'mp4v'
videoWriter = cv2.VideoWriter(video_dir, cv2.VideoWriter_fourcc(*fourcc),
fps, img_size)
calib = dataset.get_calibration(0)
for data_idx in range(len(dataset)):
#objects = dataset.get_label_objects(data_idx)
# Load and resize input image
image = dataset.get_image(data_idx)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#scp.misc.imsave('new.png', image)
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
img_height, img_width, img_channel = image.shape
print("picture-shape")
print(len(image))
print(len(image[0]))
pc_velo = dataset.get_lidar(data_idx)[:, 0:3]
print(len(pc_velo))
velo_len = len(pc_velo)
#calib = dataset.get_calibration(data_idx)
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# scp.misc.imsave('new0.png', rb_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
index = np.where(street_prediction == True)
chang = len(index[0])
print(chang)
# test = np.zeros((velo_len,2),dtype=np.int)
# for tmp0 in range(chang):
# test[tmp0][0]=index[0][tmp0]
# test[tmp0][1]=index[1][tmp0]
print("suoyindayin")
# if (chang>0):
# print(test[0][0])
# print(test[0][1])
pts_2d = calib.project_velo_to_image(pc_velo)
print(pts_2d.shape)
# print(pts_2d[1][0])
# print(pts_2d[1][1])
fig = mlab.figure(figure=None,
bgcolor=(0, 0, 0),
fgcolor=None,
engine=None,
size=(1000, 500))
fov_inds = (pts_2d[:,0]<1242) & (pts_2d[:,0]>=0) & \
(pts_2d[:,1]<370) & (pts_2d[:,1]>=0)
print(fov_inds.shape)
# print(fov_inds[1000])
# print(pc_velo.shape)
print("okok")
fov_inds = fov_inds & (pc_velo[:, 0] > 0)
print(fov_inds.shape)
imgfov_pts_2d = pts_2d[fov_inds, :]
imgfov_pc_velo = pc_velo[fov_inds, :]
pts_2d0 = calib.project_velo_to_image(imgfov_pc_velo)
fov_inds0 = (pts_2d0[:,0]<len(image[0])) & (pts_2d0[:,0]>=0) & \
(pts_2d0[:,1]<len(image)) & (pts_2d0[:,1]>=0)
fov_inds0 = fov_inds0 & (imgfov_pc_velo[:, 0] > 2.0)
print(fov_inds0.shape)
print(street_prediction.shape)
print(pts_2d0.shape)
# if(chang>0):
# print(int(imgfov_pts_2d[5,0]))
# print(int(imgfov_pts_2d[5,1]))
# print(street_prediction[int(imgfov_pts_2d[5,1]),int(imgfov_pts_2d[5,0])])
if (chang > 0):
for i in range(len(fov_inds0)):
if ((pts_2d0[i, 1] < len(street_prediction)) &
(pts_2d0[i, 0] < len(street_prediction[0]))):
fov_inds0[i] = fov_inds0[i] & (
street_prediction[int(pts_2d0[i, 1]),
int(pts_2d0[i, 0])] == True)
imgfov_pc_velo0 = imgfov_pc_velo[fov_inds0, :]
print("number")
green_image = tv_utils.fast_overlay(image, street_prediction)
# pub point-cloud topic
print(imgfov_pc_velo0.shape)
number = len(imgfov_pc_velo0)
header = std_msgs.msg.Header()
header.stamp = rospy.Time.now()
header.frame_id = "velodyne"
points = pc2.create_cloud_xyz32(header, imgfov_pc_velo0)
# point=Point()
# for t in range(0,number):
# point_x=imgfov_pc_velo0[t][0]
# point_y=imgfov_pc_velo0[t][1]
# point_z=imgfov_pc_velo0[t][2]
# point_msg.points[t].point.x=point_x
# point_msg.points[t].point.y=point_y
# point_msg.points[t].point.z=point_z
# point_pub.publish(points)
# videoWriter.write(green_image)
# bridge=CvBridge()
# picture_pub.publish(bridge.cv2_to_imgmsg(green_image,"rgb8"))
# minx=imgfov_pc_velo0[0][0]
# miny=imgfov_pc_velo0[0][1]
# minz=imgfov_pc_velo0[0][2]
# maxx=imgfov_pc_velo0[0][0]
# maxy=imgfov_pc_velo0[0][1]
# maxz=imgfov_pc_velo0[0][2]
# for t in range(len(imgfov_pc_velo0)):
# minx=min(minx,imgfov_pc_velo0[t][0])
# miny=min(miny,imgfov_pc_velo0[t][1])
# minz=min(minz,imgfov_pc_velo0[t][2])
# maxx=max(maxx,imgfov_pc_velo0[t][0])
# maxy=max(maxy,imgfov_pc_velo0[t][1])
# maxz=max(maxz,imgfov_pc_velo0[t][2])
# print(minx,miny,minz,maxx,maxy,maxz)
# width=1024
# height=1024
# img_res=np.zeros([width,height,3],dtype=np.uint8)
# for p in range(len(imgfov_pc_velo0)):
# velo_x=5*(int(imgfov_pc_velo0[p][0])+50)
# velo_y=5*(int(imgfov_pc_velo0[p][1])+50)
# img_res[velo_x][velo_y]=255
# scale=25
# if((velo_x>scale)&(velo_x+scale<1024)&(velo_y>scale)&(velo_y+scale<1024)):
# for q in range(scale):
# for m in range(scale):
# img_res[velo_x-q][velo_y-m]=255
# img_res[velo_x-q][velo_y+m]=255
# img_res[velo_x+q][velo_y-m]=255
# img_res[velo_x+q][velo_y+m]=255
# scp.misc.imsave('res.png',img_res)
draw_lidar(imgfov_pc_velo0, fig=fig)
# for obj in objects:
# if obj.type == 'DontCare': continue
# # Draw 3d bounding box
# box3d_pts_2d, box3d_pts_3d = utils.compute_box_3d(obj, calib.P)
# box3d_pts_3d_velo = calib.project_rect_to_velo(box3d_pts_3d)
# # Draw heading arrow
# ori3d_pts_2d, ori3d_pts_3d = utils.compute_orientation_3d(obj, calib.P)
# ori3d_pts_3d_velo = calib.project_rect_to_velo(ori3d_pts_3d)
# x1, y1, z1 = ori3d_pts_3d_velo[0, :]
# x2, y2, z2 = ori3d_pts_3d_velo[1, :]
# draw_gt_boxes3d([box3d_pts_3d_velo], fig=fig)
# mlab.plot3d([x1, x2], [y1, y2], [z1, z2], color=(0.5, 0.5, 0.5),
# tube_radius=None, line_width=1, figure=fig)
#mlab.show(1)
# Plot the hard prediction as green overlay
# Save output images to disk.
if FLAGS.output_image is None:
output_base_name = image
else:
output_base_name = FLAGS.output_image
# raw_image_name = output_base_name.split('.')[0] + '_raw.png'
# rb_image_name = output_base_name.split('.')[0] + '_rb.png'
# green_image_name = output_base_name.split('.')[0] + '_green.png'
# scp.misc.imsave('1.png', output_image)
# scp.misc.imsave('2.png', rb_image)
# scp.misc.imsave('3.png', green_image)
raw_input()
def test_park_iacas():
params = net.monodepth_main.monodepth_parameters(
encoder='vgg',
height=256,
width=512,
batch_size=8,
num_threads=8,
num_epochs=50,
do_stereo=False,
wrap_mode='border',
use_deconv=False,
alpha_image_loss=0.85,
disp_gradient_loss_weight=0.1,
lr_loss_weight=1.0,
full_summary=True)
root_path = '/home/chen-tian/data/code/learningReloc/'
data_path = '/home/chen-tian/data/SelfData/apple/'
runs_dir = 'RUNS/KittiSeg_pretrained/'
net_dir = 'homo_net/'
test_params = net.utils.utils.test_parameters(
root_path=root_path + net_dir,
data_path=data_path,
filenames_file=root_path +
'net/utils/filenames//kitti_odom_color_depth.txt',
dataset='kitti',
mode='test',
checkpoint_path=root_path + 'net/data/model/model_kitti',
log_directory=root_path + net_dir + runs_dir,
output_directory=data_path + 'learningReloc/output/',
kitti_calib=data_path + 'dataset/sequences/00/calib.txt',
trajectory_file=data_path + 'dataset/poses/00.txt',
height_origin=370,
width_origin=1226,
calib_ext_file='',
calib_int_file='',
ground_truth_image='')
video_name = '/home/chen-tian/data/SelfData/apple/IMG_0015.MOV'
cap = cv2.VideoCapture(video_name)
segnet = seg_net()
segnet.build_net(test_params)
image_lists = segnet.load_data(test_params.filenames_file)
cnt = 0
ret, frame = cap.read()
while ret:
#cv2.imshow('test', frame)
ret, frame = cap.read()
img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
h, w, c = img.shape
img = cv2.resize(img, (w / 4, h / 4))
# for left, right in image_lists:
# img = cv2.imread(test_params.data_path + left, cv2.IMREAD_UNCHANGED)
# img = scp.misc.imread(test_params.data_path + left)
tic = time.clock()
output = segnet.run_sess(img)
toc = time.clock()
print 'time cost', toc - tic
shape = img.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = tv_seg.make_overlay(img, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a 'hard' prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(img, street_prediction)
cv2.imshow('kitti', green_image)
cv2.imwrite(test_params.output_directory + '/%d.png' % cnt,
green_image)
cnt += 1
cv2.waitKey(10)
def callback(image, Pointcloud):
print("ros topic input")
gen = point_cloud2.read_points(Pointcloud,
field_names=("x", "y", "z"),
skip_nans=True)
n = 30000
# for q in gen:
# n=n+1
# print(n)
pc_velo = np.zeros([n, 3])
i = 0
for p in gen:
pc_velo[i, 0] = p[0]
pc_velo[i, 1] = p[1]
pc_velo[i, 2] = p[2]
i = i + 1
print(i)
image = brideg.imgmsg_to_cv2(image)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# undistort
h, w = image.shape[:2]
k = np.array(P_0)
d = np.array(distort)
mapx, mapy = cv2.initUndistortRectifyMap(k, d, None, k, (w, h), 5)
image = cv2.remap(image, mapx, mapy, cv2.INTER_LINEAR)
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image,
size=(image_height, image_width),
interp='cubic')
img_height, img_width, img_channel = image.shape
# fig = mlab.figure(figure=None, bgcolor=(0, 0, 0),
# fgcolor=None, engine=None, size=(1000, 500))
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
index = np.where(street_prediction == True)
chang = len(index[0])
print(chang)
#pts_2d=calib.project_velo_to_image(pc_velo)
pc_4 = expend3(pc_velo)
cam_3d = np.dot(pc_4, np.transpose(P_1))
pts_3d = np.dot(cam_3d, np.transpose(P_0))
pts_3d[:, 0] /= cam_3d[:, 2]
pts_3d[:, 1] /= cam_3d[:, 2]
pts_2d = pts_3d[:, 0:2]
print(pts_2d.shape)
print("image coordinate")
print(pts_2d[1, 0])
print(pts_2d[1, 1])
# fov_inds = (pts_2d[:,0]<640) & (pts_2d[:,0]>=0) & \
# (pts_2d[:,1]<480) & (pts_2d[:,1]>=0)
# fov_inds = (pts_2d[:,0]<640) & (pts_2d[:,0]>0) & \
# (pts_2d[:,1]<480) & (pts_2d[:,1]>0)
# fov_inds = fov_inds & (pc_velo[:,0]<0)
fov_inds = pc_velo[:, 0] > 0
print(fov_inds.shape)
print(pts_2d.shape)
imgfov_pc_velo = pc_velo[fov_inds, :]
print(imgfov_pc_velo.shape)
#pts_2d0=calib.project_velo_to_image(imgfov_pc_velo)
pc_4_0 = expend3(imgfov_pc_velo)
cam_3d_0 = np.dot(pc_4_0, np.transpose(P_1))
pts_3d_0 = np.dot(cam_3d_0, np.transpose(P_0))
pts_3d_0[:, 0] /= cam_3d_0[:, 2]
pts_3d_0[:, 1] /= cam_3d_0[:, 2]
pts_2d0 = pts_3d_0[:, 0:2]
print("camera")
print(pts_2d0.shape)
print("image size")
print(len(image[0]))
print(len(image))
fov_inds0 = (pts_2d0[:,0]<len(image[0])) & (pts_2d0[:,0]>=0) & \
(pts_2d0[:,1]<len(image)) & (pts_2d0[:,1]>=0)
imgfov_pc_velo = pc_velo[fov_inds, :]
#fov_inds0 = fov_inds0 & (imgfov_pc_velo[:,0]>2.0)
print(fov_inds0.shape)
#imgfov_pc_velo0 = imgfov_pc_velo[fov_inds0, :]
#print(imgfov_pc_velo0.shape)
if (chang > 0):
for i in range(len(fov_inds0)):
if ((pts_2d0[i, 1] < len(street_prediction)) &
(pts_2d0[i, 0] < len(street_prediction[0])) &
(pts_2d0[i, 0] >= 0) & (pts_2d0[i, 1] >= 0)):
fov_inds0[i] = fov_inds0[i] & (
street_prediction[int(pts_2d0[i, 1]),
int(pts_2d0[i, 0])] == True)
# if(chang>0):
# for i in range(len(fov_inds)):
# if((pts_2d0[i,1]<len(street_prediction))&(pts_2d0[i,0]<len(street_prediction[0]))):
# fov_inds[i]=fov_inds[i] & (street_prediction[int(pts_2d0[i,1]),int(pts_2d0[i,0])]==True)
#imgfov_pc_velo0 = imgfov_pc_velo[fov_inds0, :]
print("number")
green_image = tv_utils.fast_overlay(image, street_prediction)
imgfov_pc_velo0 = imgfov_pc_velo[fov_inds0, :]
# pub point-cloud topic
print(imgfov_pc_velo0.shape)
videoWriter.write(green_image)
number = len(imgfov_pc_velo0)
# draw_lidar(pc_velo, fig=fig)
header = std_msgs.msg.Header()
header.stamp = rospy.Time.now()
header.frame_id = "velodyne"
points = pc2.create_cloud_xyz32(header, imgfov_pc_velo0)
point_pub.publish(points)
for i in range(2):
# Load and reseize Image
image_file = "/home/fregu856/Zenuity/MultiNet/data/demo/test%d.jpg" % i
image = scp.misc.imread(image_file)
hypes_road = subhypes['road']
image_height = hypes_road['jitter']['image_height']
image_width = hypes_road['jitter']['image_width']
image = scp.misc.imresize(image, (image_height,
image_width, 3),
interp='cubic')
# Run KittiSeg model on image
feed_dict = {image_pl: image}
output = sess.run(eval_list, feed_dict=feed_dict)
seg_softmax = output[0]
# Create Segmentation Overlay
shape = image.shape
seg_softmax = seg_softmax[:, 1].reshape(shape[0], shape[1])
hard = seg_softmax > 0.5
image = tv_utils.fast_overlay(image, hard)
image = image.astype(np.uint8)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
cv2.imshow("test", image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def run_eval(load_out, output_folder, data_file):
meta_hypes, subhypes, submodules, decoded_logits, sess, image_pl = load_out
#if the model list is having 3 models; classification, detection and segentation then
assert (len(meta_hypes['model_list']) == 3)
# inf_out['pred_boxes_new'], inf_out['pred_confidences']
seg_softmax = decoded_logits['segmentation'][
'softmax'] # softmax in segmenation
pred_boxes_new = decoded_logits['detection'][
'pred_boxes_new'] #bounding boxes in detection
pred_confidences = decoded_logits['detection'][
'pred_confidences'] #confidence level in detection
road_softmax = decoded_logits['road']['softmax'][
0] #softmax in classification
eval_list = [seg_softmax, pred_boxes_new, pred_confidences, road_softmax]
#process the image by resizing
def my_process(image):
return process_image(subhypes, image)
#calculate the evaluation run time
if FLAGS.speed_test:
eval_runtime(sess, subhypes, image_pl, eval_list, data_file)
exit(0)
test_constant_input(subhypes)
test_segmentation_input(subhypes)
import utils.train_utils as dec_utils
#generate the output
gen = _output_generator(sess, eval_list, image_pl, data_file, my_process)
for image_file, output in gen:
#read the image
image = scp.misc.imread(image_file)
#process the image by resizing it
image = process_image(subhypes, image)
#shape of an image
shape = image.shape
seg_softmax, pred_boxes_new, pred_confidences, road_softmax = output
# Create Segmentation Overlay
shape = image.shape
seg_softmax = seg_softmax[:, 1].reshape(shape[0], shape[1])
#if threshold value is greater than 0.5, it is classified as hard segmentation
hard = seg_softmax > 0.5
overlay_image = utils.fast_overlay(image, hard)
# Draw Detection Boxes
new_img, rects = dec_utils.add_rectangles(
subhypes['detection'], [overlay_image],
pred_confidences,
pred_boxes_new,
show_removed=False,
use_stitching=True,
rnn_len=subhypes['detection']['rnn_len'],
min_conf=0.50,
tau=subhypes['detection']['tau'])
# Draw road classification
highway = (np.argmax(road_softmax) == 1)
new_img = road_draw(new_img, highway)
# Save image file
im_name = os.path.basename(image_file)
new_im_file = os.path.join(output_folder, im_name)
im_name = os.path.basename(image_file)
new_im_file = os.path.join(output_folder, im_name)
scp.misc.imsave(new_im_file, new_img)
logging.info("Plotting file: {}".format(new_im_file))
eval_runtime(sess, subhypes, image_pl, eval_list, data_file)
exit(0)
def evaluate(hypes, sess, image_pl, inf_out):
#softmax
softmax = inf_out['softmax']
data_dir = hypes['dirs']['data_dir']
eval_dict = {}
for phase in ['train', 'val']:
data_file = hypes['data']['{}_file'.format(phase)]
data_file = os.path.join(data_dir, data_file)
image_dir = os.path.dirname(data_file)
thresh = np.array(range(0, 256))/255.0
total_fp = np.zeros(thresh.shape)
total_fn = np.zeros(thresh.shape)
total_posnum = 0
total_negnum = 0
image_list = []
with open(data_file) as file:
for i, datum in enumerate(file):
datum = datum.rstrip()
image_file, gt_file = datum.split(" ")
image_file = os.path.join(image_dir, image_file)
gt_file = os.path.join(image_dir, gt_file)
image = scp.misc.imread(image_file, mode='RGB')
gt_image = scp.misc.imread(gt_file, mode='RGB')
#accoriding to the image rescaling options specified in hypes
if hypes['jitter']['fix_shape']:
shape = image.shape
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
assert(image_height >= shape[0])
assert(image_width >= shape[1])
offset_x = (image_height - shape[0])//2
offset_y = (image_width - shape[1])//2
new_image = np.zeros([image_height, image_width, 3])
new_image[offset_x:offset_x+shape[0],
offset_y:offset_y+shape[1]] = image
input_image = new_image
elif hypes['jitter']['reseize_image']:
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
gt_image_old = gt_image
image, gt_image = resize_label_image(image, gt_image,
image_height,
image_width)
input_image = image
else:
input_image = image
shape = input_image.shape
feed_dict = {image_pl: input_image}
output = sess.run([softmax], feed_dict=feed_dict)
output_im = output[0][:, 1].reshape(shape[0], shape[1])
if hypes['jitter']['fix_shape']:
gt_shape = gt_image.shape
output_im = output_im[offset_x:offset_x+gt_shape[0],
offset_y:offset_y+gt_shape[1]]
#if this is a validation process
if phase == 'val':
# Saving RB Plot
ov_image = seg.make_overlay(image, output_im)
name = os.path.basename(image_file)
image_list.append((name, ov_image))
name2 = name.split('.')[0] + '_green.png'
hard = output_im > 0.5
green_image = utils.fast_overlay(image, hard)
image_list.append((name2, green_image))
FN, FP, posNum, negNum = eval_image(hypes,
gt_image, output_im)
total_fp += FP
total_fn += FN
total_posnum += posNum
total_negnum += negNum
eval_dict[phase] = seg.pxEval_maximizeFMeasure(
total_posnum, total_negnum, total_fn, total_fp, thresh=thresh)
if phase == 'val':
start_time = time.time()
for i in xrange(10):
sess.run([softmax], feed_dict=feed_dict)
dt = (time.time() - start_time)/10
eval_list = []
#performance measures
for phase in ['train', 'val']:
eval_list.append(('[{}] MaxF1'.format(phase),
100*eval_dict[phase]['MaxF']))
eval_list.append(('[{}] BestThresh'.format(phase),
100*eval_dict[phase]['BestThresh']))
eval_list.append(('[{}] Average Precision'.format(phase),
100*eval_dict[phase]['AvgPrec']))
eval_list.append(('Speed (msec)', 1000*dt))
eval_list.append(('Speed (fps)', 1/dt))
return eval_list, image_list
def create_test_output(hypes, sess, image_pl, softmax):
data_file = hypes['data']['test_file']
image_dir = os.path.dirname(data_file)
logdir = "test_images/"
logdir_rb = "test_images_rb/"
logdir_green = "test_images_green/"
logging.info(
"Images will be written to {}/test_images_{{green, rg}}".format(
logdir))
logdir = os.path.join(hypes['dirs']['output_dir'], logdir)
logdir_rb = os.path.join(hypes['dirs']['output_dir'], logdir_rb)
logdir_green = os.path.join(hypes['dirs']['output_dir'], logdir_green)
num_classes = hypes['arch']['num_classes']
if not os.path.exists(logdir):
os.mkdir(logdir)
if not os.path.exists(logdir_rb):
os.mkdir(logdir_rb)
if not os.path.exists(logdir_green):
os.mkdir(logdir_green)
with open(data_file) as file:
for i, image_file in enumerate(file):
image_file = image_file.rstrip()
image_file = os.path.join(image_dir, image_file)
image = scp.misc.imread(image_file)
shape = image.shape
feed_dict = {image_pl: image}
output = sess.run([softmax['softmax']], feed_dict=feed_dict)
output_images = []
ov_images = []
green_images = []
for i in range(num_classes):
output_im = output[0][:, i].reshape(shape[0], shape[1])
output_images.append(output_im)
ov_images.append(seg.make_overlay(image, output_im))
hard = output_im > 0.5
green_images.append(utils.fast_overlay(image, hard))
full_name = os.path.basename(image_file)
name = os.path.splitext(full_name)[0]
ext = os.path.splitext(full_name)[1]
logging.info("Writing file: %s", full_name)
for i in range(num_classes):
name_i = name + str(i) + '.' + ext
save_file = os.path.join(logdir, name_i)
scp.misc.imsave(save_file, output_images[i])
save_file = os.path.join(logdir_rb, name_i)
scp.misc.imsave(save_file, ov_images[i])
save_file = os.path.join(logdir_green, name_i)
scp.misc.imsave(save_file, green_images[i])
def main(_):
tv_utils.set_gpus_to_use()
if FLAGS.input_image is None:
logging.error("No input_image was given.")
logging.info(
"Usage: python demo.py --input_image data/test.png "
"[--output_image output_image] [--logdir /path/to/weights] "
"[--gpus GPUs_to_use] ")
exit(1)
if FLAGS.logdir is None:
# Download and use weights from the MultiNet Paper
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
else:
runs_dir = 'RUNS'
maybe_download_and_extract(runs_dir)
logdir = os.path.join(runs_dir, default_run)
else:
logging.info("Using weights found in {}".format(FLAGS.logdir))
logdir = FLAGS.logdir
# Loading hyperparameters from logdir
hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes')
logging.info("Hypes loaded successfully.")
# Loading tv modules (encoder.py, decoder.py, eval.py) from logdir
modules = tv_utils.load_modules_from_logdir(logdir)
logging.info("Modules loaded successfully. Starting to build tf graph.")
# Create tf graph and build module.
with tf.Graph().as_default():
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules,
image=image)
logging.info("Graph build successfully.")
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
# Load weights from logdir
core.load_weights(logdir, sess, saver)
logging.info("Weights loaded successfully.")
input_image = FLAGS.input_image
logging.info("Starting inference using {} as input".format(input_image))
# Load and resize input image
image = scp.misc.imread(input_image)
if hypes['jitter']['reseize_image']:
# Resize input only, if specified in hypes
image_height = hypes['jitter']['image_height']
image_width = hypes['jitter']['image_width']
image = scp.misc.imresize(image, size=(image_height, image_width),
interp='cubic')
# Run KittiSeg model on image
feed = {image_pl: image}
softmax = prediction['softmax']
output = sess.run([softmax], feed_dict=feed)
# Reshape output from flat vector to 2D Image
shape = image.shape
output_image = output[0][:, 1].reshape(shape[0], shape[1])
# Plot confidences as red-blue overlay
rb_image = seg.make_overlay(image, output_image)
# Accept all pixel with conf >= 0.5 as positive prediction
# This creates a `hard` prediction result for class street
threshold = 0.5
street_prediction = output_image > threshold
# Plot the hard prediction as green overlay
green_image = tv_utils.fast_overlay(image, street_prediction)
# Save output images to disk.
if FLAGS.output_image is None:
output_base_name = input_image
else:
output_base_name = FLAGS.output_image
raw_image_name = output_base_name.split('.')[0] + '_raw.png'
rb_image_name = output_base_name.split('.')[0] + '_rb.png'
green_image_name = output_base_name.split('.')[0] + '_green.png'
scp.misc.imsave(raw_image_name, output_image)
scp.misc.imsave(rb_image_name, rb_image)
scp.misc.imsave(green_image_name, green_image)
logging.info("")
logging.info("Raw output image has been saved to: {}".format(
os.path.realpath(raw_image_name)))
logging.info("Red-Blue overlay of confs have been saved to: {}".format(
os.path.realpath(rb_image_name)))
logging.info("Green plot of predictions have been saved to: {}".format(
os.path.realpath(green_image_name)))
logging.info("")
logging.warning("Do NOT use this Code to evaluate multiple images.")
logging.warning("Demo.py is **very slow** and designed "
"to be a tutorial to show how the KittiSeg works.")
logging.warning("")
logging.warning("Please see this comment, if you like to apply demo.py to"
"multiple images see:")
logging.warning("https://github.com/MarvinTeichmann/KittiBox/"
"issues/15#issuecomment-301800058")
def create_test_output(hypes, sess, image_pl, softmax, data_file):
# data_dir = hypes['dirs']['data_dir']
# data_file = os.path.join(data_dir, test_file)
image_dir = os.path.dirname(data_file)
logdir = "test_images_grey/"
logdir_rb = "test_images_rb/"
logdir_green = "test_images_green/"
logging.info(
"Images will be written to {}/test_images_{{green, rg}}".format(
logdir))
logdir = os.path.join(hypes['dirs']['output_dir'], logdir)
logdir_rb = os.path.join(hypes['dirs']['output_dir'], logdir_rb)
logdir_green = os.path.join(hypes['dirs']['output_dir'], logdir_green)
if not os.path.exists(logdir):
os.mkdir(logdir)
if not os.path.exists(logdir_rb):
os.mkdir(logdir_rb)
if not os.path.exists(logdir_green):
os.mkdir(logdir_green)
image_list = []
with open(data_file) as file:
for i, image_file in enumerate(file):
t = time.time()
image_file = image_file.rstrip()
image_file = os.path.join(image_dir, image_file)
image = scp.misc.imread(image_file)
shape = image.shape
feed_dict = {image_pl: image}
output = sess.run([softmax['softmax']],
feed_dict=feed_dict) # !!!!
output_im = output[0][:, 1].reshape(shape[0], shape[1])
ov_image = seg.make_overlay(image, output_im)
hard = output_im > 0.5
green_image = utils.fast_overlay(image, hard)
name = os.path.basename(image_file)
body, ext = name.split(".")
new_name = body + "_grey." + ext
save_file = os.path.join(logdir, new_name)
logging.info("Writing file: %s", save_file)
scp.misc.imsave(save_file, output_im)
new_name = body + "_rb." + ext
save_file = os.path.join(logdir_rb, new_name)
scp.misc.imsave(save_file, ov_image)
new_name = body + "_green." + ext
save_file = os.path.join(logdir_green, new_name)
scp.misc.imsave(save_file, green_image)
elapsed = time.time() - t
print("elapsed time: " + str(elapsed))
