def predict(self, image, point_cloud, frame_calib):
with tf.Graph().as_default():
model = AvodModel(self.model_config,
train_val_test='test',
dataset=self.dataset)
# The model should return a dictionary of predictions
prediction_dict = model.build()
self._saver = tf.train.Saver()
feed_dict = model.create_pred_feed_dict(image, point_cloud,
frame_calib)
sess = self._create_sess()
restored_checkpoint = self._restore_chkpt()
self._saver.restore(sess, restored_checkpoint)
predictions = sess.run(prediction_dict, feed_dict=feed_dict)
box_rep = self.model_config.avod_config.avod_box_representation
proposals_and_scores = \
self.get_rpn_proposals_and_scores(predictions)
predictions_and_scores = \
self.get_avod_predicted_boxes_3d_and_scores(predictions,
box_rep)
print("Attempting to draw predictions")
drawn_image = self._draw_predictions(image, predictions_and_scores,
frame_calib)
def get_proposal_network(model_config, dataset, model_path, GPU_INDEX=0):
with tf.Graph().as_default():
with tf.device('/gpu:'+str(GPU_INDEX)):
rpn_model = AvodModel(model_config,
train_val_test='test',
dataset=dataset)
rpn_pred = rpn_model.build()
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
saver.restore(sess, model_path)
return rpn_pred, sess, rpn_model
def test_avod_loss(self):
# tests the set up for the model and the loss
# Use "val" so that the first sample is loaded each time
avod_model = AvodModel(self.model_config,
train_val_test="val",
dataset=self.dataset)
predictions = avod_model.build()
loss, total_loss = avod_model.loss(predictions)
feed_dict = avod_model.create_feed_dict()
with self.test_session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
loss_dict_out = sess.run(loss, feed_dict=feed_dict)
print('Losses ', loss_dict_out)
def test(model_config, eval_config,
dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
#model_evaluator = Evaluator(model, dataset_config, eval_config)
#model_evaluator.run_latest_checkpoints()
# Create a variable tensor to hold the global step
global_step_tensor = tf.Variable(0, trainable=False, name='global_step')
allow_gpu_mem_growth = eval_config.allow_gpu_mem_growth
if allow_gpu_mem_growth:
# GPU memory config
config = tf.ConfigProto()
config.gpu_options.allow_growth = allow_gpu_mem_growth
_sess = tf.Session(config=config)
else:
_sess = tf.Session()
_prediction_dict = model.build()
_saver = tf.train.Saver()
trainer_utils.load_checkpoints(model_config.paths_config.checkpoint_dir,
_saver)
num_checkpoints = len(_saver.last_checkpoints)
print("test:",num_checkpoints)
checkpoint_to_restore = _saver.last_checkpoints[num_checkpoints-1]
_saver.restore(_sess, checkpoint_to_restore)
num_samples = model.dataset.num_samples
num_valid_samples = 0
current_epoch = model.dataset.epochs_completed
while current_epoch == model.dataset.epochs_completed:
# Keep track of feed_dict speed
start_time = time.time()
feed_dict = model.create_feed_dict()
feed_dict_time = time.time() - start_time
# Get sample name from model
sample_name = model.sample_info['sample_name']
num_valid_samples += 1
print("Step: {} / {}, Inference on sample {}".format(
num_valid_samples, num_samples,
sample_name))
print("test mode")
inference_start_time = time.time()
# Don't calculate loss or run summaries for test
predictions = _sess.run(_prediction_dict,
feed_dict=feed_dict)
inference_time = time.time() - inference_start_time
print("inference time:", inference_time)
predictions_and_scores = get_avod_predicted_boxes_3d_and_scores(predictions)
#print(predictions_and_scores)
#im_path = os.path.join(dataset_dir, 'training/image_2/{:06d}.png'.format(img_idx))
#im = cv2.imread(im_path)
#cv2.imshow('result',im)
#cv2.waitKey(30)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
gt_classes = ['Car']
fig_size = (10, 6.1)
avod_score_threshold = 0.1
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
if len(prediction_boxes_3d) > 0:
dataset_dir = model.dataset.dataset_dir
sample_name = (model.dataset.sample_names[model.dataset._index_in_epoch - 1])
img_idx = int(sample_name)
print("frame_index",img_idx)
image_path = model.dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
if model.dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
filtered_gt_objs = model.dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
stereo_calib = calib_utils.read_calibration(dataset.calib_dir,
img_idx)
calib_p2 = stereo_calib.p2
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, calib_p2,
truncate=True, image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
True,
True,
gt_classes,
False)
#cv2.imshow('result',pred_fig)
print(type(pred_fig))
pred_fig.canvas.draw()
img = np.fromstring(pred_fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
img = img.reshape(pred_fig.canvas.get_width_height()[::-1] + (3,))
cv2.imshow('result',img)
#draw bird view
kitti_utils = model.dataset.kitti_utils
print(img.shape[0:2])
point_cloud = kitti_utils.get_point_cloud(
'lidar', img_idx, (370, 1242))
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
density_map = np.array(bev_images.get("density_map"))
_, box_points_norm = box_3d_projector.project_to_bev(
final_prediction_boxes_3d, [[-40, 40], [0, 70]])
density_map = draw_boxes(density_map, box_points_norm)
cv2.imshow('lidar',density_map)
cv2.waitKey(-1)
