def test(split, save_result=False):
if save_result and not os.path.exists('./rcnn_data_' + split):
os.mkdir('./rcnn_data_' + split)
is_training = False
#dataset = Dataset(NUM_POINT, '/data/ssd/public/jlliu/Kitti/object', split, is_training=is_training)
dataset = Dataset(NUM_POINT, KITTI_PATH, split, is_training=is_training)
# data loading threads
produce_thread = Thread(target=dataset.load, args=(False, ))
produce_thread.start()
with tf.Graph().as_default():
with tf.device('/gpu:0'):
rpn_model = RPN(BATCH_SIZE,
NUM_POINT,
num_channel=4,
is_training=is_training)
pls = rpn_model.placeholders
end_points = rpn_model.end_points
box_center, box_angle, box_size = rpn_model.box_encoder.tf_decode(
end_points)
box_center = box_center + end_points['fg_points_xyz']
#box_center = tf.reshape(box_center, [BATCH_SIZE * NUM_FG_POINT,3])
#box_angle = tf.reshape(box_angle, [BATCH_SIZE * NUM_FG_POINT])
#box_size = tf.reshape(box_size, [BATCH_SIZE * NUM_FG_POINT,3])
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = tf.Session(config=config)
saver.restore(sess, FLAGS.model_path)
with tf.Graph().as_default():
with tf.device('/gpu:0'):
#img_seg_net = ImgSegNet(BATCH_SIZE, NUM_POINT, num_channel=4, bn_decay=None, is_training=is_training)
#seg_softmax = img_seg_net.get_seg_softmax()
#saver1 = tf.train.Saver()
img_seg_net = ImgSegNet(BATCH_SIZE, NUM_POINT)
#img_seg_net.load_graph('/data/ssd/public/jlliu/models/research/deeplab/exp/frozen_inference_graph.pb')
img_seg_net.load_graph(FLAGS.img_model_path)
seg_softmax = img_seg_net.get_seg_softmax()
full_seg = img_seg_net.get_semantic_seg()
# Create another session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess1 = tf.Session(config=config)
#saver1.restore(sess1, FLAGS.img_model_path)
log_string(str(datetime.now()))
# To collect statistics
total_correct = 0
total_seen = 0
num_batches = 0
tp = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
fp = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
fn = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
proposal_boxes = []
gt_boxes = []
nms_indices = []
while (True):
batch_data, is_last_batch = dataset.get_next_batch(BATCH_SIZE,
need_id=True)
start = datetime.now()
feed_dict = {
pls['pointclouds']: batch_data['pointcloud'],
pls['img_inputs']: batch_data['images'],
pls['calib']: batch_data['calib'],
pls['seg_labels']: batch_data['seg_label'],
pls['center_bin_x_labels']: batch_data['center_x_cls'],
pls['center_bin_z_labels']: batch_data['center_z_cls'],
pls['center_x_residuals_labels']: batch_data['center_x_res'],
pls['center_y_residuals_labels']: batch_data['center_y_res'],
pls['center_z_residuals_labels']: batch_data['center_z_res'],
pls['heading_bin_labels']: batch_data['angle_cls'],
pls['heading_residuals_labels']: batch_data['angle_res'],
pls['size_class_labels']: batch_data['size_cls'],
pls['size_residuals_labels']: batch_data['size_res'],
pls['gt_box_of_point']: batch_data['gt_box_of_point'],
pls['is_training_pl']: is_training,
}
# segmentaion with image
seg_pls = img_seg_net.placeholders
img_seg_logits, full_img_seg = sess1.run(
[seg_softmax, full_seg],
feed_dict={
seg_pls['pointclouds']: batch_data['pointcloud'],
seg_pls['img_inputs']: batch_data['images'],
seg_pls['calib']: batch_data['calib'],
seg_pls['seg_labels']: batch_data['seg_label'],
#seg_pls['is_training_pl']: is_training
})
# convert to binary segmentation
img_seg_binary = np.zeros((BATCH_SIZE, NUM_POINT, 2))
img_seg_binary[..., 0] = img_seg_logits[..., 0]
img_seg_binary[..., 1] = np.sum(img_seg_logits[..., 1:], axis=-1)
img_seg_binary *= np.array([0, 1]) # weights
feed_dict[pls['img_seg_softmax']] = img_seg_binary
'''
# label to one_hot
targets = batch_data['seg_label']
img_seg_logits = np.eye(NUM_SEG_CLASSES)[targets]
#img_seg_logits *= np.array([2, 2, 2, 2]) # weights
feed_dict[pls['img_seg_softmax']] = img_seg_logits
'''
logits_val, indices_val, centers_val, angles_val, sizes_val, corners_val, ind_val, scores_val \
= sess.run([
end_points['foreground_logits'], end_points['fg_point_indices'],
box_center, box_angle, box_size, end_points['proposal_boxes'],
end_points['nms_indices'], end_points['proposal_scores']], feed_dict=feed_dict)
print('inference time: ', datetime.now() - start)
# segmentation acc
preds_val = np.argmax(logits_val, 2)
num_batches += 1
for c in ['Car', 'Pedestrian', 'Cyclist']:
one_hot_class = g_type2onehotclass[c]
tp[c] += np.sum(
np.logical_and(preds_val == batch_data['seg_label'],
batch_data['seg_label'] == one_hot_class))
fp[c] += np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] != one_hot_class))
fn[c] += np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] == one_hot_class))
# results
for i in range(BATCH_SIZE):
proposal_boxes.append(corners_val[i])
gt_boxes.append(batch_data['gt_boxes'][i])
nms_indices.append(ind_val[i])
frame_data = {
'frame_id': batch_data['ids'][i],
'segmentation': preds_val[i],
'centers': centers_val[i],
'angles': angles_val[i],
'sizes': sizes_val[i],
'proposal_boxes': corners_val[i],
'nms_indices': ind_val[i],
'scores': scores_val[i],
'pc_choices': batch_data['pc_choice'][i]
}
# save frame data
if save_result:
with open(
os.path.join('./rcnn_data_' + split,
batch_data['ids'][i] + '.pkl'),
'wb') as fout:
pickle.dump(frame_data, fout)
np.save(
os.path.join('./rcnn_data_' + split,
batch_data['ids'][i] + '_seg.npy'),
full_img_seg[i])
if is_last_batch:
break
log_string('saved prediction')
dataset.stop_loading()
produce_thread.join()
'''
all_indices = np.tile(np.arange(1024), (len(proposal_boxes),))
iou2d, iou3d = compute_box3d_iou(proposal_boxes, point_gt_boxes, all_indices)
print('IOU2d: ', np.mean(iou2d))
print('IOU3d: ', np.mean(iou3d))
'''
if split in ['train', 'val']:
recall = compute_proposal_recall(proposal_boxes, gt_boxes, nms_indices)
print('Average recall: ', recall)
print(tp, fp, fn)
for c in ['Car', 'Pedestrian', 'Cyclist']:
if (tp[c] + fn[c] == 0) or (tp[c] + fp[c]) == 0:
continue
print(c + ' segmentation recall: %f'% \
(float(tp[c])/(tp[c]+fn[c])))
print(c + ' segmentation precision: %f'% \
(float(tp[c])/(tp[c]+fp[c])))
def train_one_epoch(sess, ops, pls, train_writer, more=False):
is_training = True
log_string(str(datetime.now()))
# To collect statistics
total_correct = 0
total_seen = 0
total_tp = 0
total_fp = 0
total_fn = 0
loss_sum = 0
iou2ds_sum = 0
iou3ds_sum = 0
total_nms = 0
total_proposal_recall = 0
# Training with batches
batch_idx = 0
while (True):
batch_data, is_last_batch = TRAIN_DATASET.get_next_batch(BATCH_SIZE)
feed_dict = {
pls['pointclouds']:
batch_data['pointcloud'],
pls['img_inputs']:
batch_data['images'],
pls['calib']:
batch_data['calib'],
pls['seg_labels']:
batch_data['seg_label'],
pls['center_bin_x_labels']:
batch_data['center_x_cls'],
pls['center_bin_z_labels']:
batch_data['center_z_cls'],
pls['center_x_residuals_labels']:
batch_data['center_x_res'],
pls['center_y_residuals_labels']:
batch_data['center_y_res'],
pls['center_z_residuals_labels']:
batch_data['center_z_res'],
pls['heading_bin_labels']:
batch_data['angle_cls'],
pls['heading_residuals_labels']:
batch_data['angle_res'],
pls['size_class_labels']:
batch_data['size_cls'],
pls['size_residuals_labels']:
batch_data['size_res'],
pls['gt_box_of_point']:
batch_data['gt_box_of_point'],
pls['img_seg_softmax']:
np.zeros((BATCH_SIZE, NUM_POINT, NUM_SEG_CLASSES)),
pls['is_training_pl']:
is_training,
}
if more:
summary, step, loss_val, _, logits_val, iou2ds, iou3ds, proposal_boxes, nms_indices \
= sess.run([
ops['merged'], ops['step'], ops['loss'], ops['train_op'],
ops['end_points']['foreground_logits'],
ops['end_points']['iou2ds'], ops['end_points']['iou3ds'],
ops['end_points']['proposal_boxes'], ops['end_points']['nms_indices']], feed_dict=feed_dict)
iou2ds_sum += np.sum(iou2ds)
iou3ds_sum += np.sum(iou3ds)
total_nms += len(iou2ds)
# average on each frame
proposal_recall = train_util.compute_proposal_recall(
proposal_boxes, batch_data['gt_boxes'], nms_indices)
total_proposal_recall += proposal_recall * BATCH_SIZE
else:
summary, step, loss_val, _, logits_val = sess.run(
[
ops['merged'], ops['step'], ops['loss'], ops['train_op'],
ops['end_points']['foreground_logits']
],
feed_dict=feed_dict)
train_writer.add_summary(summary, step)
# segmentation acc
preds_val = np.argmax(logits_val, 2)
correct = np.sum(preds_val == batch_data['seg_label'])
tp = np.sum(
np.logical_and(preds_val == batch_data['seg_label'],
batch_data['seg_label'] != 0))
fp = np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] == 0))
fn = np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] != 0))
total_correct += correct
total_tp += tp
total_fp += fp
total_fn += fn
total_seen += NUM_POINT * BATCH_SIZE
loss_sum += loss_val
if (batch_idx + 1) % 10 == 0:
sample_num = 10 * BATCH_SIZE
log_string(' -- %03d --' % (batch_idx + 1))
log_string('mean loss: %f' % (loss_sum / sample_num))
if total_seen > 0:
log_string('segmentation accuracy: %f' % \
(total_correct / float(total_seen)))
if total_tp + total_fn > 0 and total_tp + total_fp > 0:
log_string('segmentation recall: %f'% \
(float(total_tp)/(total_tp+total_fn)))
log_string('segmentation precision: %f'% \
(float(total_tp)/(total_tp+total_fp)))
if more:
log_string('box IoU (ground/3D): %f / %f' % \
(iou2ds_sum / float(total_nms), iou3ds_sum / float(total_nms)))
log_string('proposal recall: %f' %
(float(total_proposal_recall) / sample_num))
if np.isnan(loss_sum):
loss_endpoints = sess.run(ops['loss_endpoints'],
feed_dict=feed_dict)
print('loss_endpoints: ', loss_endpoints)
total_correct = 0
total_seen = 0
total_tp = 0
total_fp = 0
total_fn = 0
loss_sum = 0
iou2ds_sum = 0
iou3ds_sum = 0
total_nms = 0
total_proposal_recall = 0
if is_last_batch:
break
batch_idx += 1
def eval_one_epoch(sess, ops, pls, test_writer, more=False):
TEST_DATASET = Dataset(NUM_POINT,
NUM_CHANNEL,
'/data/ssd/public/jlliu/Kitti/object',
'val',
is_training=True)
test_produce_thread = Thread(target=TEST_DATASET.load, args=(False, ))
test_produce_thread.start()
global EPOCH_CNT
is_training = False
#is_training = True
log_string(str(datetime.now()))
log_string('---- EPOCH %03d EVALUATION ----' % (EPOCH_CNT))
# To collect statistics
total_correct = 0
total_seen = 0
total_tp = 0
total_fp = 0
total_fn = 0
loss_sum = 0
num_samples = 0
iou2ds_sum = 0
iou3ds_sum = 0
total_nms = 0
total_proposal_recall = 0
while (True):
batch_data, is_last_batch = TEST_DATASET.get_next_batch(BATCH_SIZE)
feed_dict = {
pls['pointclouds']:
batch_data['pointcloud'],
pls['img_inputs']:
batch_data['images'],
pls['calib']:
batch_data['calib'],
pls['seg_labels']:
batch_data['seg_label'],
pls['center_bin_x_labels']:
batch_data['center_x_cls'],
pls['center_bin_z_labels']:
batch_data['center_z_cls'],
pls['center_x_residuals_labels']:
batch_data['center_x_res'],
pls['center_y_residuals_labels']:
batch_data['center_y_res'],
pls['center_z_residuals_labels']:
batch_data['center_z_res'],
pls['heading_bin_labels']:
batch_data['angle_cls'],
pls['heading_residuals_labels']:
batch_data['angle_res'],
pls['size_class_labels']:
batch_data['size_cls'],
pls['size_residuals_labels']:
batch_data['size_res'],
pls['gt_box_of_point']:
batch_data['gt_box_of_point'],
pls['img_seg_softmax']:
np.zeros((BATCH_SIZE, NUM_POINT, NUM_SEG_CLASSES)),
pls['is_training_pl']:
is_training,
}
if more:
summary, step, loss_val, logits_val, iou2ds, iou3ds, proposal_boxes, nms_indices \
= sess.run([
ops['merged'], ops['step'], ops['loss'],
ops['end_points']['foreground_logits'],
ops['end_points']['iou2ds'], ops['end_points']['iou3ds'],
ops['end_points']['proposal_boxes'], ops['end_points']['nms_indices']], feed_dict=feed_dict)
#feed_dict=feed_dict)
iou2ds_sum += np.sum(iou2ds)
iou3ds_sum += np.sum(iou3ds)
total_nms += len(iou2ds)
# average on each frame
proposal_recall = train_util.compute_proposal_recall(
proposal_boxes, batch_data['gt_boxes'], nms_indices)
total_proposal_recall += proposal_recall * BATCH_SIZE
else:
summary, step, loss_val, logits_val = sess.run([
ops['merged'], ops['step'], ops['loss'],
ops['end_points']['foreground_logits']
],
feed_dict=feed_dict)
test_writer.add_summary(summary, step)
# segmentation acc
preds_val = np.argmax(logits_val, 2)
correct = np.sum(preds_val == batch_data['seg_label'])
tp = np.sum(
np.logical_and(preds_val == batch_data['seg_label'],
batch_data['seg_label'] != 0))
fp = np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] == 0))
fn = np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] != 0))
total_tp += tp
total_fp += fp
total_fn += fn
total_correct += correct
total_seen += NUM_POINT * BATCH_SIZE
loss_sum += loss_val
num_samples += BATCH_SIZE
if is_last_batch:
break
log_string('eval mean loss: %f' % (loss_sum / float(num_samples)))
log_string('eval segmentation accuracy: %f'% \
(total_correct / float(total_seen)))
if total_tp + total_fn > 0 and total_tp + total_fp > 0:
log_string('eval segmentation recall: %f'% \
(float(total_tp)/(total_tp+total_fn)))
log_string('eval segmentation precision: %f'% \
(float(total_tp)/(total_tp+total_fp)))
if more:
log_string('eval box IoU (ground/3D): %f / %f' % \
(iou2ds_sum / float(total_nms), iou3ds_sum / float(total_nms)))
log_string('eval proposal recall: %f' %
(float(total_proposal_recall) / num_samples))
EPOCH_CNT += 1
TEST_DATASET.stop_loading()
test_produce_thread.join()
return loss_sum / float(num_samples)
def test(split):
is_training = False
#dataset = Dataset(NUM_POINT, '/data/ssd/public/jlliu/Kitti/object', split, is_training=is_training)
dataset = Dataset(NUM_POINT, KITTI_PATH, split, is_training=True)
# data loading threads
produce_thread = Thread(target=dataset.load, args=(False, ))
produce_thread.start()
with tf.Graph().as_default():
with tf.device('/gpu:0'):
rpn_model = RPN(BATCH_SIZE,
NUM_POINT,
num_channel=4,
is_training=is_training)
pls = rpn_model.placeholders
end_points = rpn_model.end_points
box_center, box_angle, box_size = rpn_model.box_encoder.tf_decode(
end_points)
box_center = box_center + end_points['fg_points_xyz']
#box_center = tf.reshape(box_center, [BATCH_SIZE * NUM_FG_POINT,3])
#box_angle = tf.reshape(box_angle, [BATCH_SIZE * NUM_FG_POINT])
#box_size = tf.reshape(box_size, [BATCH_SIZE * NUM_FG_POINT,3])
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = tf.Session(config=config)
saver.restore(sess, FLAGS.model_path)
with tf.Graph().as_default():
with tf.device('/gpu:0'):
img_seg_net = ImgSegNet(BATCH_SIZE,
NUM_POINT,
num_channel=4,
bn_decay=None,
is_training=is_training)
seg_softmax = img_seg_net.get_seg_softmax()
#seg_net = ImgSegNet(BATCH_SIZE, NUM_POINT)
#seg_net.load_graph('./frozen_inference_graph.pb')
#seg_softmax = seg_net.get_seg_softmax()
saver1 = tf.train.Saver()
# Create another session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess1 = tf.Session(config=config)
saver1.restore(sess1, './log_img/model.ckpt')
log_string(str(datetime.now()))
# To collect statistics
total_correct = 0
total_seen = 0
num_batches = 0
tp = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
fp = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
fn = {'Car': 0, 'Pedestrian': 0, 'Cyclist': 0}
frame_ids = []
fg_indices = []
centers = []
angles = []
sizes = []
proposal_boxes = []
gt_boxes = []
nms_indices = []
scores = []
segmentation = [] # point segmentation
pc_choices = [] # point sampling indices
while (True):
batch_data, is_last_batch = dataset.get_next_batch(BATCH_SIZE,
need_id=True)
start = datetime.now()
feed_dict = {
pls['pointclouds']: batch_data['pointcloud'],
pls['img_inputs']: batch_data['images'],
pls['calib']: batch_data['calib'],
pls['seg_labels']: batch_data['seg_label'],
pls['center_bin_x_labels']: batch_data['center_x_cls'],
pls['center_bin_z_labels']: batch_data['center_z_cls'],
pls['center_x_residuals_labels']: batch_data['center_x_res'],
pls['center_y_residuals_labels']: batch_data['center_y_res'],
pls['center_z_residuals_labels']: batch_data['center_z_res'],
pls['heading_bin_labels']: batch_data['angle_cls'],
pls['heading_residuals_labels']: batch_data['angle_res'],
pls['size_class_labels']: batch_data['size_cls'],
pls['size_residuals_labels']: batch_data['size_res'],
pls['gt_box_of_point']: batch_data['gt_box_of_point'],
pls['is_training_pl']: is_training,
}
# segmentaion with image
seg_pls = img_seg_net.placeholders
img_seg_logits = sess1.run(seg_softmax,
feed_dict={
seg_pls['pointclouds']:
batch_data['pointcloud'],
seg_pls['img_inputs']:
batch_data['images'],
seg_pls['calib']:
batch_data['calib'],
seg_pls['seg_labels']:
batch_data['seg_label'],
seg_pls['is_training_pl']:
is_training
})
img_seg_logits *= np.array([0, 1]) # weights
feed_dict[pls['img_seg_softmax']] = img_seg_logits
'''
# label to one_hot
targets = batch_data['seg_label']
img_seg_logits = np.eye(NUM_SEG_CLASSES)[targets]
#img_seg_logits *= np.array([2, 2, 2, 2]) # weights
feed_dict[pls['img_seg_softmax']] = img_seg_logits
'''
logits_val, indices_val, centers_val, angles_val, sizes_val, corners_val, ind_val, scores_val \
= sess.run([
end_points['foreground_logits'], end_points['fg_point_indices'],
box_center, box_angle, box_size, end_points['proposal_boxes'],
end_points['nms_indices'], end_points['proposal_scores']], feed_dict=feed_dict)
print('inference time: ', datetime.now() - start)
# segmentation acc
preds_val = np.argmax(logits_val, 2)
num_batches += 1
for c in ['Car', 'Pedestrian', 'Cyclist']:
one_hot_class = g_type2onehotclass[c]
tp[c] += np.sum(
np.logical_and(preds_val == batch_data['seg_label'],
batch_data['seg_label'] == one_hot_class))
fp[c] += np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] != one_hot_class))
fn[c] += np.sum(
np.logical_and(preds_val != batch_data['seg_label'],
batch_data['seg_label'] == one_hot_class))
# results
for i in range(BATCH_SIZE):
frame_ids.append(batch_data['ids'][i])
segmentation.append(preds_val[i])
centers.append(centers_val[i])
angles.append(angles_val[i])
sizes.append(sizes_val[i])
proposal_boxes.append(corners_val[i])
nms_indices.append(ind_val[i])
scores.append(scores_val[i])
gt_boxes.append(batch_data['gt_boxes'][i])
pc_choices.append(batch_data['pc_choice'][i])
if is_last_batch:
#if num_batches >= 500:
break
'''
with open('rpn_out_{0}.pkl'.format(split),'wb') as fout:
pickle.dump(frame_ids, fout)
pickle.dump(segmentation, fout)
pickle.dump(centers, fout)
pickle.dump(angles, fout)
pickle.dump(sizes, fout)
pickle.dump(proposal_boxes, fout)
pickle.dump(nms_indices, fout)
pickle.dump(scores, fout)
# pickle.dump(gt_boxes, fout)
pickle.dump(pc_choices, fout)
log_string('saved prediction')
'''
dataset.stop_loading()
produce_thread.join()
'''
all_indices = np.tile(np.arange(1024), (len(proposal_boxes),))
iou2d, iou3d = compute_box3d_iou(proposal_boxes, point_gt_boxes, all_indices)
print('IOU2d: ', np.mean(iou2d))
print('IOU3d: ', np.mean(iou3d))
'''
recall = compute_proposal_recall(proposal_boxes, gt_boxes, nms_indices)
print('Average recall: ', recall)
print(tp, fp, fn)
for c in ['Car', 'Pedestrian', 'Cyclist']:
if (tp[c] + fn[c] == 0) or (tp[c] + fp[c]) == 0:
continue
print(c + ' segmentation recall: %f'% \
(float(tp[c])/(tp[c]+fn[c])))
print(c + ' segmentation precision: %f'% \
(float(tp[c])/(tp[c]+fp[c])))