def evaluate_one_epoch():
stat_dict = {}
ap_calculator = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh,
class2type_map=DATASET_CONFIG.class2type)
ap_calculator_l = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh * 2,
class2type_map=DATASET_CONFIG.class2type)
net.eval() # set model to eval mode (for bn and dp)
for batch_idx, batch_data_label in enumerate(TEST_DATALOADER):
end_points = {}
if batch_idx % 10 == 0:
print('Eval batch: %d' % (batch_idx))
for key in batch_data_label:
batch_data_label[key] = batch_data_label[key].to(device)
# Forward pass
inputs = {'point_clouds': batch_data_label['point_clouds']}
with torch.no_grad():
end_points = net(inputs, end_points)
# Compute loss
for key in batch_data_label:
end_points[key] = batch_data_label[key]
loss, end_points = criterion(inputs, end_points, DATASET_CONFIG)
# Accumulate statistics and print out
for key in end_points:
if 'loss' in key or 'acc' in key or 'ratio' in key:
if key not in stat_dict: stat_dict[key] = 0
stat_dict[key] += end_points[key].item()
batch_pred_map_cls = parse_predictions(
end_points, CONFIG_DICT, opt_ang=(FLAGS.dataset == 'sunrgbd'))
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT)
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
batch_pred_map_cls = parse_predictions(
end_points, CONFIG_DICT_L, opt_ang=(FLAGS.dataset == 'sunrgbd'))
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT_L)
ap_calculator_l.step(batch_pred_map_cls, batch_gt_map_cls)
if FLAGS.dump_results:
dump_results(end_points, DUMP_DIR + '/result/', DATASET_CONFIG,
TEST_DATASET)
# Log statistics
for key in sorted(stat_dict.keys()):
log_string('eval mean %s: %f' % (key, stat_dict[key] /
(float(batch_idx + 1))))
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
log_string('iou = 0.25, eval %s: %f' % (key, metrics_dict[key]))
metrics_dict = ap_calculator_l.compute_metrics()
for key in metrics_dict:
log_string('iou = 0.5, eval %s: %f' % (key, metrics_dict[key]))
mean_loss = stat_dict['loss'] / float(batch_idx + 1)
return mean_loss
def evaluate_one_epoch():
stat_dict = {} # collect statistics
ap_calculator = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh,
class2type_map=DATASET_CONFIG.class2type)
net.eval() # set model to eval mode (for bn and dp)
for batch_idx, batch_data_label in enumerate(TEST_DATALOADER):
if batch_idx % 10 == 0:
print('Eval batch: %d' % (batch_idx))
for key in batch_data_label:
batch_data_label[key] = batch_data_label[key].to(device)
# Forward pass
inputs = {'point_clouds': batch_data_label['point_clouds']}
with torch.no_grad():
end_points = net(inputs)
end_points['relation_type'] = FLAGS.relation_type
end_points['relation_pair'] = FLAGS.relation_pair
# Compute loss
for key in batch_data_label:
assert (key not in end_points)
end_points[key] = batch_data_label[key]
loss, end_points = criterion(end_points, DATASET_CONFIG)
# Accumulate statistics and print out
for key in end_points:
if 'loss' in key or 'acc' in key or 'ratio' in key:
if key not in stat_dict: stat_dict[key] = 0
stat_dict[key] += end_points[key].item()
batch_pred_map_cls = parse_predictions(end_points, CONFIG_DICT)
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT)
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
# Dump evaluation results for visualization
if FLAGS.dump_results and batch_idx == 0 and EPOCH_CNT % 10 == 0:
MODEL.dump_results(end_points, DUMP_DIR, DATASET_CONFIG)
# Log statistics
TEST_VISUALIZER.log_scalars(
{key: stat_dict[key] / float(batch_idx + 1)
for key in stat_dict},
(EPOCH_CNT + 1) * len(TRAIN_DATALOADER) * BATCH_SIZE)
for key in sorted(stat_dict.keys()):
log_string('eval mean %s: %f' % (key, stat_dict[key] /
(float(batch_idx + 1))))
# Evaluate average precision
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
log_string('eval %s: %f' % (key, metrics_dict[key]))
mean_loss = stat_dict['loss'] / float(batch_idx + 1)
return mean_loss
def evaluate_one_epoch():
stat_dict = {} # collect statistics
ap_calculator = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh,
class2type_map=DATASET_CONFIG.class2type)
detector.eval() # set model to eval mode (for bn and dp)
for batch_idx, batch_data_label in enumerate(TEST_DATALOADER):
for key in batch_data_label:
batch_data_label[key] = batch_data_label[key].to(device)
# Forward pass
inputs = {'point_clouds': batch_data_label['point_clouds']}
with torch.no_grad():
end_points = detector(inputs)
# Compute loss
for key in batch_data_label:
assert (key not in end_points)
end_points[key] = batch_data_label[key]
loss, end_points = test_detector_criterion(end_points, DATASET_CONFIG)
# Accumulate statistics and print out
for key in end_points:
if 'loss' in key or 'acc' in key or 'ratio' in key:
if key not in stat_dict: stat_dict[key] = 0
stat_dict[key] += end_points[key].item()
batch_pred_map_cls = parse_predictions(end_points, CONFIG_DICT)
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT)
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
# Log statistics
TEST_VISUALIZER.log_scalars(
{key: stat_dict[key] / float(batch_idx + 1)
for key in stat_dict},
(EPOCH_CNT + 1) * len(LABELED_DATALOADER) * BATCH_SIZE)
for key in sorted(stat_dict.keys()):
log_string('eval mean %s: %f' % (key, stat_dict[key] /
(float(batch_idx + 1))))
# Evaluate average precision
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
log_string('eval %s: %f' % (key, metrics_dict[key]))
TEST_VISUALIZER.log_scalars(
{
'mAP': metrics_dict['mAP'],
'AR': metrics_dict['AR']
}, (EPOCH_CNT + 1) * len(LABELED_DATALOADER) * BATCH_SIZE)
mean_loss = stat_dict['detection_loss'] / float(batch_idx + 1)
return mean_loss
def evaluate_one_epoch():
stat_dict = {} # collect statistics
ap_calculator = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh,
class2type_map=DATASET_CONFIG.class2type)
ap_calculator_l = APCalculator(ap_iou_thresh=FLAGS.ap_iou_thresh*2,
class2type_map=DATASET_CONFIG.class2type)
net.eval() # set model to eval mode (for bn and dp)
time_file = 'time_file_%s.txt' % FLAGS.dataset
time_file = os.path.join(DUMP_DIR, time_file)
f = open(time_file, 'w')
all_time = 0
for batch_idx, batch_data_label in enumerate(TEST_DATALOADER):
if batch_idx % 10 == 0:
print('Eval batch: %d'%(batch_idx))
end_points = {}
for key in batch_data_label:
batch_data_label[key] = batch_data_label[key].to(device)
inputs = {'point_clouds': batch_data_label['point_clouds']}
tic = time.time()
with torch.no_grad():
end_points = net(inputs, end_points)
toc = time.time()
t = toc - tic
all_time += t
f.write('batch_idx:%d, infer time:%f\n' % (batch_idx, t))
print('Inference time: %f'%(t))
# Compute loss
for key in batch_data_label:
end_points[key] = batch_data_label[key]
loss, end_points = criterion(inputs, end_points, DATASET_CONFIG)
# Accumulate statistics and print out
for key in end_points:
if 'loss' in key or 'acc' in key or 'ratio' in key:
if key not in stat_dict: stat_dict[key] = 0
stat_dict[key] += end_points[key].item()
batch_pred_map_cls = parse_predictions(end_points, CONFIG_DICT, opt_ang=(FLAGS.dataset == 'sunrgbd'))
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT)
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
batch_pred_map_cls = parse_predictions(end_points, CONFIG_DICT_L, opt_ang=(FLAGS.dataset == 'sunrgbd'))
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT_L)
ap_calculator_l.step(batch_pred_map_cls, batch_gt_map_cls)
if FLAGS.dump_results:
dump_results(end_points, DUMP_DIR+'/result/', DATASET_CONFIG, TEST_DATASET)
mean_time = all_time/float(batch_idx+1)
f.write('Batch number:%d\n' % (batch_idx+1))
f.write('mean infer time: %f\n' % (mean_time))
f.close()
print('Mean inference time: %f'%(mean_time))
# Log statistics
TEST_VISUALIZER.log_scalars({key:stat_dict[key]/float(batch_idx+1) for key in stat_dict},
(EPOCH_CNT+1)*len(TRAIN_DATALOADER)*BATCH_SIZE)
for key in sorted(stat_dict.keys()):
log_string('eval mean %s: %f'%(key, stat_dict[key]/(float(batch_idx+1))))
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
log_string('eval %s: %f'%(key, metrics_dict[key]))
metrics_dict = ap_calculator_l.compute_metrics()
for key in metrics_dict:
log_string('eval %s: %f'%(key, metrics_dict[key]))
mean_loss = stat_dict['loss']/float(batch_idx+1)
return mean_loss
size_batch,
sem_class_normalized_batch,
conf_thresh,
nms_iou,
DC.num_class,
per_class_proposal=per_class_proposal,
cls_nms=cls_nms)
batch_gt_map_cls = parse_groundtruths(
center_label.astype(np.float32),
heading_class_label.astype(np.float32),
heading_residual_label.astype(np.float32),
size_class_label.astype(np.float32),
size_residual_label.astype(np.float32),
sem_cls_label.astype(np.float32), box_label_mask.astype(np.float32),
DC)
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
if i_batch % 20 == 19:
print("Done", i_batch + 1, 'of', len(dataset) + 1)
# visualize the first batch
if visualize_first_batch and i_batch == 0:
for i in range(batch_size):
# take one point cloud
points_xyz = pcd[i, :, :3]
objectness_score_normalized = objectness_score_normalized_batch[i]
center = center_batch[i]
heading = heading_batch[i]
size = size_batch[i]
sem_class_class_normalized = sem_class_normalized_batch[i]
seeds_xyz = seeds_xyz_batch[i]