def test_net_on_dataset(
weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0
):
"""Run inference on a dataset."""
dataset = JsonDataset(dataset_name)
test_timer = Timer()
test_timer.tic()
if multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps, all_bodys = \
multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file,
num_images, output_dir
)
else:
all_boxes, all_segms, all_keyps, all_bodys = test_net(
weights_file, dataset_name, proposal_file, output_dir, gpu_id=gpu_id
)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(test_timer.average_time))
results = task_evaluation.evaluate_all(
dataset, all_boxes, all_segms, all_keyps, all_bodys, output_dir
)
return results
def test_net_on_dataset(
weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0
):
"""Run inference on a dataset."""
dataset = JsonDataset(dataset_name)
test_timer = Timer()
test_timer.tic()
if multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps = multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file, num_images, output_dir
)
else:
all_boxes, all_segms, all_keyps = test_net(
weights_file, dataset_name, proposal_file, output_dir, gpu_id=gpu_id
)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(test_timer.average_time))
results = task_evaluation.evaluate_all(
dataset, all_boxes, all_segms, all_keyps, output_dir
)
return results
def do_reval(dataset_name, output_dir, args):
dataset = JsonDataset(dataset_name)
dets = load_object(os.path.join(output_dir, 'detections.pkl'))
# Override config with the one saved in the detections file
if args.cfg_file is not None:
core_config.merge_cfg_from_cfg(core_config.load_cfg(dets['cfg']))
else:
core_config._merge_a_into_b(core_config.load_cfg(dets['cfg']), cfg)
# re-filter on score threshold:
dets['all_boxes'] = \
[
[
im[im[:,4] > cfg.TEST.SCORE_THRESH,:] if len(im) != 0 else []
for im in cls
]
for cls in dets['all_boxes']
]
results = task_evaluation.evaluate_all(dataset,
dets['all_boxes'],
dets['all_segms'],
dets['all_keyps'],
output_dir,
use_matlab=args.matlab_eval)
task_evaluation.log_copy_paste_friendly_results(results)
def test_net_on_dataset(weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0):
"""Run inference on a dataset."""
dataset = JsonDataset(dataset_name)
test_timer = Timer()
test_timer.tic()
model = ''
if multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps = multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file, num_images, output_dir)
else:
all_boxes, all_segms, all_keyps, model = test_net(weights_file,
dataset_name,
proposal_file,
output_dir,
gpu_id=gpu_id)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(
test_timer.average_time))
results = task_evaluation.evaluate_all(dataset, all_boxes, all_segms,
all_keyps, output_dir)
roc_data = metrics.calculate_roc(all_boxes, dataset, cfg.TEST.IOU)
froc_data = metrics.calculate_froc(all_boxes, dataset, cfg.TEST.IOU)
auc_score = {
dataset.name: {
u'box': {
u'AUC': auc(roc_data[0], roc_data[1])
}
}
}
afroc_score = np.trapz(froc_data[0], froc_data[2])
afroc = {dataset.name: {u'box': {u'AFROC': afroc_score}}}
print('Afroc score: {:.4f}'.format(afroc_score))
plot.plot_roc(roc_data, auc_score[dataset.name][u'box'][u'AUC'], dataset,
model, output_dir)
plot.plot_froc(froc_data, dataset, model, output_dir)
plot.plot_afroc(froc_data, dataset, model, output_dir)
save.np_save(np.stack(roc_data), 'roc', dataset, model, output_dir)
save.np_save(np.stack(froc_data), 'froc', dataset, model, output_dir)
results[dataset_name][u'box'].update(auc_score[dataset.name][u'box'])
results[dataset_name][u'box'].update(afroc[dataset.name][u'box'])
return results, auc_score, afroc_score
def test_net_on_dataset(weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0):
"""Run inference on a dataset."""
dataset = JsonDataset(dataset_name)
test_timer = Timer()
test_timer.tic()
################################################################
import pickle
res_file = os.path.join(output_dir,
'bbox_' + dataset_name + '_results.json')
print("res_file = {}==========================".format(res_file))
if os.path.exists(res_file):
import detectron.datasets.json_dataset_evaluator as json_dataset_evaluator
print("res_file = {} exists! Loading res_file".format(res_file))
coco_eval = json_dataset_evaluator._do_detection_eval(
dataset, res_file, output_dir)
box_results = task_evaluation._coco_eval_to_box_results(coco_eval)
results = OrderedDict([(dataset.name, box_results)])
return results
################################################################
det_name = "detections.pkl"
det_file = os.path.join(output_dir, det_name)
print("det_file = {}==========================".format(det_file))
if os.path.exists(det_file):
print("{} exists! Loading detection results".format(det_file))
res = pickle.load(open(det_file))
all_boxes = res['all_boxes']
all_segms = res['all_segms']
all_keyps = res['all_keyps']
################################################################
elif multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps = multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file, num_images, output_dir)
else:
all_boxes, all_segms, all_keyps = test_net(weights_file,
dataset_name,
proposal_file,
output_dir,
gpu_id=gpu_id)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(
test_timer.average_time))
results = task_evaluation.evaluate_all(dataset, all_boxes, all_segms,
all_keyps, output_dir)
return results
def test_net_on_dataset(weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0,
subset_pointer=None):
"""Run inference on a dataset."""
if dataset_name[:5] != 'live_':
dataset = JsonDataset(dataset_name)
test_timer = Timer()
test_timer.tic()
if multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps = multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file, num_images, output_dir)
else:
all_boxes, all_segms, all_keyps = test_net(
weights_file,
dataset_name,
proposal_file,
output_dir,
gpu_id=gpu_id,
subset_pointer=subset_pointer)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(
test_timer.average_time))
if cfg.TEST.COCO_TO_VOC:
all_boxes = coco_detects_to_voc(all_boxes)
if dataset_name[:5] == 'live_':
return None
results = task_evaluation.evaluate_all(dataset,
all_boxes,
all_segms,
all_keyps,
output_dir,
subset_pointer=subset_pointer)
if subset_pointer is not None:
# prune the subset for the following datasets:
subset_pointer.subset = subset_pointer.subset[len(dataset.get_roidb()
):]
print('remains', len(subset_pointer.subset)
) # should have 0 remains for the last set, voc_2012_train.
return results
def do_reval(dataset_name, output_dir, args):
dataset = JsonDataset(dataset_name)
dets = load_object(os.path.join(output_dir, 'detections.pkl'))
# Override config with the one saved in the detections file
if args.cfg_file is not None:
core_config.merge_cfg_from_cfg(core_config.load_cfg(dets['cfg']))
else:
core_config._merge_a_into_b(core_config.load_cfg(dets['cfg']), cfg)
results = task_evaluation.evaluate_all(dataset,
dets['all_boxes'],
dets['all_segms'],
dets['all_keyps'],
output_dir,
use_matlab=args.matlab_eval)
task_evaluation.log_copy_paste_friendly_results(results)
def do_reval(dataset_name, output_dir, args):
dataset = JsonDataset(dataset_name)
dets = load_object(os.path.join(output_dir, 'detections.pkl'))
# Override config with the one saved in the detections file
if args.cfg_file is not None:
core_config.merge_cfg_from_cfg(core_config.load_cfg(dets['cfg']))
else:
core_config._merge_a_into_b(core_config.load_cfg(dets['cfg']), cfg)
results = task_evaluation.evaluate_all(
dataset,
dets['all_boxes'],
dets['all_segms'],
dets['all_keyps'],
output_dir,
use_matlab=args.matlab_eval
)
task_evaluation.log_copy_paste_friendly_results(results)
def do_reval(dataset_name, output_dir, args):
dataset = JsonDataset(dataset_name)
with open(os.path.join(output_dir, 'detections.pkl'), 'rb') as f:
dets = pickle.load(f)
# Override config with the one saved in the detections file
if args.cfg_file is not None:
# bug: loads only already stored cfg
# core_config.merge_cfg_from_cfg(core_config.load_cfg(dets['cfg']))
# merge config from passed config file!!
core_config.merge_cfg_from_file(args.cfg_file)
else:
core_config._merge_a_into_b(core_config.load_cfg(dets['cfg']), cfg)
results = task_evaluation.evaluate_all(dataset,
dets['all_boxes'],
dets['all_segms'],
dets['all_keyps'],
output_dir,
use_matlab=args.matlab_eval)
task_evaluation.log_copy_paste_friendly_results(results)
def test_net_on_dataset(weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=False,
gpu_id=0):
load_from_tmp = False
"""Run inference on a dataset."""
dataset = JsonDataset(dataset_name)
if not load_from_tmp:
test_timer = Timer()
test_timer.tic()
if multi_gpu:
num_images = len(dataset.get_roidb())
all_boxes, all_segms, all_keyps, all_personmasks, all_parss, all_bodys = \
multi_gpu_test_net_on_dataset(
weights_file, dataset_name, proposal_file,
num_images, output_dir
)
else:
all_boxes, all_segms, all_keyps, all_personmasks, all_parss, all_bodys = test_net(
weights_file,
dataset_name,
proposal_file,
output_dir,
gpu_id=gpu_id)
test_timer.toc()
logger.info('Total inference time: {:.3f}s'.format(
test_timer.average_time))
else:
tmp_path = '/coco/results/detectron-output_mulres_intersup_mulsaclesup_lowfeat23_int05/test/dense_coco_2014_minival/generalized_rcnn/detections.pkl'
#tmp_path = '/coco/results/detectron-output_mulres_intersup/test/dense_coco_2014_minival/generalized_rcnn/detections.pkl'
#tmp_path = '/coco/results/detectron-output_mulres_intersup_onlysegpart_fliped/test/MHP_seg_val/generalized_rcnn/detections.pkl'
#tmp_path = '/coco/results/detectron-output_mulres_intersup_onlysegpart_fliped/test/CIHP_seg_val/generalized_rcnn/detections.pkl'
tmp_file = open(tmp_path, 'r')
print('detections results from: ', tmp_path)
tmp_pkl = pickle.load(tmp_file)
all_boxes = tmp_pkl['all_boxes']
all_segms = tmp_pkl['all_segms']
all_keyps = tmp_pkl['all_keyps']
if 'all_personmasks' not in tmp_pkl.keys():
all_personmasks = None
else:
all_personmasks = tmp_pkl['all_personmasks']
all_parss = tmp_pkl['all_parss']
all_bodys = tmp_pkl['all_bodys']
'''
for i in range(len(all_bodys[1])):
for j in range(len(all_bodys[1][i])):
#print("all_bodys[1][i][j]: ",all_bodys[1][i][j].shape)
all_bodys[1][i][j][0][np.where(all_bodys[1][i][j][0] == 3)] = 4
all_bodys[1][i][j][0][np.where(all_bodys[1][i][j][0] == 4)] = 3
all_bodys[1][i][j][0][np.where(all_bodys[1][i][j][0] == 6)] = 5
all_bodys[1][i][j][0][np.where(all_bodys[1][i][j][0] == 5)] = 6
'''
if cfg.VIS:
vis_wholedataset(
dataset_name,
proposal_file,
output_dir,
all_boxes=all_boxes,
all_segms=all_segms,
all_keyps=all_keyps,
all_personmasks=all_personmasks,
all_parss=all_parss,
all_bodys=all_bodys,
img_name=['COCO_val2014_000000464089.jpg'],
show_box=False,
)
results = task_evaluation.evaluate_all(dataset, all_boxes, all_segms,
all_keyps, all_personmasks,
all_parss, all_bodys, output_dir)
return results
def grid_search():
dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)
roidb, dataset, start_ind, end_ind, total_num_images = get_roidb_and_dataset(
dataset_name, proposal_file, None
)
num_images = len(roidb)
num_classes = cfg.MODEL.NUM_CLASSES
subinds = np.array_split(range(num_images), cfg.NUM_GPUS)
tag = 'detection'
output_dir = get_output_dir(cfg.TEST.DATASETS, training=False)
det_file = os.path.join(output_dir, 'detections.pkl')
outputs = load_object(det_file)
print(len(outputs))
all_dets_cache = outputs['all_boxes']
print(len(all_dets_cache))
all_boxes_cache = []
all_scores_cache = []
for i, entry in enumerate(roidb):
print(i)
max_det = all_dets_cache[1][i].shape[0]
print(max_det, num_classes)
boxes = np.zeros((max_det, 4), dtype=np.float32)
scores = np.zeros((max_det, num_classes), dtype=np.float32)
boxes[:] = -1
scores[:] = -1
for j in range(num_classes):
if len(all_dets_cache[j]) > 0:
pass
else:
continue
scores[:, j] = all_dets_cache[j][i][:, 4]
boxes[:, 0:4] = all_dets_cache[1][i][:, :4]
boxes = np.tile(boxes, (1, scores.shape[1]))
print(scores.shape, boxes.shape)
all_boxes_cache.append(boxes)
all_scores_cache.append(scores)
timers = defaultdict(Timer)
resultss = []
nmses = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.0]
threshs = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
max_per_images = [10000, 1000, 100, 10, 1]
for nms in nmses:
for thresh in threshs:
for max_per_image in max_per_images:
print("----------------------------------------------------")
print('NUM: ', nms, ' Thresh: ', thresh, ' MAX_PER_IM: ', max_per_image)
cfg.immutable(False)
cfg.TEST.NMS = nms
cfg.TEST.SCORE_THRESH = thresh
cfg.TEST.DETECTIONS_PER_IM = max_per_image
cfg.immutable(True)
all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images)
for i, entry in enumerate(roidb):
# print(i)
timers['im_detect_bbox'].tic()
scores = all_scores_cache[i]
boxes = all_boxes_cache[i]
# print(scores.shape, boxes.shape)
timers['im_detect_bbox'].toc()
timers['misc_bbox'].tic()
scores, boxes, cls_boxes_i = box_results_with_nms_and_limit(scores, boxes)
timers['misc_bbox'].toc()
extend_results(i, all_boxes, cls_boxes_i)
results = task_evaluation.evaluate_all(
dataset, all_boxes, all_segms, all_keyps, output_dir
)
print(results)
print(resultss)
f = open('grid_search.csv', 'wb')
wr = csv.writer(f, dialect='excel')
wr.writerows(resultss)
