def score_th_results(anns, models_res, models_labels):
ths = np.arange(0.05, 1, 0.05)
aps = [[] for _ in range(len(models_res))]
tps = [[] for _ in range(len(models_res))]
fps = [[] for _ in range(len(models_res))]
for sc in ths:
for j, m in enumerate(models_res):
for i in range(len(m)):
m[i][0] = m[i][0][m[i][0][:, 4] > sc]
mean_ap, eval_results, df_summary, recalls, precisions = eval_map(
m, anns, nproc=4, model_name="Ensemble")
aps[j].append(mean_ap)
tps[j].append(df_summary["TP"].values[0])
fps[j].append(df_summary["FP"].values[0])
def plot_score_th(res, label):
plt.figure()
for i, r in enumerate(res):
plt.plot(ths, r, label=models_labels[i])
plt.legend()
plt.xlabel("Score Threshold")
plt.ylabel(label)
plt.show()
plot_score_th(aps, "AP")
plot_score_th(tps, "TP")
plot_score_th(fps, "FP")
def voc_eval(result_file,
dataset,
iou_thr=0.5,
nproc=4,
conf_thresh=0.5,
show=False):
det_results = mmcv.load(result_file)
annotations = [dataset.get_ann_info(i) for i in range(len(dataset))]
if hasattr(dataset, 'year') and dataset.year == 2007:
dataset_name = 'voc07'
else:
dataset_name = dataset.CLASSES
mean_ap, eval_results = eval_map(det_results,
annotations,
scale_ranges=None,
iou_thr=iou_thr,
dataset=dataset_name,
logger='print',
nproc=nproc,
conf_thresh=conf_thresh,
show=show)
return mean_ap, eval_results
for model in models
]
cfg = Config.fromfile("saved_models/study/{}/{}.py".format(
models[0], models[0]))
cfg.data.test.test_mode = True # To avoid filtering out images without gts
dataset = build_dataset(cfg.data.test)
cfg = {'type': 'nms', 'iou_threshold': 0.7}
# cfg = {'type': 'soft_nms', 'iou_threshold': 0.3, 'sigma': 0.5, 'min_score': 1e-3, 'method': 'linear'}
# cfg = {'type': 'wbf', 'iou_threshold': 0.7}
combined_dets = loadResults(resFiles)
ensemble_dets = [ensembleDetections(dets, cfg) for dets in combined_dets]
# time = [n[1] for n in nms_output]
# print(np.mean(time))
# res = dataset.evaluate(ensemble_dets)
print()
for i in range(len(ensemble_dets)):
ensemble_dets[i][0] = ensemble_dets[i][0][
ensemble_dets[i][0][:, 4] > 0.05]
ensemble_dets[i][1] = ensemble_dets[i][1][
ensemble_dets[i][1][:, 4] > 0.05]
ensemble_dets[i][2] = ensemble_dets[i][2][
ensemble_dets[i][2][:, 4] > 0.05]
anns = [dataset.get_ann_info(n) for n in range(len(ensemble_dets))]
mean_ap, eval_results, df_summary, recalls, precisions = eval_map(
ensemble_dets, anns, nproc=4, model_name=models[0])
mmcv.dump(ensemble_dets, "results_sample.pkl")
def stats_results(
models,
filt_scores=[0],
training_results=False,
output='stat_res.csv',
dataset_config='configs/_base_/datasets/waymo_detection_1280x1920.py',
full_stats=False):
list_eval_results = []
resFiles = [
"saved_models/study/{}/results.pkl".format(model) for model in models
]
cfg = Config.fromfile(dataset_config)
cfg.data.test.test_mode = True # To avoid filtering out images without gts
dataset = build_dataset(cfg.data.test)
anns = [dataset.get_ann_info(n) for n in range(len(dataset.img_ids))]
df = pd.DataFrame()
for i, res in enumerate(resFiles):
dets = mmcv.load(res)
# img_index = 0
# dets = [dets[img_index]]
# anns = [anns[img_index]]
# dataset.data_infos = [dataset.data_infos[img_index]]
# dataset.img_ids = [img_index]
for sc in filt_scores:
filt_dets = dets
model_name = models[i]
if sc != 0:
filt_dets = [[
dets_class[dets_class[:, 4] > sc]
for dets_class in dets_img
] for dets_img in dets]
model_name += "_" + str(sc)
mean_ap, eval_results, df_summary = eval_map(filt_dets,
anns,
nproc=4,
model_name=model_name)
list_eval_results.append(eval_results)
df = pd.concat([df, df_summary])
if full_stats:
stats_gt_dets(eval_results, dataset, dets, anns, model_name)
# mean_ap, eval_results = eval_map(filt_dets, anns, nproc=4, tpfp_fn=tpfp_imagenet, iou_thr=0.7)
# mean_ap, eval_results = eval_map(filt_dets, anns, nproc=4, tpfp_fn=tpfp_imagenet)
# mean_ap, eval_results = eval_map(filt_dets, anns, nproc=4, iou_thr=0.7)
# mean_ap, eval_results = eval_map(filt_dets, anns, nproc=4)
#
# dataset.coco.load_anns(232)
# dataset.coco.dataset['images'] = [dataset.coco.dataset['images'][232]]
# anns = [v for k, v in dataset.coco.anns.items() if v['image_id']==232 ]
# dataset.coco.anns = {0: dataset.coco.anns[232]}
# dataset.coco.imgs = {0: dataset.coco.imgs[232]}
# # dataset.coco["annotations"] = {0: dataset.coco.anns[232]}
# dataset.evaluate(filt_dets)
if training_results:
cfg.data.train.test_mode = True
dataset_train = build_dataset(cfg.data.train)
anns_train = [
dataset_train.get_ann_info(n)
for n in range(len(dataset_train.img_ids))
]
resFiles = [
"saved_models/study/{}/results_training.pkl".format(model)
for model in models if os.path.exists(
"saved_models/study/{}/results_training.pkl".format(model))
]
for i, res in enumerate(resFiles):
dets = mmcv.load(res)
for sc in filt_scores:
filt_dets = dets
model_name = models[i] + "_train"
if sc != 0:
filt_dets = [[
dets_class[dets_class[:, 4] > sc]
for dets_class in dets_img
] for dets_img in dets]
model_name += "_" + str(sc)
mean_ap, eval_results, df_summary = eval_map(
filt_dets, anns_train, nproc=4, model_name=model_name)
list_eval_results.append(eval_results)
df = pd.concat([df, df_summary])
df.to_csv("stats_res/" + output)
return list_eval_results
def main():
print('cuda avilable:', torch.cuda.is_available())
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
gpu_num = torch.cuda.device_count()
with open('cfg.yaml', 'r') as fd:
cfg = yaml.load(fd, Loader=yaml.FullLoader)
print(cfg)
dataset = Dataset(cfg=cfg, phase='val')
valloader = data.DataLoader(dataset, batch_size=16)
category_id_to_name = dataset.category_id_to_name
category_name_to_id = dataset.category_name_to_id
data_list = dataset.data_list
# input_size = [3, *cfg['image_size']]
# num_classes = len(cfg['class_names'])
# model = ATSSModel(input_size, num_classes=num_classes)
# model_path = 'checkpoints/1027/160/model.pth'
# status = model.load_state_dict(torch.load(model_path, map_location=device))
# print(status)
checkpoint_path = 'checkpoints/1027/epoch=159.ckpt'
lit_model = LitATSS(cfg).load_from_checkpoint(checkpoint_path=checkpoint_path)
model = lit_model.model
model.to(device)
model.eval()
det_results = []
annotations = []
t2 = time.time()
for i, batch in enumerate(valloader):
input_images = batch[0].to(device)
annos = batch[1]
scale_shift = annos['transform']
t1 = time.time()
# det_result = detect_single_image(model, input_images, scale_shift[0][0], scale_shift[0][1:3])
det_result = detect_images(model, input_images, scale_shift[:, 0], scale_shift[:, 1:3])
det_results.extend(det_result)
# print(len(det_result))
print('detect_images time:', time.time() - t1)
for j in range(input_images.shape[0]):
gt_num = annos['obj_num'][j]
gt_bboxes_per_image = annos['bboxes'][j][:gt_num]
gt_bboxes_per_image[:, [0, 2]] = gt_bboxes_per_image[:, [0, 2]] - scale_shift[j][1]
gt_bboxes_per_image[:, [1, 3]] = gt_bboxes_per_image[:, [1, 3]] - scale_shift[j][2]
gt_bboxes_per_image = gt_bboxes_per_image / scale_shift[j][0]
gt_bboxes_per_image = gt_bboxes_per_image.numpy()
gt_labels = annos['cls'][j][:gt_num].numpy()
annotations.append({'bboxes': gt_bboxes_per_image, 'labels': gt_labels})
print('total detect time:', time.time() - t2)
mean_ap, eval_results = eval_map(det_results, annotations, iou_thr=0.75)