def evaluate(label_path,
result_path,
save_path,
current_class=['Car', 'Pedestrian', 'Cyclist'],
coco=False,
score_thresh=-1):
class_to_name = {0: 'Car',
1: 'Pedestrian',
2: 'Cyclist',
3: 'DontCare'}
gt_annos = kitti.get_label_annos(label_path)
dt_annos = kitti.get_label_annos(result_path)
# visualize(gt_annos, dt_annos)
print(len(dt_annos))
if score_thresh > 0:
dt_annos = kitti.filter_annos_low_score(dt_annos, score_thresh)
if coco:
print(get_coco_eval_result(dt_annos, dt_annos, current_class))
else:
result_str, _ = get_official_eval_result(gt_annos, dt_annos, current_class, class_to_name)
print(result_str)
with open(save_path, 'w+') as f:
f.write("\n")
f.write(result_str)
def evaluate(label_path,
result_path,
current_class=0,
coco=False,
score_thresh=-1,
eval_dist=None):
dt_annos, image_ids = kitti.get_label_annos(result_path,
return_image_ids=True,
eval_dist=eval_dist)
print('Eval {} images'.format(len(dt_annos)))
if score_thresh > 0:
dt_annos = kitti.filter_annos_low_score(dt_annos, score_thresh)
#val_image_ids = _read_imageset_file(label_split_file)
gt_annos = kitti.get_label_annos(label_path,
image_ids,
eval_dist=eval_dist)
# Playground
# print(gt_annos[48].keys())
#print(gt_annos[48]["name"])
#print(gt_annos[48]["dimensions"])
#print(gt_annos[48]["location"])
## Playground end
if coco:
print(get_coco_eval_result(gt_annos, dt_annos, current_class))
else:
print(get_official_eval_result(gt_annos, dt_annos, current_class))
def evaluate(label_path,
result_path,
label_split_file,
current_class=0,
coco=False,
score_thresh=-1):
dt_annos = get_label_annos(result_path)
if score_thresh > 0:
dt_annos = filter_annos_low_score(dt_annos, score_thresh)
val_image_ids = _read_imageset_file(label_split_file)
gt_annos = get_label_annos(label_path, val_image_ids)
if coco:
return get_coco_eval_result(gt_annos, dt_annos, current_class)
else:
return get_official_eval_result(gt_annos, dt_annos, current_class)
def evaluate(label_path,
result_path,
current_class=0,
coco=False,
score_thresh=-1,
eval_dist=None):
dt_annos, image_ids = kitti.get_label_annos(result_path, return_image_ids=True, eval_dist=eval_dist)
print('Eval {} images'.format(len(dt_annos)))
if score_thresh > 0:
dt_annos = kitti.filter_annos_low_score(dt_annos, score_thresh)
#val_image_ids = _read_imageset_file(label_split_file)
gt_annos = kitti.get_label_annos(label_path, image_ids, eval_dist=eval_dist)
if coco:
print(get_coco_eval_result(gt_annos, dt_annos, current_class))
else:
print(get_official_eval_result(gt_annos, dt_annos, current_class))
def evaluate(label_path,
result_path,
label_split_file,
current_class=0,
coco=False,
score_thresh=-1):
dt_annos = kitti.get_label_annos(result_path)
if score_thresh > 0:
dt_annos = kitti.filter_annos_low_score(dt_annos, score_thresh)
val_image_ids = _read_imageset_file(label_split_file)
val_image_ids = exists(val_image_ids, result_path)
gt_annos = kitti.get_label_annos(label_path, val_image_ids)
dt_annos = kitti.get_label_annos(result_path, val_image_ids)
if coco:
print(get_coco_eval_result(gt_annos, dt_annos, current_class))
else:
print(get_official_eval_result(gt_annos, dt_annos, current_class))
def evaluate(result_path,
dataset_path=None,
label_split_file=None,
label_path=None,
metric="new",
dataset="kitti",
current_class=0,
coco=False,
score_thresh=-1,
toground=False,
rescale_pred=None,
align_size=False,
align_front=False,
reverse_align=False,
dense_sample=False,
direct_save=False,
output_iou=False,
adapted=False):
label_split_file = label_split_file or os.path.join(
dataset_path, "val.txt")
label_path = label_path or os.path.join(dataset_path, "training",
"label_2")
if metric == "old":
from eval_old import get_official_eval_result, get_coco_eval_result, calculate_iou_partly
else:
from eval2 import get_official_eval_result, get_coco_eval_result, calculate_iou_partly
val_image_ids = _read_imageset_file(label_split_file)
dt_annos = kitti.get_label_annos(result_path, val_image_ids)
# for i in range(len(dt_annos)):
# if len(dt_annos[i]['name']) > 0:
# assert np.max(dt_annos[i]['location'][:, 2]) < 80, f"{os.path.join(result_path, '%06d.txt' % val_image_ids[i])}, Some detection > 80m!!!"
if score_thresh > 0:
dt_annos = kitti.filter_annos_low_score(dt_annos, score_thresh)
if toground:
dt_annos = annos_to_ground(
dt_annos, os.path.join(os.path.dirname(label_path), "planes"),
val_image_ids)
save_labels(dt_annos,
os.path.join(os.path.dirname(result_path), "grounded"),
val_image_ids)
if rescale_pred is not None:
for anno in dt_annos:
anno['dimensions'] *= rescale_pred
gt_annos = kitti.get_label_annos(label_path, val_image_ids)
# for i in range(len(gt_annos)):
# if len(gt_annos[i]['name']) > 0:
# assert np.max(gt_annos[i]['location'][:, 2]) < 70, f"{os.path.join(label_path, '%06d.txt' % val_image_ids[i])}, Some label > 70m!!!"
if output_iou:
target_dir = os.path.join(os.path.dirname(result_path), "with_iou")
os.makedirs(target_dir, exist_ok=True)
overlaps, _, _, _ = calculate_iou_partly(dt_annos, gt_annos, 1)
assert len(overlaps) == len(dt_annos) == len(gt_annos)
for i in range(len(overlaps)):
assert overlaps[i].shape == (len(dt_annos[i]['name']),
len(gt_annos[i]['name']))
if len(dt_annos[i]['name']) > 0 and len(gt_annos[i]['name']) > 0:
val = np.max(overlaps[i], axis=1)
else:
val = np.zeros(len(dt_annos[i]['name']))
try:
n = len(dt_annos[i]["name"])
kitti_str = []
for j in range(n):
kitti_str.append(
'%s %.2f %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f'
%
(dt_annos[i]['name'][j], dt_annos[i]['truncated'][j],
dt_annos[i]['occluded'][j], dt_annos[i]['alpha'][j],
dt_annos[i]['bbox'][j, 0], dt_annos[i]['bbox'][j, 1],
dt_annos[i]['bbox'][j, 2], dt_annos[i]['bbox'][j, 3],
dt_annos[i]['dimensions'][j, 1],
dt_annos[i]['dimensions'][j, 2],
dt_annos[i]['dimensions'][j, 0],
dt_annos[i]['location'][j, 0],
dt_annos[i]['location'][j, 1],
dt_annos[i]['location'][j, 2],
dt_annos[i]['rotation_y'][j], dt_annos[i]['score'][j],
val[j]))
with open(
os.path.join(target_dir,
"%06d.txt" % val_image_ids[i]), "w") as f:
f.write("\n".join(kitti_str))
except:
pdb.set_trace()
target_dir = os.path.join(os.path.dirname(result_path), "with_iou_gt")
os.makedirs(target_dir, exist_ok=True)
for i in range(len(overlaps)):
assert overlaps[i].shape == (len(dt_annos[i]['name']),
len(gt_annos[i]['name']))
if len(dt_annos[i]['name']) > 0 and len(gt_annos[i]['name']) > 0:
val = np.max(overlaps[i], axis=0)
else:
val = np.zeros(len(gt_annos[i]['name']))
try:
n = len(gt_annos[i]["name"])
kitti_str = []
for j in range(n):
kitti_str.append(
'%s %.2f %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f'
%
(gt_annos[i]['name'][j], gt_annos[i]['truncated'][j],
gt_annos[i]['occluded'][j], gt_annos[i]['alpha'][j],
gt_annos[i]['bbox'][j, 0], gt_annos[i]['bbox'][j, 1],
gt_annos[i]['bbox'][j, 2], gt_annos[i]['bbox'][j, 3],
gt_annos[i]['dimensions'][j, 1],
gt_annos[i]['dimensions'][j, 2],
gt_annos[i]['dimensions'][j, 0],
gt_annos[i]['location'][j, 0],
gt_annos[i]['location'][j, 1],
gt_annos[i]['location'][j, 2],
gt_annos[i]['rotation_y'][j], gt_annos[i]['score'][j],
val[j]))
with open(
os.path.join(target_dir,
"%06d.txt" % val_image_ids[i]), "w") as f:
f.write("\n".join(kitti_str))
except:
pdb.set_trace()
if align_size:
overlaps, _, _, _ = calculate_iou_partly(dt_annos, gt_annos, 1)
assert len(overlaps) == len(dt_annos) == len(gt_annos)
for i in range(len(overlaps)):
assert overlaps[i].shape == (len(dt_annos[i]['name']),
len(gt_annos[i]['name']))
if len(dt_annos[i]['name']) > 0 and len(gt_annos[i]['name']) > 0:
val = np.max(overlaps[i], axis=1)
idx = np.argmax(overlaps[i], axis=1)
for j in range(len(dt_annos[i]['name'])):
if val[j] > 0.2:
dt_annos[i]['dimensions'][
j, :] = gt_annos[i]['dimensions'][idx[j], :]
save_labels(dt_annos,
os.path.join(os.path.dirname(result_path), "align_size"),
val_image_ids)
if align_front:
overlaps, _, _, _ = calculate_iou_partly(dt_annos, gt_annos, 1)
assert len(overlaps) == len(dt_annos) == len(gt_annos)
for i in range(len(overlaps)):
assert overlaps[i].shape == (len(dt_annos[i]['name']),
len(gt_annos[i]['name']))
if len(dt_annos[i]['name']) > 0 and len(gt_annos[i]['name']) > 0:
val = np.max(overlaps[i], axis=1)
idx = np.argmax(overlaps[i], axis=1)
for j in range(len(dt_annos[i]['name'])):
if val[j] > 0.2:
dist = np.linalg.norm(dt_annos[i]['location'][j, :])
alpha = dt_annos[i]['alpha'][j]
alpha = np.arctan2(np.sin(alpha), np.cos(alpha))
if np.abs(np.sin(alpha)) * dist > dt_annos[i][
'dimensions'][j, 2] / 2.0:
shift = (
dt_annos[i]['dimensions'][j, 2] -
gt_annos[i]['dimensions'][idx[j], 2]) / 2.0
if 0 < alpha:
angle = -dt_annos[i]['rotation_y'][j]
else:
angle = -dt_annos[i]['rotation_y'][j] + np.pi
dt_annos[i]['location'][j,
0] += shift * np.cos(angle)
dt_annos[i]['location'][j,
2] += shift * np.sin(angle)
if np.abs(np.cos(alpha)) * dist > dt_annos[i][
'dimensions'][j, 1] / 2.0:
shift = (
dt_annos[i]['dimensions'][j, 1] -
gt_annos[i]['dimensions'][idx[j], 1]) / 2.0
if -np.pi / 2.0 < alpha < np.pi / 2.0:
angle = -dt_annos[i]['rotation_y'][
j] - np.pi / 2.0
else:
angle = -dt_annos[i]['rotation_y'][
j] + np.pi / 2.0
dt_annos[i]['location'][j,
0] += shift * np.cos(angle)
dt_annos[i]['location'][j,
2] += shift * np.sin(angle)
dt_annos[i]['dimensions'][
j, :] = gt_annos[i]['dimensions'][idx[j], :]
save_labels(dt_annos,
os.path.join(os.path.dirname(result_path), "align_front"),
val_image_ids)
if reverse_align:
import sys
sys.path.insert(0, "..")
from config_path import dataset_paths
src = get_model(label_path)
dst = get_model(result_path)
print("label_path:", label_path)
print("result_path:", result_path)
print(f"{src} -> {dst}")
with open(os.path.join(dataset_paths[src],
"label_normal_val.json")) as f:
src = json.load(f)
with open(os.path.join(dataset_paths[dst],
"label_normal_val.json")) as f:
dst = json.load(f)
mapping = get_scale_map(src, dst)
for i in range(len(gt_annos)):
if len(gt_annos[i]['name']) > 0:
gt_annos[i]["dimensions"] = mapping(gt_annos[i]["dimensions"])
save_labels(
gt_annos,
os.path.join(os.path.dirname(result_path), "reverse_align"),
val_image_ids)
if not output_iou:
if coco:
return get_coco_eval_result(gt_annos, dt_annos, current_class)
else:
ap_result_str, ap_dict = get_official_eval_result(
gt_annos,
dt_annos,
current_class,
dataset,
dense_sample=dense_sample)
if direct_save:
result_path = os.path.dirname(result_path)
fname = os.path.basename(result_path) + "_val20"
if toground:
fname += "_ground"
if align_size:
fname += "_align_size"
if reverse_align:
fname += "_reverse_align"
if adapted:
fname += "_adapted"
print(
f"Saving to {os.path.join(os.path.dirname(result_path), fname+'.pkl')}"
)
with open(
os.path.join(os.path.dirname(result_path),
fname + '.pkl'), "wb") as fb:
pickle.dump(ap_dict["result"], fb)
with open(
os.path.join(os.path.dirname(result_path),
fname + '.txt'), "w") as f:
f.write(ap_result_str)
return ap_result_str, ap_dict