def validation_epoch_end(self, outputs: List[Tuple[List[MetadataDict], List[MetadataDict]]]) -> None:
predictions, targets = self._restructure_validation_outputs(outputs)
mAP, ap_per_class = mean_average_precision_for_boxes(
targets,
predictions,
iou_threshold=self.conf.evaluation.iou_threshold,
verbose=False
)
# TODO: Add DDP support
self.log('mAP', mAP, prog_bar=True)
self.log_ap_per_class(ap_per_class)
def zfturbo_compute_mAP(normalized_gt_df,
normalized_pred_df,
id_to_label,
gt_class_id="class_id",
pred_class_id="label",
verbose=False):
"""
:gt: is ground truth dataframe. The co-ordinates are normalized based on image height and
width using normalize_bb()
:pred: is a prediction dataframe. The co-ordinates are normalized based on image height and
width using normalize_bb()
:id_to_label: maps the ids in gt and pred to string labels
:return: tuple, where first value is mAP and second values is dict with AP for each class.
"""
# create copies
normalized_gt_df_copy = normalized_gt_df.copy(deep=True)
normalized_pred_df_copy = normalized_pred_df.copy(deep=True)
# replace numeric ids to string labels
# https://stackoverflow.com/questions/22100130/pandas-replace-multiple-values-one-column
normalized_gt_df_copy[gt_class_id] = normalized_gt_df_copy[
gt_class_id].astype(int).map(id_to_label)
normalized_pred_df_copy[pred_class_id] = normalized_pred_df_copy[
pred_class_id].astype(int).map(id_to_label)
# convert dataframe to numpy array format as required by package
normalized_gt_df_np = normalized_gt_df_copy[[
"image_id", gt_class_id, "x_min", "x_max", "y_min", "y_max"
]].values
normalized_pred_df_np = normalized_pred_df_copy[[
"image_id", pred_class_id, "confidence_score", "x_min", "x_max",
"y_min", "y_max"
]].values
# compute mAP
mean_ap, average_precisions = mean_average_precision_for_boxes(
normalized_gt_df_np,
normalized_pred_df_np,
iou_threshold=0.4,
exclude_not_in_annotations=True,
verbose=verbose)
return mean_ap, sorted(average_precisions.items(),
key=operator.itemgetter(1),
reverse=True)
def validate_v1(annotations, predictions, verbose=True):
thr_value = 0.4
num_classes = 15
if type(annotations) is str:
ann = pd.read_csv(annotations)
else:
ann = annotations
if type(predictions) is str:
pr = pd.read_csv(predictions)
else:
pr = predictions
# Fix different IDs
unique_preds = pr['image_id'].unique()
if verbose:
print('Prediction Ids: {}'.format(len(unique_preds)))
unique_ann = ann['id'].unique()
if verbose:
print('Annotation Ids: {}'.format(len(unique_ann)))
ann = ann[ann['id'].isin(unique_preds)]
unique_ann = ann['id'].unique()
if verbose:
print('Reduced annotation Ids: {}'.format(len(unique_ann)))
ann = prepare_annotations(ann)
pr = prepare_predictions(pr)
mean_ap, average_precisions = mean_average_precision_for_boxes(
ann, pr, iou_threshold=thr_value, verbose=verbose)
map = np.zeros(num_classes, dtype=np.float32)
for i in range(num_classes):
try:
if verbose:
print('Class: {:2d} Entries: {:5d} AP: {:.6f}'.format(
i, int(average_precisions[str(i)][1]),
average_precisions[str(i)][0]))
map[i] = average_precisions[str(i)][0]
except Exception as e:
if verbose:
print('No class found: {}'.format(i))
map[i] = 0
map_no_last_class = map[:-1].mean()
if verbose:
print('mAP value: {:.6f}'.format(mean_ap))
print('mAP value no last class: {:.6f}'.format(map_no_last_class))
return map_no_last_class
def get_mean_average_precision(
annotation_path,
predictions_path,
iou_threshold=0.4,
meta_path='/home/semyon/data/VinBigData/train_meta.csv',
verbose=False):
'''
param: annotation_path: path to .csv with columns ['image_id', 'class_name', 'x_min', 'x_max', 'y_min', 'y_max']
param: predictions_path: path to .csv with columns ['image_id', 'class_name', 'rad_id', 'x_min', 'x_max', 'y_min', 'y_max'], where 'rad_id' contains confidence
'''
if isinstance(annotation_path, pd.DataFrame) and isinstance(
predictions_path, pd.DataFrame):
ann_df = annotation_path.copy()
pred_df = predictions_path.copy()
else:
ann_df = pd.read_csv(annotation_path)
pred_df = pd.read_csv(predictions_path)
meta_df = pd.read_csv(meta_path).set_index('image_id')
# inplace norm coordinates
norm_coordinates(ann_df, meta_df)
norm_coordinates(pred_df, meta_df)
# annotations
new_cols = ['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']
old_cols = ['image_id', 'class_name', 'x_min', 'x_max', 'y_min', 'y_max']
for new_col_name, old_col_name in zip(new_cols, old_cols):
ann_df[new_col_name] = ann_df[old_col_name]
# predictions
new_cols = ['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']
old_cols = [
'image_id', 'class_name', 'rad_id', 'x_min', 'x_max', 'y_min', 'y_max'
]
for new_col_name, old_col_name in zip(new_cols, old_cols):
pred_df[new_col_name] = pred_df[old_col_name]
ann = ann_df[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin',
'YMax']].values
pred = pred_df[[
'ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax'
]].values
mean_ap, average_precisions = mean_average_precision_for_boxes(
ann, pred, iou_threshold=iou_threshold, verbose=verbose)
return mean_ap, average_precisions
def evaluate(*pred_df, outdir=evaldir, filtered=False, ensemble=False):
ann = pd.read_csv(outdir / 'annotations_df.csv')
if filtered:
det = pd.read_csv(outdir / 'pred_df_filtered.csv')
elif ensemble:
preds = []
for i, pred in enumerate(pred_df):
preds.append(pred)
ensemble_models(preds[0], preds[1], path=evaldir, evaluation=True)
det = pd.read_csv(outdir / 'pred_df_ensembled.csv')
else:
det = pd.read_csv(outdir / 'pred_df.csv')
ann = ann[['image_id', 'class_name', 'x_min', 'x_max', 'y_min',
'y_max']].values
det = det[[
'image_id', 'class_name', 'score', 'x_min', 'x_max', 'y_min', 'y_max'
]].values
mean_ap, average_precisions = mean_average_precision_for_boxes(
ann, det, iou_threshold=0.4)
1. 加上左上点坐标还原到原图的坐标
2. 再加一次极大值抑制
3. 连接相邻的boxes
"""
# 保存为csv格式
# ImageID,LabelName,XMin,XMax,YMin,YMax
with open('./model/%s/dev_%d_label.csv'%(save_f, global_steps),'w',encoding='utf-8') as f:
f.write('ImageID,LabelName,XMin,YMin,XMax,YMax\n')
for key in raw_label:
line = raw_label[key]
x,y = n2s[key][:2]
for _ in line:
f.write('%s,%s,%f,%f,%f,%f\n'%(key, i2n[int(_[-1])], _[0]/y, _[1]/x, _[2]/y, _[3]/x))
# ImageID,LabelName,Conf,XMin,XMax,YMin,YMax
with open('./model/%s/dev_%d.csv'%(save_f, global_steps),'w',encoding='utf-8') as f:
f.write('ImageID,LabelName,Conf,XMin,YMin,XMax,YMax\n')
for key in result:
line = result[key]
x,y = n2s[key][:2]
for _ in line:
f.write('%s,%s,%f,%f,%f,%f,%f\n'%(key, i2n[int(_[-1])],_[-2], _[0]/y, _[1]/x, _[2]/y, _[3]/x))
ann = pd.read_csv('./model/%s/dev_%d_label.csv'%(save_f, global_steps))
det = pd.read_csv('./model/%s/dev_%d.csv'%(save_f, global_steps))
ann = ann[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values
det = det[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values
mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det, iou_threshold=0.5)
print('')
save_prefix += '_' + str(params[el])
save_prefix += '.csv'
save_path1 = "{}/{}".format(params['out_folder'], save_prefix)
if os.path.isfile(save_path1):
print('File already exists: {}. Skip'.format(save_path1))
continue
ensemble_preds.to_csv(save_path1, index=False)
ensemble_preds = ensemble_preds[[
'img_id', 'label', 'score', 'x1', 'x2', 'y1', 'y2'
]].values
ann_path = 'E:/Projects_M2/2019_06_Google_Open_Images/input/COCO2017/annotations/instances_val2017.csv'
ann = pd.read_csv(ann_path, dtype={'img_id': np.str, 'label': np.str})
ann_numpy = ann[['img_id', 'label', 'x1', 'x2', 'y1', 'y2']].values
mean_ap, average_precisions = mean_average_precision_for_boxes(
ann_numpy, ensemble_preds, iou_threshold=0.5, verbose=False)
coco_preds, detections = convert_csv_predictions_to_coco(
"{}/{}".format(params['out_folder'], save_prefix))
print("Ensemble [{}] Weights: {} Params: {} mAP: {:.6f}".format(
len(weights), weights, params, mean_ap))
print("Coco preds: {}".format(coco_preds))
out = open("{}/{}.txt".format(params['out_folder'], save_prefix[:-4]),
'w')
out.write('{}\n'.format(pred_list))
out.write('{}\n'.format(weights))
out.write('{}\n'.format(params))
out.write('{}\n'.format(mean_ap))
out.write('{}\n'.format(coco_preds))
out.close()
def annotation_validation(self, dataloader: TrainingAbnormalDataSet, _model: BaseModel) -> dict:
from src.modeling.models.retinaNetFPN.retinaNetFPN import RetinaNetFPN
model = RetinaNetFPN()
model.load_state_dict(_model.state_dict())
model.to(config.validation_device)
model.eval()
self.log.info("Beginning Validation")
dataloader.display_metrics(dataloader.get_metrics())
data = iter(DataLoader(dataloader, batch_size=config.batch_size, num_workers=4, collate_fn=self.collater))
total = (len(dataloader) // config.batch_size) + 1
# idx 0 == correct, idx 1 == incorrect
stats = {
'healthy': [0, 0],
'abnormal': [0, 0]
}
labels = ['healthy', 'abnormal']
det = []
ann = []
image_id = 0
image_id = 0
for _, i in tqdm(enumerate(range(total)), total=len(range(total)), desc="Validating the model"):
batch = next(data)
for ky, val in batch.items():
# If we can, try to load up the batched data into the device (try to only send what is needed)
if isinstance(batch[ky], torch.Tensor):
batch[ky] = batch[ky].to(config.validation_device)
predictions = model(batch)
for idx, pred in enumerate(predictions):
annotation = batch['annotations'][idx]
for p_idx in range(len(pred['boxes'])):
det.append([f'{image_id}', pred['labels'][p_idx].item(), pred['scores'][p_idx].item(), pred['boxes'][p_idx][0].item() / 256.0, pred['boxes'][p_idx][1].item() / 256.0, pred['boxes'][p_idx][2].item() / 256.0, pred['boxes'][p_idx][3].item() / 256.0])
for a_idx in range(len(batch['annotations'][idx]['boxes'])):
ann.append([f'{image_id}', torch.argmax(annotation['labels'][a_idx], 0).item(), annotation['boxes'][a_idx][0].item() / 256.0, annotation['boxes'][a_idx][1].item() / 256.0, annotation['boxes'][a_idx][2].item() / 256.0, annotation['boxes'][a_idx][3].item() / 256.0])
image_id += 1
for idx in range(len(ann)):
ann[idx][1] = 'healthy' if ann[idx][1] == 0 else 'abnormal'
for idx in range(len(det)):
det[idx][1] = 'healthy' if det[idx][1] == 0 else 'abnormal'
mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det)
# table = []
# for stat in stats:
# table.append([stat, stats[stat][0], stats[stat][1]])
#
# self.log.info(f'\n-- Validation Report --\n{tabulate(table, headers=["Type", "Correct", "Incorrect"])}')
return stats
# coding:utf-8
from map_boxes import mean_average_precision_for_boxes
ann = ann[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values
det = det[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin',
'YMax']].values
mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det)
print(mean_ap, average_precisions)
def validate(cfg, Meta_data, split):
notation_file = utils.load_obj(split)
anns = []
dets = []
anns_medium = []
dets_medium = []
anns_small = []
dets_small = []
for i, image in enumerate(notation_file):
print('Image {}/{} is under processing...'.format(i, len(notation_file)))
image_height = image['height']
image_width = image['width']
# processing annotations
ground_truth = np.zeros(len(CATEGORIES))
objs = image["annotations"]
#print('processing annotations...')
for obj in objs:
class_id = obj['category_id']
ann_image_id = image['file_name']
ann_label_name = CATEGORIES[class_id]
ann_bbox = obj['bbox'] # x0, y0, x1, y1
ann_bbox[0] /= image_width
ann_bbox[1] /= image_height
ann_bbox[2] /= image_width
ann_bbox[3] /= image_height
ann = [ann_image_id, ann_label_name, ann_bbox[0], ann_bbox[2], ann_bbox[1], ann_bbox[3]]
#print(ann)
anns.append(ann)
if (ann_bbox[2] - ann_bbox[0]) >= 0.02:
anns_medium.append(ann)
else:
anns_small.append(ann)
# processing predictions
file_name = image['file_name']
img = cv2.imread(file_name)
t0 = time.clock()
outputs = predictor(img)
t1 = time.clock() - t0
print('Exeution time is {}'.format(t1))
scores = outputs["instances"].scores.to('cpu').numpy()
pred_classes = outputs["instances"].pred_classes.to('cpu').numpy()
bbox = outputs["instances"].pred_boxes.tensor.to('cpu').numpy()
N = scores.shape[0]
#print("processing predictions...")
for i in range(N):
det_image_id = image['file_name']
class_id = pred_classes[i]
det_label_name = CATEGORIES[class_id]
det_bbox = bbox[i] # x0, y0, x1, y1
det_bbox[0] /= image_width
det_bbox[1] /= image_height
det_bbox[2] /= image_width
det_bbox[3] /= image_height
det = [det_image_id, det_label_name, scores[i], det_bbox[0], det_bbox[2], det_bbox[1], det_bbox[3]]
#print(det)
dets.append(det)
if (det_bbox[2] - det_bbox[0]) >= 0.02:
dets_medium.append(det)
else:
dets_small.append(det)
# ann = ann[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values
# det = det[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values
print("overall_metrics:")
mean_ap, average_precisions = mean_average_precision_for_boxes(anns, dets)
print("small_object_metrics:")
mean_average_precision_for_boxes(anns_small, dets_small, obj_cate='small')
print("medium_object_metrics:")
mean_average_precision_for_boxes(anns_medium, dets_medium, obj_cate='medium')
"""
Author: Roman Solovyev, IPPM RAS
URL: https://github.com/ZFTurbo
"""
from map_boxes import mean_average_precision_for_boxes
import pandas as pd
if __name__ == '__main__':
# Version 1
annotations_file = 'example/annotations.csv'
detections_file = 'example/detections.csv'
mean_ap, average_precisions = mean_average_precision_for_boxes(annotations_file, detections_file)
# Version 2
ann = pd.read_csv('example/annotations.csv')
det = pd.read_csv('example/detections.csv')
ann = ann[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values
det = det[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values
mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det)
