def compute_average_precision_detection(ground_truth,
prediction,
tiou_thresholds=np.linspace(
0.5, 0.95, 10)):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as
true positive. This code is greatly inspired by Pascal VOC devkit.
Parameters
----------
ground_truth : df
Data frame containing the ground truth instances.
Required fields: ['video-id', 't-start', 't-end']
prediction : df
Data frame containing the prediction instances.
Required fields: ['video-id, 't-start', 't-end', 'score']
tiou_thresholds : 1darray, optional
Temporal intersection over union threshold.
Outputs
-------
ap : float
Average precision score.
"""
npos = float(len(ground_truth))
lock_gt = np.ones((len(tiou_thresholds), len(ground_truth))) * -1
# Sort predictions by decreasing score order.
sort_idx = prediction['score'].values.argsort()[::-1]
prediction = prediction.loc[sort_idx].reset_index(drop=True)
# Initialize true positive and false positive vectors.
tp = np.zeros((len(tiou_thresholds), len(prediction)))
fp = np.zeros((len(tiou_thresholds), len(prediction)))
# Adaptation to query faster
ground_truth_gbvn = ground_truth.groupby('video-id')
# Assigning true positive to truly grount truth instances.
for idx, this_pred in tqdm.tqdm(prediction.iterrows()):
try:
# Check if there is at least one ground truth in the video associated.
ground_truth_videoid = ground_truth_gbvn.get_group(
this_pred['video-id'])
except Exception as e:
fp[:, idx] = 1
continue
this_gt = ground_truth_videoid.reset_index()
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for tidx, tiou_thr in enumerate(tiou_thresholds):
for jdx in tiou_sorted_idx:
if tiou_arr[jdx] < tiou_thr:
fp[tidx, idx] = 1
break
if lock_gt[tidx, this_gt.loc[jdx]['index']] >= 0:
continue
# Assign as true positive after the filters above.
tp[tidx, idx] = 1
lock_gt[tidx, this_gt.loc[jdx]['index']] = idx
break
if fp[tidx, idx] == 0 and tp[tidx, idx] == 0:
fp[tidx, idx] = 1
ap = np.zeros(len(tiou_thresholds))
for tidx in range(len(tiou_thresholds)):
# Computing prec-rec
this_tp = np.cumsum(tp[tidx, :]).astype(np.float)
this_fp = np.cumsum(fp[tidx, :]).astype(np.float)
rec = this_tp / npos
prec = this_tp / (this_tp + this_fp)
ap[tidx] = interpolated_prec_rec(prec, rec)
return ap
def compute_average_precision_detection(ground_truth, prediction, tiou_thr=0.5):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as
true positive. This code is greatly inspired by Pascal VOC devkit.
Parameters
----------
ground_truth : df
Data frame containing the ground truth instances.
Required fields: ['video-id', 't-start', 't-end']
prediction : df
Data frame containing the prediction instances.
Required fields: ['video-id, 't-start', 't-end', 'score']
tiou_thr : (float, optional)
Temporal intersection over union threshold.
Outputs
-------
ap : float
Average precision score.
"""
npos = float(len(ground_truth))
lock_gt = np.ones(len(ground_truth)) * -1
# Sort predictions by decreasing score order.
sort_idx = prediction['score'].values.argsort()[::-1]
prediction = prediction.loc[sort_idx].reset_index(drop=True)
# Initialize true positive and false positive vectors.
tp = np.zeros(len(prediction))
fp = np.zeros(len(prediction))
# Assigning true positive to truly grount truth instances.
for idx in range(len(prediction)):
this_pred = prediction.loc[idx]
gt_idx = ground_truth['video-id'] == this_pred['video-id']
# Check if there is at least one ground truth in the video associated.
if not gt_idx.any():
fp[idx] = 1
continue
this_gt = ground_truth.loc[gt_idx].reset_index()
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for jdx in tiou_sorted_idx:
if tiou_arr[jdx] < tiou_thr:
fp[idx] = 1
break
if lock_gt[this_gt.loc[jdx]['index']] >= 0:
continue
# Assign as true positive after the filters above.
tp[idx] = 1
lock_gt[this_gt.loc[jdx]['index']] = idx
break
# Computing prec-rec
tp = np.cumsum(tp).astype(np.float)
fp = np.cumsum(fp).astype(np.float)
rec = tp / npos
prec = tp / (tp + fp)
return interpolated_prec_rec(rec, prec)
def compute_average_precision_detection(ground_truth, prediction, tiou_thresholds=np.linspace(0.5, 0.95, 10)):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as
true positive. This code is greatly inspired by Pascal VOC devkit.
Parameters
----------
ground_truth : df
Data frame containing the ground truth instances.
Required fields: ['video-id', 't-start', 't-end']
prediction : df
Data frame containing the prediction instances.
Required fields: ['video-id, 't-start', 't-end', 'score']
tiou_thresholds : 1darray, optional
Temporal intersection over union threshold.
Outputs
-------
ap : float
Average precision score.
"""
ap = np.zeros(len(tiou_thresholds))
if prediction.empty:
return ap
npos = float(len(ground_truth))
lock_gt = np.ones((len(tiou_thresholds),len(ground_truth))) * -1
# Sort predictions by decreasing score order.
sort_idx = prediction['score'].values.argsort()[::-1]
prediction = prediction.loc[sort_idx].reset_index(drop=True)
# Initialize true positive and false positive vectors.
tp = np.zeros((len(tiou_thresholds), len(prediction)))
fp = np.zeros((len(tiou_thresholds), len(prediction)))
# Adaptation to query faster
ground_truth_gbvn = ground_truth.groupby('video-id')
# Assigning true positive to truly grount truth instances.
for idx, this_pred in prediction.iterrows():
try:
# Check if there is at least one ground truth in the video associated.
ground_truth_videoid = ground_truth_gbvn.get_group(this_pred['video-id'])
except Exception as e:
fp[:, idx] = 1
continue
this_gt = ground_truth_videoid.reset_index()
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for tidx, tiou_thr in enumerate(tiou_thresholds):
for jdx in tiou_sorted_idx:
if tiou_arr[jdx] < tiou_thr:
fp[tidx, idx] = 1
break
if lock_gt[tidx, this_gt.loc[jdx]['index']] >= 0:
continue
# Assign as true positive after the filters above.
tp[tidx, idx] = 1
lock_gt[tidx, this_gt.loc[jdx]['index']] = idx
break
if fp[tidx, idx] == 0 and tp[tidx, idx] == 0:
fp[tidx, idx] = 1
tp_cumsum = np.cumsum(tp, axis=1).astype(np.float)
fp_cumsum = np.cumsum(fp, axis=1).astype(np.float)
recall_cumsum = tp_cumsum / npos
precision_cumsum = tp_cumsum / (tp_cumsum + fp_cumsum)
for tidx in range(len(tiou_thresholds)):
ap[tidx] = interpolated_prec_rec(precision_cumsum[tidx,:], recall_cumsum[tidx,:])
return ap
def compute_average_precision_detection(ground_truth, prediction, tiou_thr=0.5):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as
true positive. This code is greatly inspired by Pascal VOC devkit.
Parameters
----------
ground_truth : df
Data frame containing the ground truth instances.
Required fields: ['video-id', 't-start', 't-end']
prediction : df
Data frame containing the prediction instances.
Required fields: ['video-id, 't-start', 't-end', 'score']
tiou_thr : (float, optional)
Temporal intersection over union threshold.
Outputs
-------
ap : float
Average precision score.
"""
npos = float(len(ground_truth))
lock_gt = np.ones(len(ground_truth)) * -1
# Sort predictions by decreasing score order.
sort_idx = prediction['score'].values.argsort()[::-1]
prediction = prediction.loc[sort_idx].reset_index(drop=True)
# Initialize true positive and false positive vectors.
tp = np.zeros(len(prediction))
fp = np.zeros(len(prediction))
# Assigning true positive to truly grount truth instances.
for idx in range(len(prediction)):
this_pred = prediction.loc[idx]
gt_idx = ground_truth['video-id'] == this_pred['video-id']
# Check if there is at least one ground truth in the video associated.
if not gt_idx.any():
fp[idx] = 1
continue
this_gt = ground_truth.loc[gt_idx].reset_index()
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for jdx in tiou_sorted_idx:
if tiou_arr[jdx] < tiou_thr:
fp[idx] = 1
break
if lock_gt[this_gt.loc[jdx]['index']] >= 0:
continue
# Assign as true positive after the filters above.
tp[idx] = 1
lock_gt[this_gt.loc[jdx]['index']] = idx
break
# Computing prec-rec
tp = np.cumsum(tp).astype(np.float)
fp = np.cumsum(fp).astype(np.float)
rec = tp / npos
prec = tp / (tp + fp)
return interpolated_prec_rec(prec, rec)
def analyze_fp_error_types(prediction,
ground_truth,
tiou_thr,
matched_gt_id_col_name,
min_tiou_thr=0.1,
fp_error_types_legned={'True Positive': 0,
'Double Detection Err': 1,
'Wrong Label Err': 2,
'Localization Err': 3,
'Confusion Err': 4,
'Background Err': 5}):
"""Assumes that prediction is sorted by 'prediction-id' column """
fp_error_types = {}
# Adaptation to query faster
ground_truth_gbvn = ground_truth.groupby('video-id')
fp_error_types = np.zeros(len(prediction))
this_prediction = prediction[np.isnan(prediction[matched_gt_id_col_name])].reset_index(drop=True)
this_prediction.sort_values(by='video-id', inplace=True)
this_prediction.reset_index(drop=True, inplace=True)
current_video_id = None
for idx, this_pred in this_prediction.iterrows():
if this_pred['video-id'] != current_video_id:
try:
this_gt = ground_truth_gbvn.get_group(this_pred['video-id']).reset_index()
except:
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Background Err']
current_video_id = this_pred['video-id']
continue
current_video_id = this_pred['video-id']
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
gt_with_max_tiou_label = this_gt.loc[tiou_arr.argmax()]['label']
top_tiou = tiou_arr.max()
this_pred_label = this_pred['label']
if top_tiou >= tiou_thr:
if gt_with_max_tiou_label == this_pred_label:
# double detection error
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Double Detection Err']
else:
# wrong label error
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Wrong Label Err']
elif top_tiou >= min_tiou_thr:
if gt_with_max_tiou_label == this_pred_label:
# localization error
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Localization Err']
else:
# confusion error
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Confusion Err']
else:
# background error
fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Background Err']
return fp_error_types
def compute_average_precision_detection(ground_truth, prediction, tiou_thresholds=np.linspace(0.5, 0.95, 10),
normalize_ap=False, average_num_instance_per_class=None,
minimum_normalized_precision_threshold_for_detection=0.05):
"""Compute average precision (detection task) between ground truth and
predictions data frames. If multiple predictions occurs for the same
predicted segment, only the one with highest score is matches as
true positive. This code is greatly inspired by Pascal VOC devkit.
Parameters
----------
ground_truth : df
Data frame containing the ground truth instances.
Required fields: ['video-id', 't-start', 't-end']
prediction : df
Data frame containing the prediction instances.
Required fields: ['video-id, 't-start', 't-end', 'score']
tiou_thresholds : 1darray, optional
Temporal intersection over union threshold.
Outputs
-------
ap : float
Average precision score.
"""
gt_id_lst = np.unique(ground_truth['gt-id'].values)
gt_id_to_index = dict(zip(gt_id_lst, range(len(gt_id_lst))))
lock_gt = np.ones((len(tiou_thresholds), len(gt_id_to_index))) * -1
npos = float(len(gt_id_lst))
# Sort predictions by decreasing score order.
sort_idx = prediction['score'].values.argsort()[::-1]
prediction = prediction.loc[sort_idx].reset_index(drop=True)
# Initialize true positive and false positive vectors.
tp = np.zeros((len(tiou_thresholds), len(prediction)))
fp = np.zeros((len(tiou_thresholds), len(prediction)))
matched_gt_id = np.nan*np.zeros((len(tiou_thresholds), len(prediction)))
ap = np.zeros(len(tiou_thresholds))
if prediction.empty:
return ap, matched_gt_id, prediction['prediction-id'].values, 0, 0
# Adaptation to query faster
ground_truth_gbvn = ground_truth.groupby('video-id')
# Assigning true positive to truly grount truth instances.
for idx, this_pred in prediction.iterrows():
try:
# Check if there is at least one ground truth in the video associated.
ground_truth_videoid = ground_truth_gbvn.get_group(this_pred['video-id'])
except Exception as e:
fp[:, idx] = 1
continue
this_gt = ground_truth_videoid.reset_index()
tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values,
this_gt[['t-start', 't-end']].values)
# We would like to retrieve the predictions with highest tiou score.
tiou_sorted_idx = tiou_arr.argsort()[::-1]
for tidx, tiou_thr in enumerate(tiou_thresholds):
for jdx in tiou_sorted_idx:
if tiou_arr[jdx] < tiou_thr:
fp[tidx, idx] = 1
break
if lock_gt[tidx, gt_id_to_index[this_gt.loc[jdx]['gt-id']]] >= 0:
continue
# Assign as true positive after the filters above.
tp[tidx, idx] = 1
lock_gt[tidx, gt_id_to_index[this_gt.loc[jdx]['gt-id']]] = idx
matched_gt_id[tidx, idx] = this_gt.loc[jdx]['gt-id']
break
if fp[tidx, idx] == 0 and tp[tidx, idx] == 0:
fp[tidx, idx] = 1
tp_cumsum = np.cumsum(tp, axis=1).astype(np.float)
fp_cumsum = np.cumsum(fp, axis=1).astype(np.float)
recall_cumsum = tp_cumsum / npos
if normalize_ap:
precision_cumsum = recall_cumsum * average_num_instance_per_class / \
(recall_cumsum * average_num_instance_per_class + fp_cumsum)
discard_index = precision_cumsum <= minimum_normalized_precision_threshold_for_detection
tp[discard_index] = 0
fp[discard_index] = 1
matched_gt_id[discard_index] = np.nan
tp_cumsum = np.cumsum(tp, axis=1).astype(np.float)
fp_cumsum = np.cumsum(fp, axis=1).astype(np.float)
recall_cumsum = tp_cumsum / npos
precision_cumsum = recall_cumsum * average_num_instance_per_class / \
(recall_cumsum * average_num_instance_per_class + fp_cumsum)
else:
precision_cumsum = recall_cumsum * npos / (recall_cumsum * npos + fp_cumsum)
for tidx in range(len(tiou_thresholds)):
ap[tidx] = interpolated_prec_rec(precision_cumsum[tidx, :], recall_cumsum[tidx, :])
recall = recall_cumsum[:, -1]
precision = precision_cumsum[:, -1]
return ap, matched_gt_id, prediction['prediction-id'].values, recall, precision
