def compute_average_precision_detection(ground_truth, prediction, tiou_thresholds=np.linspace( 0.5, 0.95, 10)): """Compute average precision (detection task) between ground truth and predictions data frames. If multiple predictions occurs for the same predicted segment, only the one with highest score is matches as true positive. This code is greatly inspired by Pascal VOC devkit. Parameters ---------- ground_truth : df Data frame containing the ground truth instances. Required fields: ['video-id', 't-start', 't-end'] prediction : df Data frame containing the prediction instances. Required fields: ['video-id, 't-start', 't-end', 'score'] tiou_thresholds : 1darray, optional Temporal intersection over union threshold. Outputs ------- ap : float Average precision score. """ npos = float(len(ground_truth)) lock_gt = np.ones((len(tiou_thresholds), len(ground_truth))) * -1 # Sort predictions by decreasing score order. sort_idx = prediction['score'].values.argsort()[::-1] prediction = prediction.loc[sort_idx].reset_index(drop=True) # Initialize true positive and false positive vectors. tp = np.zeros((len(tiou_thresholds), len(prediction))) fp = np.zeros((len(tiou_thresholds), len(prediction))) # Adaptation to query faster ground_truth_gbvn = ground_truth.groupby('video-id') # Assigning true positive to truly grount truth instances. for idx, this_pred in tqdm.tqdm(prediction.iterrows()): try: # Check if there is at least one ground truth in the video associated. ground_truth_videoid = ground_truth_gbvn.get_group( this_pred['video-id']) except Exception as e: fp[:, idx] = 1 continue this_gt = ground_truth_videoid.reset_index() tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. tiou_sorted_idx = tiou_arr.argsort()[::-1] for tidx, tiou_thr in enumerate(tiou_thresholds): for jdx in tiou_sorted_idx: if tiou_arr[jdx] < tiou_thr: fp[tidx, idx] = 1 break if lock_gt[tidx, this_gt.loc[jdx]['index']] >= 0: continue # Assign as true positive after the filters above. tp[tidx, idx] = 1 lock_gt[tidx, this_gt.loc[jdx]['index']] = idx break if fp[tidx, idx] == 0 and tp[tidx, idx] == 0: fp[tidx, idx] = 1 ap = np.zeros(len(tiou_thresholds)) for tidx in range(len(tiou_thresholds)): # Computing prec-rec this_tp = np.cumsum(tp[tidx, :]).astype(np.float) this_fp = np.cumsum(fp[tidx, :]).astype(np.float) rec = this_tp / npos prec = this_tp / (this_tp + this_fp) ap[tidx] = interpolated_prec_rec(prec, rec) return ap
def compute_average_precision_detection(ground_truth, prediction, tiou_thr=0.5): """Compute average precision (detection task) between ground truth and predictions data frames. If multiple predictions occurs for the same predicted segment, only the one with highest score is matches as true positive. This code is greatly inspired by Pascal VOC devkit. Parameters ---------- ground_truth : df Data frame containing the ground truth instances. Required fields: ['video-id', 't-start', 't-end'] prediction : df Data frame containing the prediction instances. Required fields: ['video-id, 't-start', 't-end', 'score'] tiou_thr : (float, optional) Temporal intersection over union threshold. Outputs ------- ap : float Average precision score. """ npos = float(len(ground_truth)) lock_gt = np.ones(len(ground_truth)) * -1 # Sort predictions by decreasing score order. sort_idx = prediction['score'].values.argsort()[::-1] prediction = prediction.loc[sort_idx].reset_index(drop=True) # Initialize true positive and false positive vectors. tp = np.zeros(len(prediction)) fp = np.zeros(len(prediction)) # Assigning true positive to truly grount truth instances. for idx in range(len(prediction)): this_pred = prediction.loc[idx] gt_idx = ground_truth['video-id'] == this_pred['video-id'] # Check if there is at least one ground truth in the video associated. if not gt_idx.any(): fp[idx] = 1 continue this_gt = ground_truth.loc[gt_idx].reset_index() tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. tiou_sorted_idx = tiou_arr.argsort()[::-1] for jdx in tiou_sorted_idx: if tiou_arr[jdx] < tiou_thr: fp[idx] = 1 break if lock_gt[this_gt.loc[jdx]['index']] >= 0: continue # Assign as true positive after the filters above. tp[idx] = 1 lock_gt[this_gt.loc[jdx]['index']] = idx break # Computing prec-rec tp = np.cumsum(tp).astype(np.float) fp = np.cumsum(fp).astype(np.float) rec = tp / npos prec = tp / (tp + fp) return interpolated_prec_rec(rec, prec)
def compute_average_precision_detection(ground_truth, prediction, tiou_thresholds=np.linspace(0.5, 0.95, 10)): """Compute average precision (detection task) between ground truth and predictions data frames. If multiple predictions occurs for the same predicted segment, only the one with highest score is matches as true positive. This code is greatly inspired by Pascal VOC devkit. Parameters ---------- ground_truth : df Data frame containing the ground truth instances. Required fields: ['video-id', 't-start', 't-end'] prediction : df Data frame containing the prediction instances. Required fields: ['video-id, 't-start', 't-end', 'score'] tiou_thresholds : 1darray, optional Temporal intersection over union threshold. Outputs ------- ap : float Average precision score. """ ap = np.zeros(len(tiou_thresholds)) if prediction.empty: return ap npos = float(len(ground_truth)) lock_gt = np.ones((len(tiou_thresholds),len(ground_truth))) * -1 # Sort predictions by decreasing score order. sort_idx = prediction['score'].values.argsort()[::-1] prediction = prediction.loc[sort_idx].reset_index(drop=True) # Initialize true positive and false positive vectors. tp = np.zeros((len(tiou_thresholds), len(prediction))) fp = np.zeros((len(tiou_thresholds), len(prediction))) # Adaptation to query faster ground_truth_gbvn = ground_truth.groupby('video-id') # Assigning true positive to truly grount truth instances. for idx, this_pred in prediction.iterrows(): try: # Check if there is at least one ground truth in the video associated. ground_truth_videoid = ground_truth_gbvn.get_group(this_pred['video-id']) except Exception as e: fp[:, idx] = 1 continue this_gt = ground_truth_videoid.reset_index() tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. tiou_sorted_idx = tiou_arr.argsort()[::-1] for tidx, tiou_thr in enumerate(tiou_thresholds): for jdx in tiou_sorted_idx: if tiou_arr[jdx] < tiou_thr: fp[tidx, idx] = 1 break if lock_gt[tidx, this_gt.loc[jdx]['index']] >= 0: continue # Assign as true positive after the filters above. tp[tidx, idx] = 1 lock_gt[tidx, this_gt.loc[jdx]['index']] = idx break if fp[tidx, idx] == 0 and tp[tidx, idx] == 0: fp[tidx, idx] = 1 tp_cumsum = np.cumsum(tp, axis=1).astype(np.float) fp_cumsum = np.cumsum(fp, axis=1).astype(np.float) recall_cumsum = tp_cumsum / npos precision_cumsum = tp_cumsum / (tp_cumsum + fp_cumsum) for tidx in range(len(tiou_thresholds)): ap[tidx] = interpolated_prec_rec(precision_cumsum[tidx,:], recall_cumsum[tidx,:]) return ap
def compute_average_precision_detection(ground_truth, prediction, tiou_thr=0.5): """Compute average precision (detection task) between ground truth and predictions data frames. If multiple predictions occurs for the same predicted segment, only the one with highest score is matches as true positive. This code is greatly inspired by Pascal VOC devkit. Parameters ---------- ground_truth : df Data frame containing the ground truth instances. Required fields: ['video-id', 't-start', 't-end'] prediction : df Data frame containing the prediction instances. Required fields: ['video-id, 't-start', 't-end', 'score'] tiou_thr : (float, optional) Temporal intersection over union threshold. Outputs ------- ap : float Average precision score. """ npos = float(len(ground_truth)) lock_gt = np.ones(len(ground_truth)) * -1 # Sort predictions by decreasing score order. sort_idx = prediction['score'].values.argsort()[::-1] prediction = prediction.loc[sort_idx].reset_index(drop=True) # Initialize true positive and false positive vectors. tp = np.zeros(len(prediction)) fp = np.zeros(len(prediction)) # Assigning true positive to truly grount truth instances. for idx in range(len(prediction)): this_pred = prediction.loc[idx] gt_idx = ground_truth['video-id'] == this_pred['video-id'] # Check if there is at least one ground truth in the video associated. if not gt_idx.any(): fp[idx] = 1 continue this_gt = ground_truth.loc[gt_idx].reset_index() tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. tiou_sorted_idx = tiou_arr.argsort()[::-1] for jdx in tiou_sorted_idx: if tiou_arr[jdx] < tiou_thr: fp[idx] = 1 break if lock_gt[this_gt.loc[jdx]['index']] >= 0: continue # Assign as true positive after the filters above. tp[idx] = 1 lock_gt[this_gt.loc[jdx]['index']] = idx break # Computing prec-rec tp = np.cumsum(tp).astype(np.float) fp = np.cumsum(fp).astype(np.float) rec = tp / npos prec = tp / (tp + fp) return interpolated_prec_rec(prec, rec)
def analyze_fp_error_types(prediction, ground_truth, tiou_thr, matched_gt_id_col_name, min_tiou_thr=0.1, fp_error_types_legned={'True Positive': 0, 'Double Detection Err': 1, 'Wrong Label Err': 2, 'Localization Err': 3, 'Confusion Err': 4, 'Background Err': 5}): """Assumes that prediction is sorted by 'prediction-id' column """ fp_error_types = {} # Adaptation to query faster ground_truth_gbvn = ground_truth.groupby('video-id') fp_error_types = np.zeros(len(prediction)) this_prediction = prediction[np.isnan(prediction[matched_gt_id_col_name])].reset_index(drop=True) this_prediction.sort_values(by='video-id', inplace=True) this_prediction.reset_index(drop=True, inplace=True) current_video_id = None for idx, this_pred in this_prediction.iterrows(): if this_pred['video-id'] != current_video_id: try: this_gt = ground_truth_gbvn.get_group(this_pred['video-id']).reset_index() except: fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Background Err'] current_video_id = this_pred['video-id'] continue current_video_id = this_pred['video-id'] tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. gt_with_max_tiou_label = this_gt.loc[tiou_arr.argmax()]['label'] top_tiou = tiou_arr.max() this_pred_label = this_pred['label'] if top_tiou >= tiou_thr: if gt_with_max_tiou_label == this_pred_label: # double detection error fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Double Detection Err'] else: # wrong label error fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Wrong Label Err'] elif top_tiou >= min_tiou_thr: if gt_with_max_tiou_label == this_pred_label: # localization error fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Localization Err'] else: # confusion error fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Confusion Err'] else: # background error fp_error_types[this_pred['prediction-id']] = fp_error_types_legned['Background Err'] return fp_error_types
def compute_average_precision_detection(ground_truth, prediction, tiou_thresholds=np.linspace(0.5, 0.95, 10), normalize_ap=False, average_num_instance_per_class=None, minimum_normalized_precision_threshold_for_detection=0.05): """Compute average precision (detection task) between ground truth and predictions data frames. If multiple predictions occurs for the same predicted segment, only the one with highest score is matches as true positive. This code is greatly inspired by Pascal VOC devkit. Parameters ---------- ground_truth : df Data frame containing the ground truth instances. Required fields: ['video-id', 't-start', 't-end'] prediction : df Data frame containing the prediction instances. Required fields: ['video-id, 't-start', 't-end', 'score'] tiou_thresholds : 1darray, optional Temporal intersection over union threshold. Outputs ------- ap : float Average precision score. """ gt_id_lst = np.unique(ground_truth['gt-id'].values) gt_id_to_index = dict(zip(gt_id_lst, range(len(gt_id_lst)))) lock_gt = np.ones((len(tiou_thresholds), len(gt_id_to_index))) * -1 npos = float(len(gt_id_lst)) # Sort predictions by decreasing score order. sort_idx = prediction['score'].values.argsort()[::-1] prediction = prediction.loc[sort_idx].reset_index(drop=True) # Initialize true positive and false positive vectors. tp = np.zeros((len(tiou_thresholds), len(prediction))) fp = np.zeros((len(tiou_thresholds), len(prediction))) matched_gt_id = np.nan*np.zeros((len(tiou_thresholds), len(prediction))) ap = np.zeros(len(tiou_thresholds)) if prediction.empty: return ap, matched_gt_id, prediction['prediction-id'].values, 0, 0 # Adaptation to query faster ground_truth_gbvn = ground_truth.groupby('video-id') # Assigning true positive to truly grount truth instances. for idx, this_pred in prediction.iterrows(): try: # Check if there is at least one ground truth in the video associated. ground_truth_videoid = ground_truth_gbvn.get_group(this_pred['video-id']) except Exception as e: fp[:, idx] = 1 continue this_gt = ground_truth_videoid.reset_index() tiou_arr = segment_iou(this_pred[['t-start', 't-end']].values, this_gt[['t-start', 't-end']].values) # We would like to retrieve the predictions with highest tiou score. tiou_sorted_idx = tiou_arr.argsort()[::-1] for tidx, tiou_thr in enumerate(tiou_thresholds): for jdx in tiou_sorted_idx: if tiou_arr[jdx] < tiou_thr: fp[tidx, idx] = 1 break if lock_gt[tidx, gt_id_to_index[this_gt.loc[jdx]['gt-id']]] >= 0: continue # Assign as true positive after the filters above. tp[tidx, idx] = 1 lock_gt[tidx, gt_id_to_index[this_gt.loc[jdx]['gt-id']]] = idx matched_gt_id[tidx, idx] = this_gt.loc[jdx]['gt-id'] break if fp[tidx, idx] == 0 and tp[tidx, idx] == 0: fp[tidx, idx] = 1 tp_cumsum = np.cumsum(tp, axis=1).astype(np.float) fp_cumsum = np.cumsum(fp, axis=1).astype(np.float) recall_cumsum = tp_cumsum / npos if normalize_ap: precision_cumsum = recall_cumsum * average_num_instance_per_class / \ (recall_cumsum * average_num_instance_per_class + fp_cumsum) discard_index = precision_cumsum <= minimum_normalized_precision_threshold_for_detection tp[discard_index] = 0 fp[discard_index] = 1 matched_gt_id[discard_index] = np.nan tp_cumsum = np.cumsum(tp, axis=1).astype(np.float) fp_cumsum = np.cumsum(fp, axis=1).astype(np.float) recall_cumsum = tp_cumsum / npos precision_cumsum = recall_cumsum * average_num_instance_per_class / \ (recall_cumsum * average_num_instance_per_class + fp_cumsum) else: precision_cumsum = recall_cumsum * npos / (recall_cumsum * npos + fp_cumsum) for tidx in range(len(tiou_thresholds)): ap[tidx] = interpolated_prec_rec(precision_cumsum[tidx, :], recall_cumsum[tidx, :]) recall = recall_cumsum[:, -1] precision = precision_cumsum[:, -1] return ap, matched_gt_id, prediction['prediction-id'].values, recall, precision