def compute_gt_annotations(anchors,
annotations,
negative_overlap=0.4,
positive_overlap=0.5):
"""Obtain indices of gt annotations with the greatest overlap.
Args:
anchors: np.array of annotations of shape (N, 4) for (x1, y1, x2, y2).
annotations: np.array of shape (N, 5) for (x1, y1, x2, y2, label).
negative_overlap: IoU overlap for negative anchors
(all anchors with overlap < negative_overlap are negative).
positive_overlap: IoU overlap or positive anchors
(all anchors with overlap > positive_overlap are positive).
Returns:
positive_indices: indices of positive anchors
ignore_indices: indices of ignored anchors
argmax_overlaps_inds: ordered overlaps indices
"""
if compute_overlap is None:
raise ImportError('To use `compute_overlap`, the C extensions must be '
'built using `python setup.py build_ext --inplace`')
overlaps = compute_overlap(anchors.astype('float64'),
annotations.astype('float64'))
argmax_overlaps_inds = np.argmax(overlaps, axis=1)
max_overlaps = overlaps[np.arange(overlaps.shape[0]), argmax_overlaps_inds]
# assign "dont care" labels
positive_indices = max_overlaps >= positive_overlap
ignore_indices = (max_overlaps > negative_overlap) & ~positive_indices
return positive_indices, ignore_indices, argmax_overlaps_inds
def match_nodes(gt, res):
"""Loads all data that matches each pattern and compares the graphs.
Args:
gt (numpy.array): data array to match to unique.
res (numpy.array): ground truth array with all cells labeled uniquely.
Returns:
numpy.array: IoU of ground truth cells and predicted cells.
"""
num_frames = gt.shape[0]
iou = np.zeros((num_frames, np.max(gt) + 1, np.max(res) + 1))
# Compute IOUs only when neccesary
# If bboxs for true and pred do not overlap with each other, the assignment
# is immediate. Otherwise use pixelwise IOU to determine which cell is which
# Regionprops expects one frame at a time
for frame in range(num_frames):
gt_frame = gt[frame]
res_frame = res[frame]
gt_props = regionprops(np.squeeze(gt_frame.astype('int')))
gt_boxes = [np.array(gt_prop.bbox) for gt_prop in gt_props]
gt_boxes = np.array(gt_boxes).astype('double')
gt_box_labels = [int(gt_prop.label) for gt_prop in gt_props]
res_props = regionprops(np.squeeze(res_frame.astype('int')))
res_boxes = [np.array(res_prop.bbox) for res_prop in res_props]
res_boxes = np.array(res_boxes).astype('double')
res_box_labels = [int(res_prop.label) for res_prop in res_props]
# has the form [gt_bbox, res_bbox]
overlaps = compute_overlap(gt_boxes, res_boxes)
# Find the bboxes that have overlap at all
# (ind_ corresponds to box number - starting at 0)
ind_gt, ind_res = np.nonzero(overlaps)
# frame_ious = np.zeros(overlaps.shape)
for index in range(ind_gt.shape[0]):
iou_gt_idx = gt_box_labels[ind_gt[index]]
iou_res_idx = res_box_labels[ind_res[index]]
intersection = np.logical_and(gt_frame == iou_gt_idx,
res_frame == iou_res_idx)
union = np.logical_or(gt_frame == iou_gt_idx,
res_frame == iou_res_idx)
iou[frame, iou_gt_idx,
iou_res_idx] = intersection.sum() / union.sum()
return iou
def match_nodes(pattern1, pattern2):
gt = load_data(pattern1)
res = load_data(pattern2)
num_frames = gt.shape[0]
iou = np.zeros((num_frames, np.max(gt) + 1, np.max(res) + 1))
# Compute IOUs only when neccesary
# If bboxs for true and pred do not overlap with each other, the assignment
# is immediate. Otherwise use pixel-wise IOU to determine which cell is which
# Regionprops expects one frame at a time
for frame in range(num_frames):
gt_frame = gt[frame]
res_frame = res[frame]
gt_props = skimage.measure.regionprops(
np.squeeze(gt_frame.astype('int')))
gt_boxes = [np.array(gt_prop.bbox) for gt_prop in gt_props]
gt_boxes = np.array(gt_boxes).astype('double')
gt_box_labels = [int(gt_prop.label) for gt_prop in gt_props]
res_props = skimage.measure.regionprops(
np.squeeze(res_frame.astype('int')))
res_boxes = [np.array(res_prop.bbox) for res_prop in res_props]
res_boxes = np.array(res_boxes).astype('double')
res_box_labels = [int(res_prop.label) for res_prop in res_props]
overlaps = compute_overlap(
gt_boxes, res_boxes) # has the form [gt_bbox, res_bbox]
# Find the bboxes that have overlap at all (ind_ corresponds to box number - starting at 0)
ind_gt, ind_res = np.nonzero(overlaps)
for index in range(ind_gt.shape[0]):
iou_gt_idx = gt_box_labels[ind_gt[index]]
iou_res_idx = res_box_labels[ind_res[index]]
intersection = np.logical_and(gt_frame == iou_gt_idx,
res_frame == iou_res_idx)
union = np.logical_or(gt_frame == iou_gt_idx,
res_frame == iou_res_idx)
iou[frame, iou_gt_idx,
iou_res_idx] = intersection.sum() / union.sum()
gtcells, rescells = np.where(np.nansum(iou, axis=0) >= 1)
return gtcells, rescells
def _calc_iou(self):
"""Calculates IoU matrix for each pairwise comparison between true and
predicted. Additionally, if seg is True, records a 1 for each pair of
objects where $|Tbigcap P| > 0.5 * |T|$
"""
def get_box_labels(images):
props = regionprops(np.squeeze(images))
boxes, labels = [], []
for prop in props:
boxes.append(np.array(prop.bbox))
labels.append(int(prop.label))
boxes = np.array(boxes).astype('double')
return boxes, labels
self.iou = np.zeros((self.n_true, self.n_pred))
if self.seg:
self.seg_thresh = np.zeros((self.n_true, self.n_pred))
# Use bounding boxes to find masks that are likely to overlap
y_true_boxes, y_true_labels = get_box_labels(self.y_true.astype('int'))
y_pred_boxes, y_pred_labels = get_box_labels(self.y_pred.astype('int'))
# has the form [gt_bbox, res_bbox]
overlaps = compute_overlap(y_true_boxes, y_pred_boxes)
# Find the bboxes that have overlap at all
# (ind_ corresponds to box number - starting at 0)
ind_true, ind_pred = np.nonzero(overlaps)
for index in range(ind_true.shape[0]):
iou_y_true_idx = y_true_labels[ind_true[index]]
iou_y_pred_idx = y_pred_labels[ind_pred[index]]
intersection = np.logical_and(self.y_true == iou_y_true_idx,
self.y_pred == iou_y_pred_idx)
union = np.logical_or(self.y_true == iou_y_true_idx,
self.y_pred == iou_y_pred_idx)
# Subtract 1 from index to account for skipping 0
self.iou[iou_y_true_idx - 1,
iou_y_pred_idx - 1] = intersection.sum() / union.sum()
if (self.seg) & \
(intersection.sum() > 0.5 * np.sum(self.y_true == index)):
self.seg_thresh[iou_y_true_idx - 1, iou_y_pred_idx - 1] = 1
def evaluate(generator,
model,
iou_threshold=0.5,
score_threshold=0.05,
max_detections=100):
"""Evaluate a given dataset using a given model.
Args:
generator: The generator that represents the dataset to evaluate.
model: The model to evaluate.
iou_threshold: The threshold used to consider when a detection
is positive or negative.
score_threshold: The score confidence threshold to use for detections.
max_detections: The maximum number of detections to use per image.
Returns:
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections, _ = _get_detections(generator,
model,
score_threshold=score_threshold,
max_detections=max_detections)
all_annotations, _ = _get_annotations(generator)
average_precisions = {}
# all_detections = pickle.load(open('all_detections.pkl', 'rb'))
# all_annotations = pickle.load(open('all_annotations.pkl', 'rb'))
# pickle.dump(all_detections, open('all_detections.pkl', 'wb'))
# pickle.dump(all_annotations, open('all_annotations.pkl', 'wb'))
# process detections and annotations
for label in range(generator.num_classes):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.y.shape[0]):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
# type `double` is required for `compute_overlap`
annotations = annotations.astype('double')
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned]
if max_overlap >= iou_threshold and assigned not in detected:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected.append(assigned)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0, 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = _compute_ap(recall, precision)
average_precisions[label] = average_precision, num_annotations
return average_precisions