def __init__(self,
class_list,
detection_file,
mask_id,
mask_root='',
chosen_label=None,
box=None):
"""
Initialisation function for detection instance where full instance heatmap is available on file
:param class_list: list of label probabilities for each class, ordered to match the class labelling convention
of corresponding ground-truth data.
:param detection_file: location of the .npy file containing the heatmap masks for all detections within an image
:param mask_id: the id of the mask within the detection file that corresponds with this instance
:param mask_root: root location for all mask files (defaults to current directory)
:param chosen_label: the chosen label for the detection (optional). If None will be the maximum label in class
list. Note this is only used for mAP and moLRP, PDQ uses class_list.
:param box: bounding box to be used for box-style detection analysis like mAP and moLRP (optional).
If None box will be made based on provided probabilistic segmentation heatmap (probability > _HEATMAP_THRESH).
"""
super(ProbSegDetInst, self).__init__(class_list)
self.detection_file = detection_file
self.mask_id = mask_id
self.mask_root = mask_root
if chosen_label is None:
self.chosen_label = np.argmax(self.class_list)
else:
self.chosen_label = chosen_label
if box is None:
self.box = utils.generate_bounding_box_from_mask(
self.calc_heatmap() > _HEATMAP_THRESH)
self.box = box
def find_roi(img_size, mean, cov):
"""
Function for finding the region of interest for a probability heatmap generated by a Gaussian corner.
This region of interest is the area with most change therein, with probabilities above 0.0027 and below 0.9973
:param img_size: tuple: formatted (n_rows, n_cols) depicting the size of the image
:param mean: list: formatted [mu_y, mu_x] describes the location of the mean of the Gaussian corner.
:param cov: 2D array: formatted [[variance_y, corr], [corr, variance_x]] describes the covariance of the
Gaussian corner.
:return: roi_box formatted [x1, y1, x2, y2] depicting the corners of the region of interest (inclusive)
"""
# Calculate approximate ROI
stdy = cov[0, 0]**0.5
stdx = cov[1, 1]**0.5
minx = int(max(mean[1] - stdx * 5, 0))
miny = int(max(mean[0] - stdy * 5, 0))
maxx = int(min(mean[1] + stdx * 5, img_size[1] - 1))
maxy = int(min(mean[0] + stdy * 5, img_size[0] - 1))
# If the covariance is singular, we can't do any better in our estimate.
if np.abs(np.linalg.det(cov)) < 1e-8:
return minx, miny, maxx, maxy
# produce list of positions [y,x] to compare to the given mean location
approx_roi_shape = (maxy + 1 - miny, maxx + 1 - minx)
positions = np.indices(approx_roi_shape).T.reshape(-1, 2)
positions[:, 0] += miny
positions[:, 1] += minx
# Calculate the mahalanobis distances to those locations (number of standard deviations)
# Can only do this for non-singular matrices
mdists = cdist(positions,
np.array([mean]),
metric='mahalanobis',
VI=np.linalg.inv(cov))
mdists = mdists.reshape(approx_roi_shape[1], approx_roi_shape[0]).T
# Shift around the mean to change which corner of the pixel we're using for the mahalanobis distance
dist_meany = max(min(int(mean[0] - miny), img_size[0] - 1), 0)
dist_meanx = max(min(int(mean[1] - minx), img_size[1] - 1), 0)
if 0 < dist_meany < img_size[0] - 1:
mdists[:dist_meany, :] = mdists[1:dist_meany + 1, :]
if 0 < dist_meanx < img_size[1] - 1:
mdists[:, :dist_meanx] = mdists[:, 1:dist_meanx + 1]
# Mask out samples that are outside the desired distance (extremely low probability points)
mask = mdists <= _2D_MAH_DIST_THRESH
mask[
dist_meany,
dist_meanx] = True # Force the pixel containing the mean to be true, we always care about that
roi_box = utils.generate_bounding_box_from_mask(mask)
return roi_box[0] + minx, roi_box[1] + miny, roi_box[2] + minx, roi_box[
3] + miny
def __init__(self, segmentation_mask, true_class_label, image_id, instance_id, bounding_box=None, num_pixels=None):
self.segmentation_mask = segmentation_mask
self.class_label = true_class_label
self.image_id = image_id
self.instance_id = instance_id
if bounding_box is not None and len(bounding_box) > 0:
self.bounding_box = bounding_box
else:
self.bounding_box = utils.generate_bounding_box_from_mask(segmentation_mask)
if num_pixels is not None and num_pixels > 0:
self.num_pixels = num_pixels
else:
self.num_pixels = np.count_nonzero(segmentation_mask)
def __init__(self,
segmentation_mask,
true_class_label,
coco_bounding_box=None,
num_pixels=None,
coco_ignore=False,
coco_iscrowd=False,
coco_area=None):
"""
GroundTruthInstance object initialisation function. GroundTruthInstance objects are designed to hold
data pertinent for a ground-truth object in the context of evaluation. Namely, the segmentation mask,
its class label, its bounding box and number of pixels. To ensure accurate conversion from our format
back to COCO format, we also store (where possible) the bounding box, area, and ignore and iscrowd flags
provided by COCO formatted ground-truths.
:param segmentation_mask: Boolean 2D array of pixels defining the pixels which make-up the ground-truth object
:param true_class_label: integer dictating the class label (value between 0 and 1 - <number_of_classes>)
:param coco_bounding_box: bounding box provided by COCO annotations (but converted to [x1, y1, x2, y2] format)
(default None)
:param num_pixels: number of pixels in detection, calculated from segmentation_mask if not provided (default None)
:param coco_ignore: Boolean defining if object is to be ignored under COCO annotation
:param coco_iscrowd: Boolean defining if object is labelled as crowd under COCO annotation
:param coco_area: Float defining area of ground-truth object as defined under COCO annotation
"""
self.segmentation_mask = segmentation_mask
self.class_label = true_class_label
self.coco_ignore = coco_ignore
self.coco_iscrowd = coco_iscrowd
self.coco_area = coco_area
self.bounding_box = utils.generate_bounding_box_from_mask(
segmentation_mask)
# If coco_bounding_box is not supplied, equate it to the provided bounding_box
if coco_bounding_box is not None and len(coco_bounding_box) > 0:
self.coco_bounding_box = coco_bounding_box
else:
self.coco_bounding_box = self.bounding_box.copy()
# Calculate the number of pixels based on segmentation mask if unprovided at initialisation
if num_pixels is not None and num_pixels > 0:
self.num_pixels = num_pixels
else:
self.num_pixels = np.count_nonzero(segmentation_mask)
# ensure that a coco area is provided if not explicitly
if self.coco_area is None:
self.coco_area = self.num_pixels
def __iter__(self):
"""
Read ground truth for a particular image
:param image_data: The image data from the labels json
:param masks: A greyscale image containing all the masks in the image
:return: image_generator: generator of GroundTruthInstances objects for each gt instance present in
the given image.
"""
if len(self._image_data) > 0:
for instance_name in sorted(self._image_data.keys()):
if not instance_name.startswith(
'_'): # Ignore metadata attributes
detection_data = self._image_data[instance_name]
class_id = rvc1_class_list.get_class_id(
detection_data['class'])
if class_id is not None:
mask_id = int(detection_data['mask_id'])
# Add bounding box data if available and if not create bounding box from mask
if 'bounding_box' in detection_data:
bbox = [
int(v) for v in detection_data['bounding_box']
]
else:
bbox = utils.generate_bounding_box_from_mask(
self._masks == mask_id)
# Define ground-truth segmentation mask using original mask or generating bbox mask if necessary
if self._bbox_gt:
seg_mask = np.zeros(self._masks.shape,
dtype=np.bool)
seg_mask[bbox[1]:bbox[3] + 1,
bbox[0]:bbox[2] + 1] = True
else:
seg_mask = (self._masks == mask_id)
yield data_holders.GroundTruthInstance(
true_class_label=class_id,
segmentation_mask=seg_mask,
coco_bounding_box=bbox,
num_pixels=int(detection_data.get(
'num_pixels', -1)))
def generate_coco_ground_truth_and_detections(param_sequence,
n_classes,
use_heatmap=True):
"""
Function for creating ground-truth dictionary and detections list in COCO format from parameter sequence of
GroundTruthInstances and DetectionInstances.
Note currently assumes classIDs and imageIDs start at 0
:param param_sequence: ParamSequenceHolder containing all GroundTruthInstances and DetectionInstances
across all sequences being evaluated.
:param use_heatmap: Boolean dictating that BBoxes should be calculated using outskirts of heatmap rather than the
box corner locations of PBoxDetInst or BBoxDetInst objects'
:return:(coco_gt_dict, coco_det_list).
coco_gt_dict: Dictionary of ground-truth annotations in COCO format
coco_det_list: List of detections in COCO format
"""
coco_gt_dict = {
"annotations": [],
"type": "instances",
"categories": [],
"images": []
}
coco_det_list = []
# note that stored img_ids and stored_labels are formatted in original format (assuming start at 0)
stored_labels = []
# note stored annotations is formatted {<coco_image_id> : [<coco_instance_id1>, <coco_instance_id2> ...]}
current_ann_id = 1
coco_ann_ids = [[] for _ in range(len(param_sequence))]
coco_det_ids = [[] for _ in range(len(param_sequence))]
coco_img_ids = []
# go through each image to create gt dict and det list of dicts
for img_idx, (img_gt_instances,
img_det_instances) in enumerate(param_sequence):
# Handle images with no gt instances
if len(img_gt_instances) == 0:
# Add blank image to the images list
coco_gt_dict['images'].append({
'id':
img_idx + 1,
'height':
_BLANK_IMG_SHAPE[0],
'file_name':
'{}.jpg'.format(img_idx + 1),
'width':
_BLANK_IMG_SHAPE[1]
})
coco_img_ids += [img_idx + 1]
else:
# Add the image to the image list (invented file_name)
coco_gt_dict['images'].append({
'id':
img_idx + 1,
'height':
img_gt_instances[0].segmentation_mask.shape[0],
'file_name':
'{}.jpg'.format(img_idx + 1),
'width':
img_gt_instances[0].segmentation_mask.shape[1]
})
coco_img_ids += [img_idx + 1]
# Go through all ground-truth instances, adding categories where needed
for gt_idx, gt_instance in enumerate(img_gt_instances):
# Check if current instance's categoryID is currently in categories list and if not update accordingly
if gt_instance.class_label not in stored_labels:
stored_labels.append(gt_instance.class_label)
# artificial supercategory and category name and id will be the same
coco_gt_dict['categories'].append({
"supercategory":
"object",
"name":
str(gt_instance.class_label + 1),
"id":
gt_instance.class_label + 1
})
# Add annotation to annotations list
# Convert gt_box from rvc1 format to coco format
coco_gt_box = [
float(box_val) for box_val in gt_instance.coco_bounding_box
]
coco_gt_box[2] -= coco_gt_box[0]
coco_gt_box[3] -= coco_gt_box[1]
# Use the coco_iscrowd, coco_ignore, and coco_area aspects of GroundTruthInstance to ensure consistency
coco_gt_dict['annotations'].append({
'bbox':
coco_gt_box,
'iscrowd':
gt_instance.coco_iscrowd,
'ignore':
gt_instance.coco_ignore,
'category_id':
gt_instance.class_label + 1,
'image_id':
img_idx + 1,
'id':
current_ann_id,
'area':
gt_instance.coco_area
})
coco_ann_ids[img_idx] += [current_ann_id]
current_ann_id += 1
# Create coco detections for each detection in this image
for det_idx, det_instance in enumerate(img_det_instances):
if isinstance(det_instance, ProbSegDetInst):
coco_det_class = det_instance.chosen_label
else:
coco_det_class = int(det_instance.get_max_class() + 1)
coco_det_score = float(det_instance.get_max_score())
coco_det_img = img_idx + 1
# If using heatmap, generate a bounding box that fits the heatmap
if use_heatmap:
coco_det_box = utils.generate_bounding_box_from_mask(
det_instance.calc_heatmap(
img_gt_instances[0].segmentation_mask.shape) >
_HEATMAP_THRESH)
else:
if isinstance(det_instance,
(PBoxDetInst, BBoxDetInst, ProbSegDetInst)):
coco_det_box = [
float(boxval) for boxval in det_instance.box
]
else:
raise ValueError(
"Cannot create bbox for detection! Not using heatmap, PBoxDetInst, or BBoxDetInst"
)
# Convert the box from rvc1 format to coco format
coco_det_box[2] -= coco_det_box[0]
coco_det_box[3] -= coco_det_box[1]
coco_det_list.append({
'bbox': coco_det_box,
'category_id': coco_det_class,
'score': coco_det_score,
'image_id': coco_det_img
})
coco_det_ids[img_idx] += [len(coco_det_list)]
# Assuming that 80 classes are expected for COCO calculations, add categories if we have less than 80.
for i in range(n_classes):
num_categories = len(coco_gt_dict['categories'])
#print(num_categories, coco_gt_dict['categories']);
need_new_cat = True
for j in range(num_categories):
if i + 1 == coco_gt_dict['categories'][j]['id']:
need_new_cat = False
break
if need_new_cat:
print("Missed a category. Appending")
coco_gt_dict['categories'].append({
"supercategory": "object",
"name": str(i + 1),
"id": i + 1
})
return coco_gt_dict, coco_det_list
def test_generates_correct_bbox(self):
mask = np.zeros((480, 640), dtype=np.bool)
for i in range(0, 50):
mask[120 + i, 360 - i:360 + i] = True
bbox = utils.generate_bounding_box_from_mask(mask)
self.assertEqual([311, 121, 408, 169], bbox)
def test_raises_if_no_pixels(self):
mask = np.zeros((480, 640), dtype=np.bool)
with self.assertRaises(ValueError):
utils.generate_bounding_box_from_mask(mask)