def test_can_find_missing_boxes(self):
detections = 3
class_count = 2
item1 = DatasetItem(id=1, annotations=[
BboxObject(i * 10, 10, 10, 10, label=i,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections) if i % 2 == 0
])
item2 = DatasetItem(id=2, annotations=[
BboxObject(i * 10, 10, 10, 10, label=(i + 1) % class_count,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections) if i % 2 == 1
])
iou_thresh = 0.5
conf_thresh = 0.5
comp = Comparator(iou_threshold=iou_thresh, conf_threshold=conf_thresh)
result = comp.compare_item_bboxes(item1, item2)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(len([it for it in item1.annotations \
if conf_thresh < it.attributes['score']]),
len(a_greater))
self.assertEqual(len([it for it in item2.annotations \
if conf_thresh < it.attributes['score']]),
len(b_greater))
self.assertEqual(0, len(matches))
def __iter__(self):
items = [
DatasetItem(id=0,
subset='train',
annotations=[
PointsObject([1, 2, 0, 2, 4, 1], [0, 1, 2],
label=3,
group=1,
id=1),
BboxObject(1, 2, 3, 4, label=3, group=1),
PointsObject([5, 6, 0, 7], group=2, id=2),
BboxObject(1, 2, 3, 4, group=2),
]),
DatasetItem(id=1,
subset='train',
annotations=[
PointsObject([1, 2, 0, 2, 4, 1],
label=5,
group=3,
id=3),
BboxObject(1, 2, 3, 4, label=5, group=3),
]),
DatasetItem(id=2,
subset='val',
annotations=[
PointsObject([0, 2, 0, 2, 4, 1],
label=2,
group=3,
id=3),
BboxObject(0, 2, 4, 4, label=2, group=3),
]),
]
return iter(items)
def DISABLED_test_roi_nms():
ROI = namedtuple('ROI', ['conf', 'x', 'y', 'w', 'h', 'label'])
class_count = 3
noisy_count = 3
rois = [
ROI(0.3, 10, 40, 30, 10, 0),
ROI(0.5, 70, 90, 7, 10, 0),
ROI(0.7, 5, 20, 40, 60, 2),
ROI(0.9, 30, 20, 10, 40, 1),
]
pixel_jitter = 10
detections = []
for i, roi in enumerate(rois):
detections.append(
BboxObject(roi.x,
roi.y,
roi.w,
roi.h,
label=roi.label,
attributes={'score': roi.conf}))
for j in range(noisy_count):
cls_conf = roi.conf * j / noisy_count
cls = (i + j) % class_count
box = [roi.x, roi.y, roi.w, roi.h]
offset = (np.random.rand(4) - 0.5) * pixel_jitter
detections.append(
BboxObject(*(box + offset),
label=cls,
attributes={'score': cls_conf}))
image = np.zeros((100, 100, 3))
for i, det in enumerate(detections):
roi = ROI(det.attributes['score'], *det.get_bbox(), det.label)
p1 = (int(roi.x), int(roi.y))
p2 = (int(roi.x + roi.w), int(roi.y + roi.h))
c = (0, 1 * (i % (1 + noisy_count) == 0), 1)
cv2.rectangle(image, p1, p2, c)
cv2.putText(image, 'd%s-%s-%.2f' % (i, roi.label, roi.conf), p1,
cv2.FONT_HERSHEY_SIMPLEX, 0.25, c)
cv2.imshow('nms_image', image)
cv2.waitKey(0)
nms_boxes = RISE.nms(detections, iou_thresh=0.25)
print(len(detections), len(nms_boxes))
for i, det in enumerate(nms_boxes):
roi = ROI(det.attributes['score'], *det.get_bbox(), det.label)
p1 = (int(roi.x), int(roi.y))
p2 = (int(roi.x + roi.w), int(roi.y + roi.h))
c = (0, 1, 0)
cv2.rectangle(image, p1, p2, c)
cv2.putText(image, 'p%s-%s-%.2f' % (i, roi.label, roi.conf), p1,
cv2.FONT_HERSHEY_SIMPLEX, 0.25, c)
cv2.imshow('nms_image', image)
cv2.waitKey(0)
def __iter__(self):
items = [
DatasetItem(id=1,
annotations=[
BboxObject(2, 3, 4, 5, label=2, id=1),
BboxObject(2, 3, 4, 5, label=3, id=2),
]),
DatasetItem(id=2,
annotations=[
BboxObject(5, 4, 6, 5, label=3, id=1),
]),
]
for item in items:
yield item
def test_no_bbox_diff_with_same_item(self):
detections = 3
anns = [
BboxObject(i * 10, 10, 10, 10, label=i,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
]
item = DatasetItem(id=0, annotations=anns)
iou_thresh = 0.5
conf_thresh = 0.5
comp = Comparator(iou_threshold=iou_thresh, conf_threshold=conf_thresh)
result = comp.compare_item_bboxes(item, item)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len([it for it in item.annotations \
if conf_thresh < it.attributes['score']]),
len(matches))
for a_bbox, b_bbox in matches:
self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
self.assertEqual(a_bbox.label, b_bbox.label)
self.assertLess(conf_thresh, a_bbox.attributes['score'])
self.assertLess(conf_thresh, b_bbox.attributes['score'])
def __iter__(self):
items = [
DatasetItem(id=1, annotations=[
LabelObject(2, id=1),
]),
DatasetItem(id=2,
image=np.zeros((5, 5, 3)),
annotations=[
LabelObject(3, id=3),
BboxObject(0,
0,
5,
5,
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
PolygonObject(
[0, 0, 4, 0, 4, 4],
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
]),
]
for item in items:
yield item
def __iter__(self):
items = [
DatasetItem(
id=0,
image=np.zeros((5, 5, 3)),
subset='train',
annotations=[
BboxObject(0,
0,
5,
5,
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
PolygonObject([0, 0, 4, 0, 4, 4],
label=3,
id=4,
group=4,
attributes={'is_crowd': False}),
MaskObject(
np.array(
[[0, 1, 1, 1, 0], [0, 0, 1, 1, 0],
[0, 0, 0, 1, 0], [0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]],
# only internal fragment (without the border),
# but not everywhere...
dtype=np.bool),
attributes={'is_crowd': False},
label=3,
id=4,
group=4),
]),
]
return iter(items)
def __iter__(self):
items = [
DatasetItem(id=100,
subset='train',
annotations=[
CaptionObject('hello', id=1),
CaptionObject('world', id=2, group=5),
LabelObject(2,
id=3,
attributes={
'x': 1,
'y': '2',
}),
BboxObject(1,
2,
3,
4,
label=4,
id=4,
attributes={
'score': 10.0,
}),
BboxObject(5, 6, 7, 8, id=5, group=5),
PointsObject([1, 2, 2, 0, 1, 1], label=0,
id=5),
MaskObject(label=3,
id=5,
image=np.ones((2, 3))),
]),
DatasetItem(id=21,
subset='train',
annotations=[
CaptionObject('test'),
LabelObject(2),
BboxObject(1, 2, 3, 4, 5, id=42, group=42)
]),
DatasetItem(id=2,
subset='val',
annotations=[
PolyLineObject([1, 2, 3, 4, 5, 6, 7, 8],
id=11),
PolygonObject([1, 2, 3, 4, 5, 6, 7, 8], id=12),
]),
DatasetItem(id=42, subset='test'),
]
return iter(items)
def save_annotations(self, item):
for ann in item.annotations:
if ann.type != AnnotationType.points:
continue
elem = {
'id': self._get_ann_id(ann),
'image_id': _cast(item.id, int, 0),
'category_id': _cast(ann.label, int, -1) + 1,
}
if 'score' in ann.attributes:
elem['score'] = float(ann.attributes['score'])
keypoints = []
points = ann.get_points()
visibility = ann.visibility
for index in range(0, len(points), 2):
kp = points[index:index + 2]
state = visibility[index // 2].value
keypoints.extend([*kp, state])
num_visible = len([v for v in visibility \
if v == PointsObject.Visibility.visible])
bbox = find(item.annotations, lambda x: \
x.group == ann.group and \
x.type == AnnotationType.bbox and
x.label == ann.label)
if bbox is None:
bbox = BboxObject(*ann.get_bbox())
elem.update({
'segmentation': bbox.get_polygon(),
'area': bbox.area(),
'bbox': bbox.get_bbox(),
'iscrowd': 0,
'keypoints': keypoints,
'num_keypoints': num_visible,
})
self.annotations.append(elem)
def _process(self, image):
detections = []
for i, roi in enumerate(self.rois):
roi_sum = self.roi_value(roi, image)
roi_base_sum = self.roi_base_sums[i]
first_run = roi_base_sum is None
if first_run:
roi_base_sum = roi_sum
self.roi_base_sums[i] = roi_base_sum
cls_conf = roi_sum / roi_base_sum
if roi.threshold < roi_sum / roi_base_sum:
cls = roi.label
detections.append(
BboxObject(roi.x,
roi.y,
roi.w,
roi.h,
label=cls,
attributes={'score': cls_conf}))
if first_run:
continue
for j in range(self.fp_count):
if roi.threshold < cls_conf:
cls = roi.label
else:
cls = (i + j) % self.class_count
box = [roi.x, roi.y, roi.w, roi.h]
offset = (np.random.rand(4) - 0.5) * self.pixel_jitter
detections.append(
BboxObject(*(box + offset),
label=cls,
attributes={'score': cls_conf}))
return detections
def _get_annotations(self, item, subset_name):
annotations = []
det_ann = self._annotations[subset_name].get(item)
if det_ann is not None:
for desc in det_ann:
label_id, conf, left, top, right, bottom = desc
label_id = self._get_label_id(VocLabel(int(label_id)).name)
annotations.append(
BboxObject(x=float(left),
y=float(top),
w=float(right) - float(left),
h=float(bottom) - float(top),
label=label_id,
attributes={'score': float(conf)}))
return annotations
def _get_annotations(self, item, subset_name):
annotations = []
layout_ann = self._annotations[subset_name].get(item)
if layout_ann is not None:
for desc in layout_ann:
obj_id, conf, parts = desc
attributes = {
'score': conf,
'object_id': obj_id,
}
for part in parts:
part_id, bbox = part
label_id = self._get_label_id(VocBodyPart(part_id).name)
annotations.append(
BboxObject(*bbox,
label=label_id,
attributes=attributes))
return annotations
def _get_annotations(self, item):
item_annotations = []
if self._task is VocTask.segmentation:
segm_path = osp.join(self._path, VocPath.SEGMENTATION_DIR,
item + VocPath.SEGM_EXT)
if osp.isfile(segm_path):
inverse_cls_colormap = \
self._categories[AnnotationType.mask].inverse_colormap
item_annotations.append(
MaskObject(image=lazy_mask(segm_path,
inverse_cls_colormap),
attributes={'class': True}))
inst_path = osp.join(self._path, VocPath.INSTANCES_DIR,
item + VocPath.SEGM_EXT)
if osp.isfile(inst_path):
item_annotations.append(
MaskObject(image=lazy_mask(inst_path,
_inverse_inst_colormap),
attributes={'instances': True}))
cls_annotations = self._annotations.get(VocTask.classification)
if cls_annotations is not None and \
self._task is VocTask.classification:
item_labels = cls_annotations.get(item)
if item_labels is not None:
for label in item_labels:
label_id = self._get_label_id(VocLabel(label).name)
item_annotations.append(LabelObject(label_id))
det_annotations = self._annotations.get(VocTask.detection)
if det_annotations is not None:
det_annotations = det_annotations.get(item)
if det_annotations is not None:
root_elem = ET.fromstring(det_annotations)
for obj_id, object_elem in enumerate(root_elem.findall('object')):
attributes = {}
group = None
obj_label_id = None
label_elem = object_elem.find('name')
if label_elem is not None:
obj_label_id = self._get_label_id(label_elem.text)
obj_bbox = self._parse_bbox(object_elem)
if obj_label_id is None or obj_bbox is None:
continue
difficult_elem = object_elem.find('difficult')
if difficult_elem is not None:
attributes['difficult'] = (difficult_elem.text == '1')
truncated_elem = object_elem.find('truncated')
if truncated_elem is not None:
attributes['truncated'] = (truncated_elem.text == '1')
occluded_elem = object_elem.find('occluded')
if occluded_elem is not None:
attributes['occluded'] = (occluded_elem.text == '1')
pose_elem = object_elem.find('pose')
if pose_elem is not None:
attributes['pose'] = pose_elem.text
point_elem = object_elem.find('point')
if point_elem is not None:
point_x = point_elem.find('x')
point_y = point_elem.find('y')
point = [float(point_x.text), float(point_y.text)]
attributes['point'] = point
actions_elem = object_elem.find('actions')
if actions_elem is not None and \
self._task is VocTask.action_classification:
for action in VocAction:
action_elem = actions_elem.find(action.name)
if action_elem is None or action_elem.text != '1':
continue
act_label_id = self._get_label_id(action.name)
assert group in [None, obj_id]
group = obj_id
item_annotations.append(
LabelObject(act_label_id, group=obj_id))
if self._task is VocTask.person_layout:
for part_elem in object_elem.findall('part'):
part = part_elem.find('name').text
part_label_id = self._get_label_id(part)
bbox = self._parse_bbox(part_elem)
group = obj_id
item_annotations.append(
BboxObject(*bbox,
label=part_label_id,
group=obj_id))
if self._task in [
VocTask.action_classification, VocTask.person_layout
]:
if group is None:
continue
item_annotations.append(
BboxObject(*obj_bbox,
label=obj_label_id,
attributes=attributes,
id=obj_id,
group=group))
return item_annotations
def _load_annotations(self, item):
parsed = item['annotations']
loaded = []
for ann in parsed:
ann_id = ann.get('id')
ann_type = AnnotationType[ann['type']]
attributes = ann.get('attributes')
group = ann.get('group')
if ann_type == AnnotationType.label:
label_id = ann.get('label_id')
loaded.append(LabelObject(label=label_id,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.mask:
label_id = ann.get('label_id')
mask_id = str(ann.get('mask_id'))
mask_path = osp.join(self._path, DatumaroPath.ANNOTATIONS_DIR,
DatumaroPath.MASKS_DIR, mask_id + DatumaroPath.MASK_EXT)
mask = None
if osp.isfile(mask_path):
mask_cat = self._categories.get(AnnotationType.mask)
if mask_cat is not None:
mask = lazy_mask(mask_path, mask_cat.inverse_colormap)
else:
mask = lazy_image(mask_path)
loaded.append(MaskObject(label=label_id, image=mask,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.polyline:
label_id = ann.get('label_id')
points = ann.get('points')
loaded.append(PolyLineObject(points, label=label_id,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.polygon:
label_id = ann.get('label_id')
points = ann.get('points')
loaded.append(PolygonObject(points, label=label_id,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.bbox:
label_id = ann.get('label_id')
x, y, w, h = ann.get('bbox')
loaded.append(BboxObject(x, y, w, h, label=label_id,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.points:
label_id = ann.get('label_id')
points = ann.get('points')
loaded.append(PointsObject(points, label=label_id,
id=ann_id, attributes=attributes, group=group))
elif ann_type == AnnotationType.caption:
caption = ann.get('caption')
loaded.append(CaptionObject(caption,
id=ann_id, attributes=attributes, group=group))
else:
raise NotImplementedError()
return loaded
def __iter__(self):
items = [
DatasetItem(
id=0,
subset='train',
image=np.ones((4, 4, 3)),
annotations=[
# Bbox + single polygon
BboxObject(0,
1,
2,
3,
label=2,
group=1,
id=1,
attributes={'is_crowd': False}),
PolygonObject([0, 1, 2, 1, 2, 3, 0, 3],
attributes={'is_crowd': False},
label=2,
group=1,
id=1),
]),
DatasetItem(
id=1,
subset='train',
annotations=[
# Mask + bbox
MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0],
[1, 1, 0, 0], [0, 0, 1, 0]],
dtype=np.bool),
attributes={'is_crowd': True},
label=4,
group=3,
id=3),
BboxObject(0,
1,
3,
3,
label=4,
group=3,
id=3,
attributes={'is_crowd': True}),
]),
DatasetItem(
id=3,
subset='val',
annotations=[
# Bbox + mask
BboxObject(0,
1,
3,
2,
label=4,
group=3,
id=3,
attributes={'is_crowd': True}),
MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0],
[1, 1, 0, 0], [0, 0, 0, 0]],
dtype=np.bool),
attributes={'is_crowd': True},
label=4,
group=3,
id=3),
]),
]
return iter(items)