def verify_data(orig_ann: Annotation, classes_matching: dict, res_project_meta: ProjectMeta) -> Annotation:
ann = orig_ann.clone()
imsize = ann.img_size
for first_class, second_class in classes_matching.items():
mask1 = np.zeros(imsize, dtype=np.bool)
mask2 = np.zeros(imsize, dtype=np.bool)
for label in ann.labels:
if label.obj_class.name == first_class:
label.geometry.draw(mask1, True)
elif label.obj_class.name == second_class:
label.geometry.draw(mask2, True)
iou_value = _compute_masks_iou(mask1, mask2)
tag_meta = res_project_meta.img_tag_metas.get(make_iou_tag_name(first_class))
tag = Tag(tag_meta, iou_value)
ann.add_tag(tag)
fp_mask = _create_fp_mask(mask1, mask2)
if fp_mask.sum() != 0:
fp_object_cls = res_project_meta.obj_classes.get(make_false_positive_name(first_class))
fp_geom = Bitmap(data=fp_mask)
fp_label = Label(fp_geom, fp_object_cls)
ann.add_label(fp_label)
fn_mask = _create_fn_mask(mask1, mask2)
if fn_mask.sum() != 0:
fn_object_cls = res_project_meta.obj_classes.get(make_false_negative_name(first_class))
fn_geom = Bitmap(data=fn_mask)
fn_label = Label(fn_geom, fn_object_cls)
ann.add_label(fn_label)
return ann
def infer_on_image(image,
graph,
model,
idx_to_class_title,
project_meta,
confidence_tag_meta,
min_confidence=0):
with graph.as_default():
[results] = model.detect([image], verbose=0)
res_labels = []
for mask_idx, class_id in enumerate(results['class_ids']):
confidence = results['scores'][mask_idx]
if confidence < min_confidence:
continue
bool_mask = results['masks'][:, :, mask_idx] != 0
class_geometry = Bitmap(data=bool_mask)
cls_title = idx_to_class_title[class_id]
label = Label(geometry=class_geometry,
obj_class=project_meta.get_obj_class(cls_title))
confidence_tag = Tag(confidence_tag_meta,
value=round(float(confidence), 4))
label = label.add_tag(confidence_tag)
res_labels.append(label)
return res_labels
def extract_labels_from_mask(mask: np.ndarray, color_id_to_obj_class: collections.Mapping) -> list:
"""
Extract multiclass instances from grayscale mask and save it to labels list.
Args:
mask: multiclass grayscale mask
color_id_to_obj_class: dict of objects classes assigned to color id (e.g. {1: ObjClass('cat), ...})
Returns:
list of labels with bitmap geometry
"""
zero_offset = 1 if 0 in color_id_to_obj_class else 0
if zero_offset > 0:
mask = mask + zero_offset
labeled, labels_count = measure.label(mask, connectivity=1, return_num=True)
objects_slices = ndimage.find_objects(labeled)
labels = []
for object_index, slices in enumerate(objects_slices, start=1):
crop = mask[slices]
sub_mask = crop * (labeled[slices] == object_index).astype(np.int)
class_index = np.max(sub_mask) - zero_offset
if class_index in color_id_to_obj_class:
bitmap = Bitmap(data=sub_mask.astype(np.bool), origin=PointLocation(slices[0].start, slices[1].start))
label = Label(geometry=bitmap, obj_class=color_id_to_obj_class.get(class_index))
labels.append(label)
return labels
def _do_infer_annotate(self, img: np.ndarray,
ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(
ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(
_replace_labels_classes(roi_ann.labels,
self._model_class_mapper,
self._model_tag_meta_mapper,
skip_missing=True))
model_img_level_tags = make_renamed_tags(
roi_ann.img_tags,
self._model_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi,
obj_class=self._intermediate_class_mapper.map(
src_label.obj_class),
tags=model_img_level_tags))
# Regardless of whether we need to save intermediate bounding boxes, also put the inference result tags
# onto the original source object from which we created a bounding box.
# This is necessary for e.g. classification models to work, so that they put the classification results
# onto the original object.
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def detection_preds_to_sly_rects(
idx_to_class, network_prediction: DetectionNetworkPrediction,
img_shape, min_score_threshold, score_tag_meta) -> list:
"""
Converts network detection results to Supervisely Labels with Rectangle geometry.
Args:
idx_to_class: Dict matching predicted boxes with appropriate ObjClass.
network_prediction: Network predictions packed into DetectionNetworkPrediction instance.
img_shape: Size(height, width) of image that was used for inference.
min_score_threshold: All detections with less scores will be dropped.
score_tag_meta: TagMeta instance for score tags.
Returns:
A list containing labels with detection rectangles.
"""
labels = []
thr_mask = np.squeeze(network_prediction.scores) > min_score_threshold
for box, class_id, score in zip(
np.squeeze(network_prediction.boxes)[thr_mask],
np.squeeze(network_prediction.classes)[thr_mask],
np.squeeze(network_prediction.scores)[thr_mask]):
xmin = round(float(box[1] * img_shape[1]))
ymin = round(float(box[0] * img_shape[0]))
xmax = round(float(box[3] * img_shape[1]))
ymax = round(float(box[2] * img_shape[0]))
rect = Rectangle(top=ymin, left=xmin, bottom=ymax, right=xmax)
class_obj = idx_to_class[int(class_id)]
label = Label(geometry=rect, obj_class=class_obj)
score_tag = Tag(score_tag_meta, value=round(float(score), 4))
label = label.add_tag(score_tag)
labels.append(label)
return labels
def to_contours(label: Label):
new_obj_cls = classes_mapping.get(label.obj_class.name)
if new_obj_cls is None:
return [label]
if not isinstance(label.geometry, Bitmap):
raise RuntimeError('Input class must be a Bitmap.')
return [Label(geometry=geom, obj_class=new_obj_cls) for geom in label.geometry.to_contours()]
def from_imgaug(cls,
img,
ia_boxes=None,
ia_masks=None,
index_to_class=None,
meta: ProjectMeta = None):
if ((ia_boxes is not None) or (ia_masks is not None)) and meta is None:
raise ValueError("Project meta has to be provided")
labels = []
if ia_boxes is not None:
for ia_box in ia_boxes:
obj_class = meta.get_obj_class(ia_box.label)
if obj_class is None:
raise KeyError(
"Class {!r} not found in project meta".format(
ia_box.label))
lbl = Label(
Rectangle(top=ia_box.y1,
left=ia_box.x1,
bottom=ia_box.y2,
right=ia_box.x2), obj_class)
labels.append(lbl)
if ia_masks is not None:
if index_to_class is None:
raise ValueError(
"mapping from index to class name is needed to transform masks to SLY format"
)
class_mask = ia_masks.get_arr()
# mask = white_mask == 255
(unique, counts) = np.unique(class_mask, return_counts=True)
for index, count in zip(unique, counts):
if index == 0:
continue
mask = class_mask == index
bitmap = Bitmap(data=mask[:, :, 0])
restore_class = meta.get_obj_class(index_to_class[index])
labels.append(Label(geometry=bitmap, obj_class=restore_class))
return cls(img_size=img.shape[:2], labels=labels)
def to_segmentation_task(self):
class_mask = {}
for label in self.labels:
if label.obj_class not in class_mask:
class_mask[label.obj_class] = np.zeros(self.img_size, np.uint8)
label.draw(class_mask[label.obj_class], color=255)
new_labels = []
for obj_class, white_mask in class_mask.items():
mask = white_mask == 255
bitmap = Bitmap(data=mask)
new_labels.append(Label(geometry=bitmap, obj_class=obj_class))
return self.clone(labels=new_labels)
def add_background(ann: Annotation, bg_class: ObjClass) -> Annotation:
"""
Adds background rectangle (size equals to image size) to annotation.
Args:
ann: Input annotation.
bg_class: ObjClass instance for background class label.
Returns:
Annotation with added background rectangle.
"""
img_size = ann.img_size
rect = Rectangle(0, 0, img_size[0] - 1, img_size[1] - 1)
new_label = Label(rect, bg_class)
return ann.add_label(new_label)
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(replace_labels_classes(
roi_ann.labels, self._model_class_mapper, skip_missing=True))
model_img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi, obj_class=self._intermediate_class_mapper.map(src_label.obj_class),
tags=model_img_level_tags))
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def segmentation_array_to_sly_bitmaps(idx_to_class: dict,
pred: np.ndarray,
origin: PointLocation = None) -> list:
"""
Converts array with segmentation results to Labels with Bitmap geometry according to idx_to_class mapping.
Args:
idx_to_class: Dict matching values in prediction array with appropriate ObjClass.
pred: Array containing raw segmentation results.
origin: Origin point for all output Bitmaps.
return:
A list containing result labels.
"""
labels = []
for cls_idx, cls_obj in idx_to_class.items():
predicted_class_pixels = (pred == cls_idx)
if np.any(predicted_class_pixels):
class_geometry = Bitmap(data=predicted_class_pixels, origin=origin)
labels.append(Label(geometry=class_geometry, obj_class=cls_obj))
return labels
def setUp(self):
self._obj_class_gt = ObjClass(name='a', geometry_type=Rectangle)
self._obj_class_pred = ObjClass(name='b', geometry_type=Rectangle)
self._confidence_tag_meta = TagMeta(name='confidence', value_type=TagValueType.ANY_NUMBER)
self._meta = ProjectMeta(
obj_classes=ObjClassCollection([self._obj_class_gt, self._obj_class_pred]),
tag_metas=TagMetaCollection([self._confidence_tag_meta]))
# Will match self._pred_obj_1
self._gt_obj_1 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(0, 0, 10, 10))
# Will match self._pred_obj_3
self._gt_obj_2 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(13, 13, 15, 15))
# Will be a false negative
self._gt_obj_3 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(43, 43, 45, 45))
# Will match self._gt_obj_1
self._pred_obj_1 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(0, 0, 9, 9),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.7)]))
# Will be a false positive (self._pred_obj_1 has higher IoU).
self._pred_obj_2 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(0, 0, 8, 8),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.6)]))
# Will match self._gt_obj_2
self._pred_obj_3 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(13, 13, 15, 15),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.1)]))
# More false positives.
self._pred_objs_fp = [
Label(obj_class=self._obj_class_pred,
geometry=Rectangle(20, 20, 30, 30),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=v / 100)]))
for v in range(15, 85, 10)]
self._metric_calculator = MAPMetric(class_mapping={'a': 'b'}, iou_threshold=0.5)
def _maybe_make_bbox_label(roi: Rectangle, bbox_class: ObjClass, tags=None) -> list:
return [Label(geometry=roi, obj_class=bbox_class, tags=tags)] if bbox_class is not None else []