def detection_preds_to_sly_rects(
idx_to_class, network_prediction: DetectionNetworkPrediction,
img_shape, min_score_threshold, score_tag_meta) -> list:
"""
Converts network detection results to Supervisely Labels with Rectangle geometry.
Args:
idx_to_class: Dict matching predicted boxes with appropriate ObjClass.
network_prediction: Network predictions packed into DetectionNetworkPrediction instance.
img_shape: Size(height, width) of image that was used for inference.
min_score_threshold: All detections with less scores will be dropped.
score_tag_meta: TagMeta instance for score tags.
Returns:
A list containing labels with detection rectangles.
"""
labels = []
thr_mask = np.squeeze(network_prediction.scores) > min_score_threshold
for box, class_id, score in zip(
np.squeeze(network_prediction.boxes)[thr_mask],
np.squeeze(network_prediction.classes)[thr_mask],
np.squeeze(network_prediction.scores)[thr_mask]):
xmin = round(float(box[1] * img_shape[1]))
ymin = round(float(box[0] * img_shape[0]))
xmax = round(float(box[3] * img_shape[1]))
ymax = round(float(box[2] * img_shape[0]))
rect = Rectangle(top=ymin, left=xmin, bottom=ymax, right=xmax)
class_obj = idx_to_class[int(class_id)]
label = Label(geometry=rect, obj_class=class_obj)
score_tag = Tag(score_tag_meta, value=round(float(score), 4))
label = label.add_tag(score_tag)
labels.append(label)
return labels
def infer_on_image(image,
graph,
model,
idx_to_class_title,
project_meta,
confidence_tag_meta,
min_confidence=0):
with graph.as_default():
[results] = model.detect([image], verbose=0)
res_labels = []
for mask_idx, class_id in enumerate(results['class_ids']):
confidence = results['scores'][mask_idx]
if confidence < min_confidence:
continue
bool_mask = results['masks'][:, :, mask_idx] != 0
class_geometry = Bitmap(data=bool_mask)
cls_title = idx_to_class_title[class_id]
label = Label(geometry=class_geometry,
obj_class=project_meta.get_obj_class(cls_title))
confidence_tag = Tag(confidence_tag_meta,
value=round(float(confidence), 4))
label = label.add_tag(confidence_tag)
res_labels.append(label)
return res_labels
def verify_data(orig_ann: Annotation, classes_matching: dict, res_project_meta: ProjectMeta) -> Annotation:
ann = orig_ann.clone()
imsize = ann.img_size
for first_class, second_class in classes_matching.items():
mask1 = np.zeros(imsize, dtype=np.bool)
mask2 = np.zeros(imsize, dtype=np.bool)
for label in ann.labels:
if label.obj_class.name == first_class:
label.geometry.draw(mask1, True)
elif label.obj_class.name == second_class:
label.geometry.draw(mask2, True)
iou_value = _compute_masks_iou(mask1, mask2)
tag_meta = res_project_meta.img_tag_metas.get(make_iou_tag_name(first_class))
tag = Tag(tag_meta, iou_value)
ann.add_tag(tag)
fp_mask = _create_fp_mask(mask1, mask2)
if fp_mask.sum() != 0:
fp_object_cls = res_project_meta.obj_classes.get(make_false_positive_name(first_class))
fp_geom = Bitmap(data=fp_mask)
fp_label = Label(fp_geom, fp_object_cls)
ann.add_label(fp_label)
fn_mask = _create_fn_mask(mask1, mask2)
if fn_mask.sum() != 0:
fn_object_cls = res_project_meta.obj_classes.get(make_false_negative_name(first_class))
fn_geom = Bitmap(data=fn_mask)
fn_label = Label(fn_geom, fn_object_cls)
ann.add_label(fn_label)
return ann
def from_json(cls, data, project_meta):
'''
The function from_json convert annotation from json format to Annotation class object. If one of the labels
of annotation in json format cannot be convert to Label class object it generate exception error.
:param data: input annotation in json format
:param project_meta: ProjectMeta class object
:return: Annotation class object
'''
img_size_dict = data[AnnotationJsonFields.IMG_SIZE]
img_height = img_size_dict[AnnotationJsonFields.IMG_SIZE_HEIGHT]
img_width = img_size_dict[AnnotationJsonFields.IMG_SIZE_WIDTH]
img_size = (img_height, img_width)
try:
labels = [
Label.from_json(label_json, project_meta)
for label_json in data[AnnotationJsonFields.LABELS]
]
except Exception:
logger.fatal(
'Failed to deserialize annotation from JSON format. One of the Label objects could not be '
'deserialized')
raise
custom_data = data.get(AnnotationJsonFields.CUSTOM_DATA, {})
prob_labels = None
if AnnotationJsonFields.PROBABILITY_LABELS in custom_data and \
AnnotationJsonFields.PROBABILITY_CLASSES in custom_data:
prob_classes = ObjClassCollection.from_json(
custom_data[AnnotationJsonFields.PROBABILITY_CLASSES])
# @TODO: tony, maybe link with project meta (add probability classes???)
prob_project_meta = ProjectMeta(obj_classes=prob_classes)
prob_labels = [
Label.from_json(label_json, prob_project_meta) for label_json
in custom_data[AnnotationJsonFields.PROBABILITY_LABELS]
]
custom_data.pop(AnnotationJsonFields.PROBABILITY_CLASSES)
custom_data.pop(AnnotationJsonFields.PROBABILITY_LABELS)
return cls(img_size=img_size,
labels=labels,
img_tags=TagCollection.from_json(
data[AnnotationJsonFields.IMG_TAGS],
project_meta.tag_metas),
img_description=data.get(
AnnotationJsonFields.IMG_DESCRIPTION, ""),
pixelwise_scores_labels=prob_labels,
custom_data=custom_data)
def _do_infer_annotate(self, img: np.ndarray,
ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(
ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(
_replace_labels_classes(roi_ann.labels,
self._model_class_mapper,
self._model_tag_meta_mapper,
skip_missing=True))
model_img_level_tags = make_renamed_tags(
roi_ann.img_tags,
self._model_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi,
obj_class=self._intermediate_class_mapper.map(
src_label.obj_class),
tags=model_img_level_tags))
# Regardless of whether we need to save intermediate bounding boxes, also put the inference result tags
# onto the original source object from which we created a bounding box.
# This is necessary for e.g. classification models to work, so that they put the classification results
# onto the original object.
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def extract_labels_from_mask(mask: np.ndarray, color_id_to_obj_class: collections.Mapping) -> list:
"""
Extract multiclass instances from grayscale mask and save it to labels list.
Args:
mask: multiclass grayscale mask
color_id_to_obj_class: dict of objects classes assigned to color id (e.g. {1: ObjClass('cat), ...})
Returns:
list of labels with bitmap geometry
"""
zero_offset = 1 if 0 in color_id_to_obj_class else 0
if zero_offset > 0:
mask = mask + zero_offset
labeled, labels_count = measure.label(mask, connectivity=1, return_num=True)
objects_slices = ndimage.find_objects(labeled)
labels = []
for object_index, slices in enumerate(objects_slices, start=1):
crop = mask[slices]
sub_mask = crop * (labeled[slices] == object_index).astype(np.int)
class_index = np.max(sub_mask) - zero_offset
if class_index in color_id_to_obj_class:
bitmap = Bitmap(data=sub_mask.astype(np.bool), origin=PointLocation(slices[0].start, slices[1].start))
label = Label(geometry=bitmap, obj_class=color_id_to_obj_class.get(class_index))
labels.append(label)
return labels
def to_contours(label: Label):
new_obj_cls = classes_mapping.get(label.obj_class.name)
if new_obj_cls is None:
return [label]
if not isinstance(label.geometry, Bitmap):
raise RuntimeError('Input class must be a Bitmap.')
return [Label(geometry=geom, obj_class=new_obj_cls) for geom in label.geometry.to_contours()]
def _skel(label: Label):
if label.obj_class.name not in classes:
return [label]
if not isinstance(label.geometry, Bitmap):
raise RuntimeError('Input class must be a Bitmap.')
return [label.clone(geometry=label.geometry.skeletonize(method_id))]
def _approx(label: Label):
if label.obj_class.name not in classes:
return [label]
if not isinstance(label.geometry, (Polygon, Polyline)):
raise RuntimeError('Input class must be a Polygon or a Line.')
return [label.clone(geometry=label.geometry.approx_dp(epsilon))]
def from_json(cls, data, project_meta):
img_size_dict = data[AnnotationJsonFields.IMG_SIZE]
img_height = img_size_dict[AnnotationJsonFields.IMG_SIZE_HEIGHT]
img_width = img_size_dict[AnnotationJsonFields.IMG_SIZE_WIDTH]
img_size = (img_height, img_width)
labels = [Label.from_json(label_json, project_meta) for label_json in data[AnnotationJsonFields.LABELS]]
return cls(img_size=img_size,
labels=labels,
img_tags=TagCollection.from_json(data[AnnotationJsonFields.IMG_TAGS], project_meta.img_tag_metas),
img_description=data.get(AnnotationJsonFields.IMG_DESCRIPTION, ""))
def from_imgaug(cls,
img,
ia_boxes=None,
ia_masks=None,
index_to_class=None,
meta: ProjectMeta = None):
if ((ia_boxes is not None) or (ia_masks is not None)) and meta is None:
raise ValueError("Project meta has to be provided")
labels = []
if ia_boxes is not None:
for ia_box in ia_boxes:
obj_class = meta.get_obj_class(ia_box.label)
if obj_class is None:
raise KeyError(
"Class {!r} not found in project meta".format(
ia_box.label))
lbl = Label(
Rectangle(top=ia_box.y1,
left=ia_box.x1,
bottom=ia_box.y2,
right=ia_box.x2), obj_class)
labels.append(lbl)
if ia_masks is not None:
if index_to_class is None:
raise ValueError(
"mapping from index to class name is needed to transform masks to SLY format"
)
class_mask = ia_masks.get_arr()
# mask = white_mask == 255
(unique, counts) = np.unique(class_mask, return_counts=True)
for index, count in zip(unique, counts):
if index == 0:
continue
mask = class_mask == index
bitmap = Bitmap(data=mask[:, :, 0])
restore_class = meta.get_obj_class(index_to_class[index])
labels.append(Label(geometry=bitmap, obj_class=restore_class))
return cls(img_size=img.shape[:2], labels=labels)
def to_segmentation_task(self):
class_mask = {}
for label in self.labels:
if label.obj_class not in class_mask:
class_mask[label.obj_class] = np.zeros(self.img_size, np.uint8)
label.draw(class_mask[label.obj_class], color=255)
new_labels = []
for obj_class, white_mask in class_mask.items():
mask = white_mask == 255
bitmap = Bitmap(data=mask)
new_labels.append(Label(geometry=bitmap, obj_class=obj_class))
return self.clone(labels=new_labels)
def add_background(ann: Annotation, bg_class: ObjClass) -> Annotation:
"""
Adds background rectangle (size equals to image size) to annotation.
Args:
ann: Input annotation.
bg_class: ObjClass instance for background class label.
Returns:
Annotation with added background rectangle.
"""
img_size = ann.img_size
rect = Rectangle(0, 0, img_size[0] - 1, img_size[1] - 1)
new_label = Label(rect, bg_class)
return ann.add_label(new_label)
def from_json(cls, data, project_meta):
img_size_dict = data[AnnotationJsonFields.IMG_SIZE]
img_height = img_size_dict[AnnotationJsonFields.IMG_SIZE_HEIGHT]
img_width = img_size_dict[AnnotationJsonFields.IMG_SIZE_WIDTH]
img_size = (img_height, img_width)
try:
labels = [Label.from_json(label_json, project_meta) for label_json in data[AnnotationJsonFields.LABELS]]
except Exception:
logger.fatal('Failed to deserialize annotation from JSON format. One of the Label objects could not be '
'deserialized')
raise
return cls(img_size=img_size,
labels=labels,
img_tags=TagCollection.from_json(data[AnnotationJsonFields.IMG_TAGS], project_meta.tag_metas),
img_description=data.get(AnnotationJsonFields.IMG_DESCRIPTION, ""))
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(replace_labels_classes(
roi_ann.labels, self._model_class_mapper, skip_missing=True))
model_img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi, obj_class=self._intermediate_class_mapper.map(src_label.obj_class),
tags=model_img_level_tags))
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def segmentation_array_to_sly_bitmaps(idx_to_class: dict,
pred: np.ndarray,
origin: PointLocation = None) -> list:
"""
Converts array with segmentation results to Labels with Bitmap geometry according to idx_to_class mapping.
Args:
idx_to_class: Dict matching values in prediction array with appropriate ObjClass.
pred: Array containing raw segmentation results.
origin: Origin point for all output Bitmaps.
return:
A list containing result labels.
"""
labels = []
for cls_idx, cls_obj in idx_to_class.items():
predicted_class_pixels = (pred == cls_idx)
if np.any(predicted_class_pixels):
class_geometry = Bitmap(data=predicted_class_pixels, origin=origin)
labels.append(Label(geometry=class_geometry, obj_class=cls_obj))
return labels
def setUp(self):
self._obj_class_gt = ObjClass(name='a', geometry_type=Rectangle)
self._obj_class_pred = ObjClass(name='b', geometry_type=Rectangle)
self._confidence_tag_meta = TagMeta(name='confidence', value_type=TagValueType.ANY_NUMBER)
self._meta = ProjectMeta(
obj_classes=ObjClassCollection([self._obj_class_gt, self._obj_class_pred]),
tag_metas=TagMetaCollection([self._confidence_tag_meta]))
# Will match self._pred_obj_1
self._gt_obj_1 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(0, 0, 10, 10))
# Will match self._pred_obj_3
self._gt_obj_2 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(13, 13, 15, 15))
# Will be a false negative
self._gt_obj_3 = Label(obj_class=self._obj_class_gt, geometry=Rectangle(43, 43, 45, 45))
# Will match self._gt_obj_1
self._pred_obj_1 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(0, 0, 9, 9),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.7)]))
# Will be a false positive (self._pred_obj_1 has higher IoU).
self._pred_obj_2 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(0, 0, 8, 8),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.6)]))
# Will match self._gt_obj_2
self._pred_obj_3 = Label(
obj_class=self._obj_class_pred,
geometry=Rectangle(13, 13, 15, 15),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=0.1)]))
# More false positives.
self._pred_objs_fp = [
Label(obj_class=self._obj_class_pred,
geometry=Rectangle(20, 20, 30, 30),
tags=TagCollection([Tag(meta=self._confidence_tag_meta, value=v / 100)]))
for v in range(15, 85, 10)]
self._metric_calculator = MAPMetric(class_mapping={'a': 'b'}, iou_threshold=0.5)
def _maybe_make_bbox_label(roi: Rectangle, bbox_class: ObjClass, tags=None) -> list:
return [Label(geometry=roi, obj_class=bbox_class, tags=tags)] if bbox_class is not None else []
