def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
all_pixelwise_scores_labels = []
for roi in self._sliding_windows.get(ann.img_size):
raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model)
all_pixelwise_scores_labels.extend(raw_roi_ann.pixelwise_scores_labels)
model_img_level_tags = make_renamed_tags(raw_roi_ann.img_tags, self._model_img_tag_meta_mapper,
make_renamed_tags)
result_ann = result_ann.add_labels(
_maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags))
model_class_name_to_id = {name: idx
for idx, name in enumerate(set(label.obj_class.name
for label in all_pixelwise_scores_labels))}
id_to_class_obj = {idx: self._model.model_out_meta.obj_classes.get(name)
for name, idx in model_class_name_to_id.items()}
summed_scores = np.zeros(ann.img_size + tuple([len(model_class_name_to_id)]))
for label in all_pixelwise_scores_labels:
class_idx = model_class_name_to_id[label.obj_class.name]
label_matching_summer_scores = label.geometry.to_bbox().get_cropped_numpy_slice(summed_scores)
label_matching_summer_scores[:, :, class_idx, np.newaxis] += label.geometry.data
# TODO consider instead filtering pixels by all-zero scores.
if np.sum(summed_scores, axis=2).min() == 0:
raise RuntimeError('Wrong sliding window moving, implementation error.')
aggregated_model_labels = raw_to_labels.segmentation_array_to_sly_bitmaps(id_to_class_obj,
np.argmax(summed_scores, axis=2))
result_ann = result_ann.add_labels(
replace_labels_classes(aggregated_model_labels, self._model_class_mapper, skip_missing=True))
return result_ann
def _do_infer_annotate(self, img: np.ndarray,
ann: Annotation) -> Annotation:
result_ann = ann.clone()
model_labels = []
for roi in self._sliding_windows.get(ann.img_size):
raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model)
all_rectangle_labels = [
label for label in raw_roi_ann.labels
if isinstance(label.geometry, Rectangle)
]
model_labels.extend(
_replace_labels_classes(all_rectangle_labels,
self._model_class_mapper,
self._model_tag_meta_mapper,
skip_missing=True))
model_img_level_tags = make_renamed_tags(
raw_roi_ann.img_tags,
self._model_tag_meta_mapper,
skip_missing=True)
result_ann = result_ann.add_labels(
_maybe_make_bbox_label(roi,
self._intermediate_bbox_class,
tags=model_img_level_tags))
nms_conf = self._config.get(NMS_AFTER, {ENABLE: False})
if nms_conf[ENABLE]:
result_ann = result_ann.add_labels(
self._general_nms(labels=model_labels,
iou_thresh=nms_conf[IOU_THRESHOLD],
confidence_tag_name=nms_conf.get(
CONFIDENCE_TAG_NAME, CONFIDENCE)))
else:
result_ann = result_ann.add_labels(model_labels)
return result_ann
def verify_data(orig_ann: Annotation, classes_matching: dict, res_project_meta: ProjectMeta) -> Annotation:
ann = orig_ann.clone()
imsize = ann.img_size
for first_class, second_class in classes_matching.items():
mask1 = np.zeros(imsize, dtype=np.bool)
mask2 = np.zeros(imsize, dtype=np.bool)
for label in ann.labels:
if label.obj_class.name == first_class:
label.geometry.draw(mask1, True)
elif label.obj_class.name == second_class:
label.geometry.draw(mask2, True)
iou_value = _compute_masks_iou(mask1, mask2)
tag_meta = res_project_meta.img_tag_metas.get(make_iou_tag_name(first_class))
tag = Tag(tag_meta, iou_value)
ann.add_tag(tag)
fp_mask = _create_fp_mask(mask1, mask2)
if fp_mask.sum() != 0:
fp_object_cls = res_project_meta.obj_classes.get(make_false_positive_name(first_class))
fp_geom = Bitmap(data=fp_mask)
fp_label = Label(fp_geom, fp_object_cls)
ann.add_label(fp_label)
fn_mask = _create_fn_mask(mask1, mask2)
if fn_mask.sum() != 0:
fn_object_cls = res_project_meta.obj_classes.get(make_false_negative_name(first_class))
fn_geom = Bitmap(data=fn_mask)
fn_label = Label(fn_geom, fn_object_cls)
ann.add_label(fn_label)
return ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
model_labels = []
roi_bbox_labels = []
for roi in self._sliding_windows.get(ann.img_size):
raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model)
# Accumulate all the labels across the sliding windows to potentially run non-max suppression over them.
# Only retain the classes that will be eventually saved to avoid running NMS on objects we will
# throw away anyway.
model_labels.extend([
label for label in raw_roi_ann.labels
if isinstance(label.geometry, Rectangle) and self._model_class_mapper.map(label.obj_class) is not None])
model_img_level_tags = make_renamed_tags(
raw_roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True)
roi_bbox_labels.extend(
_maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags))
nms_conf = self._config.get(NMS_AFTER, {ENABLE: False})
if nms_conf[ENABLE]:
confidence_tag_name = nms_conf.get(CONFIDENCE_TAG_NAME, CONFIDENCE)
model_labels = self._general_nms(
labels=model_labels, iou_thresh=nms_conf[IOU_THRESHOLD], confidence_tag_name=confidence_tag_name)
model_labels_renamed = _replace_or_drop_labels_classes(
model_labels, self._model_class_mapper, self._model_tag_meta_mapper)
result_ann = result_ann.add_labels(roi_bbox_labels + model_labels_renamed)
return result_ann
def _do_infer_annotate(self, img: np.ndarray,
ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(
ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(
_replace_labels_classes(roi_ann.labels,
self._model_class_mapper,
self._model_tag_meta_mapper,
skip_missing=True))
model_img_level_tags = make_renamed_tags(
roi_ann.img_tags,
self._model_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi,
obj_class=self._intermediate_class_mapper.map(
src_label.obj_class),
tags=model_img_level_tags))
# Regardless of whether we need to save intermediate bounding boxes, also put the inference result tags
# onto the original source object from which we created a bounding box.
# This is necessary for e.g. classification models to work, so that they put the classification results
# onto the original object.
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
inference_result_ann = self._model.inference(img, ann)
result_ann = result_ann.add_labels(
replace_labels_classes(inference_result_ann.labels, self._model_class_mapper, skip_missing=True))
renamed_tags = make_renamed_tags(inference_result_ann.img_tags,
self._model_img_tag_meta_mapper,
skip_missing=True)
result_ann = result_ann.add_tags(renamed_tags)
return result_ann
def bitwise_mask(
ann: Annotation,
class_mask: str,
classes_to_correct: List[str],
bitwise_op: Callable[[np.ndarray, np.ndarray], np.ndarray] = np.logical_and
) -> Annotation:
"""
Performs bitwise operation between two masks. Uses one target mask to correct all others.
Args
ann: Input annotation.
class_mask: Class name of target mask.
classes_to_correct: List of classes which will be corrected using target mask.
bitwise_op: Bitwise numpy function to process masks.For example: "np.logical_or", "np.logical_and",
"np.logical_xor".
Returns:
Annotation containing corrected Bitmaps.
"""
imsize = ann.img_size
def find_mask_class(labels, class_mask_name):
for label in labels:
if label.obj_class.name == class_mask_name:
if not isinstance(label.geometry, Bitmap):
raise RuntimeError(
'Class <{}> must be a Bitmap.'.format(class_mask_name))
return label
mask_label = find_mask_class(ann.labels, class_mask)
if mask_label is not None:
target_original, target_mask = mask_label.geometry.origin, mask_label.geometry.data
full_target_mask = np.full(imsize, False, bool)
full_target_mask[target_original.row:target_original.row +
target_mask.shape[0],
target_original.col:target_original.col +
target_mask.shape[1]] = target_mask
def perform_op(label):
if label.obj_class.name not in classes_to_correct or label.obj_class.name == class_mask:
return [label]
if not isinstance(label.geometry, Bitmap):
raise RuntimeError('Input class must be a Bitmap.')
new_geom = label.geometry.bitwise_mask(full_target_mask,
bitwise_op)
return [label.clone(
geometry=new_geom)] if new_geom is not None else []
res_ann = ann.transform_labels(perform_op)
else:
res_ann = ann.clone()
return res_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
roi = _make_cropped_rectangle(ann.img_size, self._config[BOUNDS])
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_ann = result_ann.add_labels(
replace_labels_classes(roi_ann.labels, self._model_class_mapper, skip_missing=True))
img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper, skip_missing=True)
result_ann = result_ann.add_labels(
_maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=roi_ann.img_tags))
result_ann = result_ann.add_tags(img_level_tags)
return result_ann
def _do_infer_annotate_generic(self, inference_fn, img, ann: Annotation):
result_ann = ann.clone()
inference_result_ann = inference_fn(img, ann)
result_ann = result_ann.add_labels(
_replace_or_drop_labels_classes(
inference_result_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper))
renamed_tags = make_renamed_tags(inference_result_ann.img_tags,
self._model_tag_meta_mapper,
skip_missing=True)
result_ann = result_ann.add_tags(renamed_tags)
return result_ann
def instance_crop(img: np.ndarray,
ann: Annotation,
class_title: str,
save_other_classes_in_crop: bool = True,
padding_config: dict = None) -> list:
"""
Crops objects of specified classes from image with configurable padding.
Args:
img: Input image array.
ann: Input annotation.
class_title: Name of class to crop.
save_other_classes_in_crop: save non-target classes in each cropped annotation.
padding_config: Dict with padding
Returns:
List of cropped [image, annotation] pairs.
"""
padding_config = take_with_default(padding_config, {})
_validate_image_annotation_shape(img, ann)
results = []
img_rect = Rectangle.from_size(img.shape[:2])
if save_other_classes_in_crop:
non_target_labels = [
label for label in ann.labels
if label.obj_class.name != class_title
]
else:
non_target_labels = []
ann_with_non_target_labels = ann.clone(labels=non_target_labels)
for label in ann.labels:
if label.obj_class.name == class_title:
src_fig_rect = label.geometry.to_bbox()
new_img_rect = _rect_from_bounds(padding_config,
img_w=src_fig_rect.width,
img_h=src_fig_rect.height)
rect_to_crop = new_img_rect.translate(src_fig_rect.top,
src_fig_rect.left)
crops = rect_to_crop.crop(img_rect)
if len(crops) == 0:
continue
rect_to_crop = crops[0]
image_crop = sly_image.crop(img, rect_to_crop)
cropped_ann = ann_with_non_target_labels.relative_crop(
rect_to_crop)
label_crops = label.relative_crop(rect_to_crop)
for label_crop in label_crops:
results.append((image_crop, cropped_ann.add_label(label_crop)))
return results
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_labels = []
for src_label, roi in self._all_filtered_bbox_rois(ann, self._config[FROM_CLASSES], self._config[PADDING]):
if roi is None:
result_labels.append(src_label)
else:
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_labels.extend(replace_labels_classes(
roi_ann.labels, self._model_class_mapper, skip_missing=True))
model_img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper,
skip_missing=True)
if self._config[SAVE]:
result_labels.append(
Label(geometry=roi, obj_class=self._intermediate_class_mapper.map(src_label.obj_class),
tags=model_img_level_tags))
result_labels.append(src_label.add_tags(model_img_level_tags))
return ann.clone(labels=result_labels)
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
roi = _make_cropped_rectangle(ann.img_size, self._config[BOUNDS])
roi_ann = _get_annotation_for_bbox(img, roi, self._model)
result_ann = result_ann.add_labels(
_replace_or_drop_labels_classes(roi_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper))
img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True)
result_ann = result_ann.add_labels(
_maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=img_level_tags))
result_ann = result_ann.add_tags(img_level_tags)
if self._config.get(SAVE_PROBABILITIES, False) is True:
result_problabels = _replace_or_drop_labels_classes(
roi_ann.pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper)
result_ann = result_ann.add_pixelwise_score_labels(result_problabels)
return result_ann
def _do_infer_annotate_generic(self, inference_fn, img, ann: Annotation):
result_ann = ann.clone()
inference_ann = inference_fn(img, ann)
result_labels = _replace_or_drop_labels_classes(
inference_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper)
result_ann = result_ann.add_labels(result_labels)
renamed_tags = make_renamed_tags(inference_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True)
result_ann = result_ann.add_tags(renamed_tags)
if self._config.get(SAVE_PROBABILITIES, False) is True:
result_problabels = _replace_or_drop_labels_classes(
inference_ann.pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper)
result_ann = result_ann.add_pixelwise_score_labels(result_problabels)
return result_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation:
result_ann = ann.clone()
all_pixelwise_scores_labels = []
for roi in self._sliding_windows.get(ann.img_size):
raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model)
all_pixelwise_scores_labels.extend(raw_roi_ann.pixelwise_scores_labels)
model_img_level_tags = make_renamed_tags(raw_roi_ann.img_tags, self._model_tag_meta_mapper,
make_renamed_tags)
result_ann = result_ann.add_labels(
_maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags))
model_class_name_to_id = {name: idx
for idx, name in enumerate(set(label.obj_class.name
for label in all_pixelwise_scores_labels))}
id_to_class_obj = {idx: self._model.model_out_meta.obj_classes.get(name)
for name, idx in model_class_name_to_id.items()}
summed_scores = np.zeros(ann.img_size + tuple([len(model_class_name_to_id)]))
summed_divisor = np.zeros_like(summed_scores)
for label in all_pixelwise_scores_labels:
class_idx = model_class_name_to_id[label.obj_class.name]
geom_bbox = label.geometry.to_bbox()
label_matching_summer_scores = geom_bbox.get_cropped_numpy_slice(summed_scores)
label_matching_summer_scores[:, :, class_idx, np.newaxis] += label.geometry.data
divisor_slice = geom_bbox.get_cropped_numpy_slice(summed_divisor)
divisor_slice[:, :, class_idx, np.newaxis] += 1.
# TODO consider instead filtering pixels by all-zero scores.
if np.sum(summed_scores, axis=2).min() == 0:
raise RuntimeError('Wrong sliding window moving, implementation error.')
aggregated_model_labels = raw_to_labels.segmentation_array_to_sly_bitmaps(id_to_class_obj,
np.argmax(summed_scores, axis=2))
result_ann = result_ann.add_labels(
_replace_or_drop_labels_classes(
aggregated_model_labels, self._model_class_mapper, self._model_tag_meta_mapper))
if self._config.get(SAVE_PROBABILITIES, False) is True:
# copied fom unet's inference.py
mean_scores = summed_scores / summed_divisor
accumulated_pixelwise_scores_labels = raw_to_labels.segmentation_scores_to_per_class_labels(
id_to_class_obj, mean_scores)
result_problabels = _replace_or_drop_labels_classes(
accumulated_pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper)
result_ann = result_ann.add_pixelwise_score_labels(result_problabels)
return result_ann