def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() all_pixelwise_scores_labels = [] for roi in self._sliding_windows.get(ann.img_size): raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model) all_pixelwise_scores_labels.extend(raw_roi_ann.pixelwise_scores_labels) model_img_level_tags = make_renamed_tags(raw_roi_ann.img_tags, self._model_img_tag_meta_mapper, make_renamed_tags) result_ann = result_ann.add_labels( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags)) model_class_name_to_id = {name: idx for idx, name in enumerate(set(label.obj_class.name for label in all_pixelwise_scores_labels))} id_to_class_obj = {idx: self._model.model_out_meta.obj_classes.get(name) for name, idx in model_class_name_to_id.items()} summed_scores = np.zeros(ann.img_size + tuple([len(model_class_name_to_id)])) for label in all_pixelwise_scores_labels: class_idx = model_class_name_to_id[label.obj_class.name] label_matching_summer_scores = label.geometry.to_bbox().get_cropped_numpy_slice(summed_scores) label_matching_summer_scores[:, :, class_idx, np.newaxis] += label.geometry.data # TODO consider instead filtering pixels by all-zero scores. if np.sum(summed_scores, axis=2).min() == 0: raise RuntimeError('Wrong sliding window moving, implementation error.') aggregated_model_labels = raw_to_labels.segmentation_array_to_sly_bitmaps(id_to_class_obj, np.argmax(summed_scores, axis=2)) result_ann = result_ann.add_labels( replace_labels_classes(aggregated_model_labels, self._model_class_mapper, skip_missing=True)) return result_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() model_labels = [] for roi in self._sliding_windows.get(ann.img_size): raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model) all_rectangle_labels = [ label for label in raw_roi_ann.labels if isinstance(label.geometry, Rectangle) ] model_labels.extend( _replace_labels_classes(all_rectangle_labels, self._model_class_mapper, self._model_tag_meta_mapper, skip_missing=True)) model_img_level_tags = make_renamed_tags( raw_roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_labels( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags)) nms_conf = self._config.get(NMS_AFTER, {ENABLE: False}) if nms_conf[ENABLE]: result_ann = result_ann.add_labels( self._general_nms(labels=model_labels, iou_thresh=nms_conf[IOU_THRESHOLD], confidence_tag_name=nms_conf.get( CONFIDENCE_TAG_NAME, CONFIDENCE))) else: result_ann = result_ann.add_labels(model_labels) return result_ann
def verify_data(orig_ann: Annotation, classes_matching: dict, res_project_meta: ProjectMeta) -> Annotation: ann = orig_ann.clone() imsize = ann.img_size for first_class, second_class in classes_matching.items(): mask1 = np.zeros(imsize, dtype=np.bool) mask2 = np.zeros(imsize, dtype=np.bool) for label in ann.labels: if label.obj_class.name == first_class: label.geometry.draw(mask1, True) elif label.obj_class.name == second_class: label.geometry.draw(mask2, True) iou_value = _compute_masks_iou(mask1, mask2) tag_meta = res_project_meta.img_tag_metas.get(make_iou_tag_name(first_class)) tag = Tag(tag_meta, iou_value) ann.add_tag(tag) fp_mask = _create_fp_mask(mask1, mask2) if fp_mask.sum() != 0: fp_object_cls = res_project_meta.obj_classes.get(make_false_positive_name(first_class)) fp_geom = Bitmap(data=fp_mask) fp_label = Label(fp_geom, fp_object_cls) ann.add_label(fp_label) fn_mask = _create_fn_mask(mask1, mask2) if fn_mask.sum() != 0: fn_object_cls = res_project_meta.obj_classes.get(make_false_negative_name(first_class)) fn_geom = Bitmap(data=fn_mask) fn_label = Label(fn_geom, fn_object_cls) ann.add_label(fn_label) return ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() model_labels = [] roi_bbox_labels = [] for roi in self._sliding_windows.get(ann.img_size): raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model) # Accumulate all the labels across the sliding windows to potentially run non-max suppression over them. # Only retain the classes that will be eventually saved to avoid running NMS on objects we will # throw away anyway. model_labels.extend([ label for label in raw_roi_ann.labels if isinstance(label.geometry, Rectangle) and self._model_class_mapper.map(label.obj_class) is not None]) model_img_level_tags = make_renamed_tags( raw_roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) roi_bbox_labels.extend( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags)) nms_conf = self._config.get(NMS_AFTER, {ENABLE: False}) if nms_conf[ENABLE]: confidence_tag_name = nms_conf.get(CONFIDENCE_TAG_NAME, CONFIDENCE) model_labels = self._general_nms( labels=model_labels, iou_thresh=nms_conf[IOU_THRESHOLD], confidence_tag_name=confidence_tag_name) model_labels_renamed = _replace_or_drop_labels_classes( model_labels, self._model_class_mapper, self._model_tag_meta_mapper) result_ann = result_ann.add_labels(roi_bbox_labels + model_labels_renamed) return result_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_labels = [] for src_label, roi in self._all_filtered_bbox_rois( ann, self._config[FROM_CLASSES], self._config[PADDING]): if roi is None: result_labels.append(src_label) else: roi_ann = _get_annotation_for_bbox(img, roi, self._model) result_labels.extend( _replace_labels_classes(roi_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper, skip_missing=True)) model_img_level_tags = make_renamed_tags( roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) if self._config[SAVE]: result_labels.append( Label(geometry=roi, obj_class=self._intermediate_class_mapper.map( src_label.obj_class), tags=model_img_level_tags)) # Regardless of whether we need to save intermediate bounding boxes, also put the inference result tags # onto the original source object from which we created a bounding box. # This is necessary for e.g. classification models to work, so that they put the classification results # onto the original object. result_labels.append(src_label.add_tags(model_img_level_tags)) return ann.clone(labels=result_labels)
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() inference_result_ann = self._model.inference(img, ann) result_ann = result_ann.add_labels( replace_labels_classes(inference_result_ann.labels, self._model_class_mapper, skip_missing=True)) renamed_tags = make_renamed_tags(inference_result_ann.img_tags, self._model_img_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_tags(renamed_tags) return result_ann
def bitwise_mask( ann: Annotation, class_mask: str, classes_to_correct: List[str], bitwise_op: Callable[[np.ndarray, np.ndarray], np.ndarray] = np.logical_and ) -> Annotation: """ Performs bitwise operation between two masks. Uses one target mask to correct all others. Args ann: Input annotation. class_mask: Class name of target mask. classes_to_correct: List of classes which will be corrected using target mask. bitwise_op: Bitwise numpy function to process masks.For example: "np.logical_or", "np.logical_and", "np.logical_xor". Returns: Annotation containing corrected Bitmaps. """ imsize = ann.img_size def find_mask_class(labels, class_mask_name): for label in labels: if label.obj_class.name == class_mask_name: if not isinstance(label.geometry, Bitmap): raise RuntimeError( 'Class <{}> must be a Bitmap.'.format(class_mask_name)) return label mask_label = find_mask_class(ann.labels, class_mask) if mask_label is not None: target_original, target_mask = mask_label.geometry.origin, mask_label.geometry.data full_target_mask = np.full(imsize, False, bool) full_target_mask[target_original.row:target_original.row + target_mask.shape[0], target_original.col:target_original.col + target_mask.shape[1]] = target_mask def perform_op(label): if label.obj_class.name not in classes_to_correct or label.obj_class.name == class_mask: return [label] if not isinstance(label.geometry, Bitmap): raise RuntimeError('Input class must be a Bitmap.') new_geom = label.geometry.bitwise_mask(full_target_mask, bitwise_op) return [label.clone( geometry=new_geom)] if new_geom is not None else [] res_ann = ann.transform_labels(perform_op) else: res_ann = ann.clone() return res_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() roi = _make_cropped_rectangle(ann.img_size, self._config[BOUNDS]) roi_ann = _get_annotation_for_bbox(img, roi, self._model) result_ann = result_ann.add_labels( replace_labels_classes(roi_ann.labels, self._model_class_mapper, skip_missing=True)) img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_labels( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=roi_ann.img_tags)) result_ann = result_ann.add_tags(img_level_tags) return result_ann
def _do_infer_annotate_generic(self, inference_fn, img, ann: Annotation): result_ann = ann.clone() inference_result_ann = inference_fn(img, ann) result_ann = result_ann.add_labels( _replace_or_drop_labels_classes( inference_result_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper)) renamed_tags = make_renamed_tags(inference_result_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_tags(renamed_tags) return result_ann
def instance_crop(img: np.ndarray, ann: Annotation, class_title: str, save_other_classes_in_crop: bool = True, padding_config: dict = None) -> list: """ Crops objects of specified classes from image with configurable padding. Args: img: Input image array. ann: Input annotation. class_title: Name of class to crop. save_other_classes_in_crop: save non-target classes in each cropped annotation. padding_config: Dict with padding Returns: List of cropped [image, annotation] pairs. """ padding_config = take_with_default(padding_config, {}) _validate_image_annotation_shape(img, ann) results = [] img_rect = Rectangle.from_size(img.shape[:2]) if save_other_classes_in_crop: non_target_labels = [ label for label in ann.labels if label.obj_class.name != class_title ] else: non_target_labels = [] ann_with_non_target_labels = ann.clone(labels=non_target_labels) for label in ann.labels: if label.obj_class.name == class_title: src_fig_rect = label.geometry.to_bbox() new_img_rect = _rect_from_bounds(padding_config, img_w=src_fig_rect.width, img_h=src_fig_rect.height) rect_to_crop = new_img_rect.translate(src_fig_rect.top, src_fig_rect.left) crops = rect_to_crop.crop(img_rect) if len(crops) == 0: continue rect_to_crop = crops[0] image_crop = sly_image.crop(img, rect_to_crop) cropped_ann = ann_with_non_target_labels.relative_crop( rect_to_crop) label_crops = label.relative_crop(rect_to_crop) for label_crop in label_crops: results.append((image_crop, cropped_ann.add_label(label_crop))) return results
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_labels = [] for src_label, roi in self._all_filtered_bbox_rois(ann, self._config[FROM_CLASSES], self._config[PADDING]): if roi is None: result_labels.append(src_label) else: roi_ann = _get_annotation_for_bbox(img, roi, self._model) result_labels.extend(replace_labels_classes( roi_ann.labels, self._model_class_mapper, skip_missing=True)) model_img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_img_tag_meta_mapper, skip_missing=True) if self._config[SAVE]: result_labels.append( Label(geometry=roi, obj_class=self._intermediate_class_mapper.map(src_label.obj_class), tags=model_img_level_tags)) result_labels.append(src_label.add_tags(model_img_level_tags)) return ann.clone(labels=result_labels)
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() roi = _make_cropped_rectangle(ann.img_size, self._config[BOUNDS]) roi_ann = _get_annotation_for_bbox(img, roi, self._model) result_ann = result_ann.add_labels( _replace_or_drop_labels_classes(roi_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper)) img_level_tags = make_renamed_tags(roi_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_labels( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=img_level_tags)) result_ann = result_ann.add_tags(img_level_tags) if self._config.get(SAVE_PROBABILITIES, False) is True: result_problabels = _replace_or_drop_labels_classes( roi_ann.pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper) result_ann = result_ann.add_pixelwise_score_labels(result_problabels) return result_ann
def _do_infer_annotate_generic(self, inference_fn, img, ann: Annotation): result_ann = ann.clone() inference_ann = inference_fn(img, ann) result_labels = _replace_or_drop_labels_classes( inference_ann.labels, self._model_class_mapper, self._model_tag_meta_mapper) result_ann = result_ann.add_labels(result_labels) renamed_tags = make_renamed_tags(inference_ann.img_tags, self._model_tag_meta_mapper, skip_missing=True) result_ann = result_ann.add_tags(renamed_tags) if self._config.get(SAVE_PROBABILITIES, False) is True: result_problabels = _replace_or_drop_labels_classes( inference_ann.pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper) result_ann = result_ann.add_pixelwise_score_labels(result_problabels) return result_ann
def _do_infer_annotate(self, img: np.ndarray, ann: Annotation) -> Annotation: result_ann = ann.clone() all_pixelwise_scores_labels = [] for roi in self._sliding_windows.get(ann.img_size): raw_roi_ann = _get_annotation_for_bbox(img, roi, self._model) all_pixelwise_scores_labels.extend(raw_roi_ann.pixelwise_scores_labels) model_img_level_tags = make_renamed_tags(raw_roi_ann.img_tags, self._model_tag_meta_mapper, make_renamed_tags) result_ann = result_ann.add_labels( _maybe_make_bbox_label(roi, self._intermediate_bbox_class, tags=model_img_level_tags)) model_class_name_to_id = {name: idx for idx, name in enumerate(set(label.obj_class.name for label in all_pixelwise_scores_labels))} id_to_class_obj = {idx: self._model.model_out_meta.obj_classes.get(name) for name, idx in model_class_name_to_id.items()} summed_scores = np.zeros(ann.img_size + tuple([len(model_class_name_to_id)])) summed_divisor = np.zeros_like(summed_scores) for label in all_pixelwise_scores_labels: class_idx = model_class_name_to_id[label.obj_class.name] geom_bbox = label.geometry.to_bbox() label_matching_summer_scores = geom_bbox.get_cropped_numpy_slice(summed_scores) label_matching_summer_scores[:, :, class_idx, np.newaxis] += label.geometry.data divisor_slice = geom_bbox.get_cropped_numpy_slice(summed_divisor) divisor_slice[:, :, class_idx, np.newaxis] += 1. # TODO consider instead filtering pixels by all-zero scores. if np.sum(summed_scores, axis=2).min() == 0: raise RuntimeError('Wrong sliding window moving, implementation error.') aggregated_model_labels = raw_to_labels.segmentation_array_to_sly_bitmaps(id_to_class_obj, np.argmax(summed_scores, axis=2)) result_ann = result_ann.add_labels( _replace_or_drop_labels_classes( aggregated_model_labels, self._model_class_mapper, self._model_tag_meta_mapper)) if self._config.get(SAVE_PROBABILITIES, False) is True: # copied fom unet's inference.py mean_scores = summed_scores / summed_divisor accumulated_pixelwise_scores_labels = raw_to_labels.segmentation_scores_to_per_class_labels( id_to_class_obj, mean_scores) result_problabels = _replace_or_drop_labels_classes( accumulated_pixelwise_scores_labels, self._model_class_mapper, self._model_tag_meta_mapper) result_ann = result_ann.add_pixelwise_score_labels(result_problabels) return result_ann