def merge_classes(cur_meta: sly.ProjectMeta, res_meta: sly.ProjectMeta, ann: sly.Annotation):
existing_names = set([obj_class.name for obj_class in res_meta.obj_classes])
mapping = {}
for obj_class in cur_meta.obj_classes:
obj_class: sly.ObjClass
if obj_class.name in mapping:
continue
dest_class = res_meta.get_obj_class(obj_class.name)
if dest_class is None:
res_meta = res_meta.add_obj_class(obj_class)
dest_class = obj_class
elif obj_class != dest_class:
new_name = generate_free_name(existing_names, obj_class.name)
dest_class = obj_class.clone(name=new_name)
res_meta = res_meta.add_obj_class(dest_class)
mapping[obj_class.name] = dest_class
new_labels = []
for label in ann.labels:
if label.obj_class.name not in mapping:
new_labels.append(label)
else:
new_labels.append(label.clone(obj_class=mapping[label.obj_class.name]))
new_ann = ann.clone(labels=new_labels)
return (res_meta, res_meta, new_ann)
def _check_and_add_class(meta: sly.ProjectMeta,
target_class: sly.ObjClass):
cls: sly.ObjClass = meta.get_obj_class(target_class.name)
if cls is None:
meta = meta.add_obj_class(target_class)
elif cls != target_class:
raise RuntimeError(
f"Project already has class '{cls.name}' with shape {cls.geometry_type.geometry_name()}. "
f"Shape conflict: shape should be of typy {target_class.geometry_type.geometry_name()}"
)
return meta
def convert_to_nonoverlapping(meta: sly.ProjectMeta, ann: sly.Annotation) -> (sly.ProjectMeta, sly.Annotation):
common_img = np.zeros(ann.img_size, np.int32) # size is (h, w)
for idx, lbl in enumerate(ann.labels, start=1):
if need_convert(lbl.obj_class.geometry_type):
if allow_render_for_any_shape(lbl) is True:
lbl.draw(common_img, color=idx)
else:
sly.logger.warn(
"Object of class {!r} (shape: {!r}) has non spatial shape {!r}. It will not be rendered."
.format(lbl.obj_class.name,
lbl.obj_class.geometry_type.geometry_name(),
lbl.geometry.geometry_name()))
new_classes = sly.ObjClassCollection()
new_labels = []
for idx, lbl in enumerate(ann.labels, start=1):
if not need_convert(lbl.obj_class.geometry_type):
new_labels.append(lbl.clone())
else:
if allow_render_for_any_shape(lbl) is False:
continue
# @TODO: get part of the common_img for speedup
mask = common_img == idx
if np.any(mask): # figure may be entirely covered by others
g = lbl.geometry
new_bmp = sly.Bitmap(data=mask)
if new_classes.get(lbl.obj_class.name) is None:
new_classes = new_classes.add(lbl.obj_class.clone(geometry_type=sly.Bitmap))
new_lbl = lbl.clone(geometry=new_bmp, obj_class=new_classes.get(lbl.obj_class.name))
new_labels.append(new_lbl)
new_meta = meta.clone(obj_classes=new_classes)
new_ann = ann.clone(labels=new_labels)
return (new_meta, new_ann)
def get_empty_gallery(meta: sly.ProjectMeta = sly.ProjectMeta()):
CNT_GRID_COLUMNS = 2
empty_gallery = {
"content": {
"projectMeta": meta.to_json(),
"annotations": {},
"layout": [[] for i in range(CNT_GRID_COLUMNS)]
},
}
return CNT_GRID_COLUMNS, empty_gallery
def inference(model, half, device, imgsz, stride, image: np.ndarray, meta: sly.ProjectMeta, conf_thres=0.25, iou_thres=0.45,
augment=False, agnostic_nms=False, debug_visualization=False) -> sly.Annotation:
names = model.module.names if hasattr(model, 'module') else model.names
img0 = image # RGB
# Padded resize
img = letterbox(img0, new_shape=imgsz, stride=stride)[0]
img = img.transpose(2, 0, 1) # to 3x416x416
img = np.ascontiguousarray(img)
img = torch.from_numpy(img).to(device)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
inf_out = model(img, augment=augment)[0]
# Apply NMS
labels = []
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, agnostic=agnostic_nms)
for i, det in enumerate(output):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape).round()
for *xyxy, conf, cls in reversed(det):
top, left, bottom, right = int(xyxy[1]), int(xyxy[0]), int(xyxy[3]), int(xyxy[2])
rect = sly.Rectangle(top, left, bottom, right)
obj_class = meta.get_obj_class(names[int(cls)])
tag = sly.Tag(meta.get_tag_meta(CONFIDENCE), round(float(conf), 4))
label = sly.Label(rect, obj_class, sly.TagCollection([tag]))
labels.append(label)
height, width = img0.shape[:2]
ann = sly.Annotation(img_size=(height, width), labels=labels)
if debug_visualization is True:
# visualize for debug purposes
vis = np.copy(img0)
ann.draw_contour(vis, thickness=2)
sly.image.write("vis.jpg", vis)
return ann.to_json()
def highlight_instances(meta: sly.ProjectMeta, ann: sly.Annotation) -> (sly.ProjectMeta, sly.Annotation):
new_classes = []
new_labels = []
for idx, label in enumerate(ann.labels):
new_cls = label.obj_class.clone(name=str(idx), color=sly.color.random_rgb())
new_lbl = label.clone(obj_class=new_cls)
new_classes.append(new_cls)
new_labels.append(new_lbl)
res_meta = meta.clone(obj_classes=sly.ObjClassCollection(new_classes))
res_ann = ann.clone(labels=new_labels)
return (res_meta, res_ann)
def transform_for_instance_segmentation(meta: sly.ProjectMeta, ann: sly.Annotation) -> (sly.ProjectMeta, sly.Annotation):
new_classes = {}
for obj_class in meta.obj_classes:
obj_class: sly.ObjClass
new_class = obj_class.clone(name=obj_class.name + "-mask")
new_classes[obj_class.name] = new_class
new_class_collection = sly.ObjClassCollection(list(new_classes.values()))
new_labels = []
for label in ann.labels:
obj_class = new_classes[label.obj_class.name]
new_labels.append(label.clone(obj_class=obj_class))
res_meta = meta.clone(obj_classes=new_class_collection)
res_ann = ann.clone(labels=new_labels)
return (res_meta, res_ann)
def rename_meta_and_annotations(meta: sly.ProjectMeta, ann: sly.Annotation, suffix="original"):
def _get_new_name(current_name):
return f"{current_name}-{suffix}"
new_classes = []
for obj_class in meta.obj_classes:
obj_class: sly.ObjClass
new_classes.append(obj_class.clone(name=_get_new_name(obj_class.name)))
new_meta = meta.clone(obj_classes=sly.ObjClassCollection(new_classes))
new_labels = []
for label in ann.labels:
dest_name = _get_new_name(label.obj_class.name)
dest_class = new_meta.get_obj_class(dest_name)
new_labels.append(label.clone(obj_class=dest_class))
new_ann = ann.clone(labels=new_labels)
return new_meta, new_ann
def transform_for_detection(meta: sly.ProjectMeta, ann: sly.Annotation) -> (sly.ProjectMeta, sly.Annotation):
new_classes = sly.ObjClassCollection()
new_labels = []
for label in ann.labels:
new_class = label.obj_class.clone(name=label.obj_class.name + "-bbox", geometry_type=sly.Rectangle)
if label.obj_class.geometry_type is sly.Rectangle:
new_labels.append(label.clone(obj_class=new_class))
if new_classes.get(new_class.name) is None:
new_classes = new_classes.add(new_class)
else:
bbox = label.geometry.to_bbox()
if new_classes.get(new_class.name) is None:
new_classes = new_classes.add(new_class)
new_labels.append(label.clone(bbox, new_class))
res_meta = meta.clone(obj_classes=new_classes)
res_ann = ann.clone(labels=new_labels)
return (res_meta, res_ann)
def merge_metas(project_meta: sly.ProjectMeta, model_meta: sly.ProjectMeta,
keep_model_classes, keep_model_tags, suffix):
res_meta = project_meta.clone()
def _merge(keep_names, res_meta, project_collection, model_collection, is_class=False):
mapping = {} # old name to new meta
for name in keep_names:
model_item = model_collection.get(name)
res_item, res_name = find_item(project_collection, model_item, suffix)
if res_item is None:
res_item = model_item.clone(name=res_name)
res_meta = res_meta.add_obj_class(res_item) if is_class else res_meta.add_tag_meta(res_item)
mapping[model_item.name] = res_item
return res_meta, mapping
res_meta, class_mapping = _merge(keep_model_classes, res_meta, res_meta.obj_classes, model_meta.obj_classes, is_class=True)
res_meta, tag_mapping = _merge(keep_model_tags, res_meta, res_meta.tag_metas, model_meta.tag_metas, is_class=False)
return res_meta, class_mapping, tag_mapping
def transform_for_segmentation(meta: sly.ProjectMeta, ann: sly.Annotation) -> (sly.ProjectMeta, sly.Annotation):
new_classes = {}
class_masks = {}
for obj_class in meta.obj_classes:
obj_class: sly.ObjClass
new_class = obj_class.clone(name=obj_class.name + "-mask")
new_classes[obj_class.name] = new_class
class_masks[obj_class.name] = np.zeros(ann.img_size, np.uint8)
new_class_collection = sly.ObjClassCollection(list(new_classes.values()))
for label in ann.labels:
label.draw(class_masks[label.obj_class.name], color=255)
new_labels = []
for class_name, white_mask in class_masks.items():
mask = white_mask == 255
obj_class = new_classes[class_name]
bitmap = sly.Bitmap(data=mask)
new_labels.append(sly.Label(geometry=bitmap, obj_class=obj_class))
res_meta = meta.clone(obj_classes=new_class_collection)
res_ann = ann.clone(labels=new_labels)
return (res_meta, res_ann)
def make_output_meta(self, input_metas_dict):
if len(self.cls_mapping) == 0:
raise RuntimeError('Empty cls_mapping for layer: {}'.format(Layer.action))
full_input_meta = ProjectMeta()
for inp_meta in input_metas_dict.values():
full_input_meta.update(inp_meta)
res_meta = deepcopy(full_input_meta)
in_class_titles = set((x['title'] for x in full_input_meta.classes.py_container))
# __other__ -> smth
if ClassConstants.OTHER in self.cls_mapping:
other_classes = in_class_titles - set(self.cls_mapping.keys())
for oclass in other_classes:
self.cls_mapping[oclass] = self.cls_mapping[ClassConstants.OTHER]
del self.cls_mapping[ClassConstants.OTHER]
missed_classes = in_class_titles - set(self.cls_mapping.keys())
if len(missed_classes) != 0:
raise RuntimeError("Some classes in mapping are missed: {}".format(missed_classes))
for src_class_title, dst_class in self.cls_mapping.items():
# __new__ -> [ list of classes ]
if src_class_title == ClassConstants.NEW:
if type(dst_class) is not list:
raise RuntimeError('Internal class mapping error in layer (NEW spec).')
for new_cls in dst_class:
res_meta.classes.add(new_cls)
# __clone__ -> dict {parent_cls_name: child_cls_name}
elif src_class_title == ClassConstants.CLONE:
if type(dst_class) is not dict:
raise RuntimeError('Internal class mapping error in layer (CLONE spec).')
for src_title, dst_title in dst_class.items():
real_src_cls = full_input_meta.classes[src_title]
if real_src_cls is None:
raise RuntimeError('Class mapping error, source class "{}" not found.'.format(src_title))
real_dst_cls = {**real_src_cls, 'title': dst_title}
res_meta.classes.add(real_dst_cls)
# smth -> __default__
elif dst_class == ClassConstants.DEFAULT:
pass
# smth -> __ignore__
elif dst_class == ClassConstants.IGNORE:
res_meta.classes.delete(src_class_title)
# smth -> new name
elif type(dst_class) is str:
res_meta.classes.rename(src_class_title, dst_class)
# smth -> new cls description
elif type(dst_class) is dict:
res_meta.classes.replace(src_class_title, dst_class)
rm_imtags = self.get_removed_tags()
if rm_imtags is not None:
res_meta.img_tags.difference(rm_imtags)
new_imtags = self.get_added_tags()
if new_imtags is not None:
res_meta.img_tags.update(new_imtags)
return res_meta
def synthesize(api: sly.Api,
task_id,
state,
meta: sly.ProjectMeta,
image_infos,
labels,
bg_images,
cache_dir,
preview=True):
progress_cb = refresh_progress_preview
if preview is False:
progress_cb = refresh_progress
augs = yaml.safe_load(state["augs"])
sly.logger.info("Init augs from yaml file")
aug.init_fg_augs(augs)
visibility_threshold = augs['objects'].get('visibility', 0.8)
classes = state["selectedClasses"]
bg_info = random.choice(bg_images)
sly.logger.info("Download background")
bg = api.image.download_np(bg_info.id)
sly.logger.debug(f"BG shape: {bg.shape}")
res_image = bg.copy()
res_labels = []
# sequence of objects that will be generated
res_classes = []
to_generate = []
for class_name in classes:
original_class: sly.ObjClass = meta.get_obj_class(class_name)
res_classes.append(original_class.clone(geometry_type=sly.Bitmap))
count_range = augs["objects"]["count"]
count = random.randint(*count_range)
for i in range(count):
to_generate.append(class_name)
random.shuffle(to_generate)
res_meta = sly.ProjectMeta(obj_classes=sly.ObjClassCollection(res_classes))
progress = sly.Progress("Processing foregrounds", len(to_generate))
progress_cb(api, task_id, progress)
progress_every = max(10, int(len(to_generate) / 20))
cover_img = np.zeros(res_image.shape[:2], np.int32) # size is (h, w)
objects_area = defaultdict(lambda: defaultdict(float))
cached_images = {}
# generate objects
for idx, class_name in enumerate(to_generate, start=1):
if class_name not in labels:
progress.iter_done_report()
continue
image_id = random.choice(list(labels[class_name].keys()))
label: sly.Label = random.choice(labels[class_name][image_id])
if image_id in cached_images:
source_image = cached_images[image_id]
else:
image_info = image_infos[image_id]
source_image = _get_image_using_cache(api, cache_dir, image_id,
image_info)
cached_images[image_id] = source_image
label_img, label_mask = get_label_foreground(source_image, label)
#sly.image.write(os.path.join(cache_dir, f"{index}_label_img.png"), label_img)
#sly.image.write(os.path.join(cache_dir, f"{index}_label_mask.png"), label_mask)
label_img, label_mask = aug.apply_to_foreground(label_img, label_mask)
#sly.image.write(os.path.join(cache_dir, f"{index}_aug_label_img.png"), label_img)
#sly.image.write(os.path.join(cache_dir, f"{index}_aug_label_mask.png"), label_mask)
label_img, label_mask = aug.resize_foreground_to_fit_into_image(
res_image, label_img, label_mask)
#label_area = g.area
find_place = False
for attempt in range(3):
origin = aug.find_origin(res_image.shape, label_mask.shape)
g = sly.Bitmap(label_mask[:, :, 0].astype(bool),
origin=sly.PointLocation(row=origin[1],
col=origin[0]))
difference = count_visibility(cover_img, g, idx, origin[0],
origin[1])
allow_placement = True
for object_idx, diff in difference.items():
new_area = objects_area[object_idx]['current'] - diff
visibility_portion = new_area / objects_area[object_idx][
'original']
if visibility_portion < visibility_threshold:
#sly.logger.warn(f"Object '{idx}', attempt {attempt + 1}: "
# f"visible portion ({visibility_portion}) < threshold ({visibility_threshold})")
allow_placement = False
break
if allow_placement is True:
find_place = True
break
else:
continue
if find_place is False:
sly.logger.warn(
f"Object '{idx}' is skipped: can not be placed to satisfy visibility threshold"
)
continue
try:
aug.place_fg_to_bg(label_img, label_mask, res_image, origin[0],
origin[1])
g.draw(cover_img, color=idx)
for object_idx, diff in difference.items():
objects_area[object_idx]['current'] -= diff
current_obj_area = g.area
objects_area[idx]['current'] = current_obj_area
objects_area[idx]['original'] = current_obj_area
res_labels.append(sly.Label(g, res_meta.get_obj_class(class_name)))
except Exception as e:
#sly.logger.warn(repr(e))
sly.logger.warn(
f"FG placement error:: label shape: {label_img.shape}; mask shape: {label_mask.shape}",
extra={"error": repr(e)})
progress.iter_done_report()
if idx % progress_every == 0: # progress.need_report():
progress_cb(api, task_id, progress)
progress_cb(api, task_id, progress)
res_ann = sly.Annotation(img_size=bg.shape[:2], labels=res_labels)
# debug visualization
# sly.image.write(os.path.join(cache_dir, "__res_img.png"), res_image)
#res_ann.draw(res_image)
#sly.image.write(os.path.join(cache_dir, "__res_ann.png"), res_image)
res_meta, res_ann = rasterize.convert_to_nonoverlapping(res_meta, res_ann)
return res_image, res_ann, res_meta