def init(data, state):
global reference_gallery
local_path = os.path.join(ag.app.data_dir, ag.reference_path.lstrip("/"))
ag.api.file.download(ag.team_id, ag.reference_path, local_path)
ref_json = sly.json.load_json_file(local_path)
for reference_key, ref_examples in ref_json["references"].items():
if len(ref_examples) >= 2:
cnt_grid_columns = 2
else:
cnt_grid_columns = 1
review_gallery = {
"content": {
"projectMeta": ag.gallery_meta.to_json(),
"annotations": {},
"layout": [[] for i in range(cnt_grid_columns)]
},
"previewOptions": ag.image_preview_options,
"options": ag.image_grid_options,
}
for idx, reference_info in enumerate(ref_examples):
image_url = reference_info["image_url"]
[top, left, bottom, right] = reference_info["bbox"]
figure_id = reference_info["geometry"]["id"]
label = sly.Label(
sly.Rectangle(top, left, bottom, right, sly_id=figure_id),
ag.gallery_meta.get_obj_class("product"))
catalog_info = catalog.index[reference_key]
review_gallery["content"]["annotations"][figure_id] = {
"url": image_url,
"labelId": figure_id, # duplicate for simplicity
"figures": [label.to_json()],
"zoomToFigure": {
"figureId": figure_id,
"factor": 1.2
},
"catalogInfo": catalog_info
}
review_gallery["content"]["layout"][idx % cnt_grid_columns].append(
figure_id)
reference_gallery[reference_key] = review_gallery
sly.logger.info(f"Number of items in catalog: {len(catalog.index)}")
sly.logger.info(f"Number of references: {len(reference_gallery)}")
data["userRef"] = {} # {1: "7861026000305"} #@TODO: for debug
data["refGrid"] = reference_gallery
state["selected"] = {}
def get_next_object(api, task_id, project_id):
project_dir = os.path.join(sly.app.SHARED_DATA, "app_tagging",
str(project_id))
image_labels = sly_json.load_json_file(
os.path.join(project_dir, "image_labels_pairs.json"))
free_pairs = sly_json.load_json_file(
os.path.join(project_dir, "free_pairs.json"))
if len(free_pairs) == 0:
# @TODO: show message to user
sly.logger.info("labeling finished")
item = image_labels[free_pairs[0]]
image_id = item[0]
ann_path = item[1]
label_index = item[2]
image_path = os.path.join(project_dir, "images", "{}.png".format(image_id))
if not sly.fs.file_exists(image_path):
api.image.download_path(image_id, image_path)
image = sly.image.read(image_path)
meta_json = sly_json.load_json_file(os.path.join(project_dir, "meta.json"))
meta = sly.ProjectMeta.from_json(meta_json)
ann_json = sly_json.load_json_file(ann_path)
ann = sly.Annotation.from_json(ann_json, meta)
label = ann.labels[label_index]
rect = label.geometry.to_bbox()
cropped_image = sly.image.crop(image, label.geometry.to_bbox())
canvas = image.copy()
label.draw_contour(canvas, thickness=3)
pad = 150
rect_context = sly.Rectangle(max(0, rect.top - pad),
max(0, rect.left - pad),
min(image.shape[0] - 1, rect.bottom + pad),
min(image.shape[1] - 1, rect.right + pad))
cropped_context = sly.image.crop(canvas, rect_context)
cropped_url, cropped_context_url = pack_images(cropped_image,
cropped_context)
api.task.set_data(task_id, [[cropped_context_url], [cropped_url]],
"data.objectToTag")
def yolo_preds_to_sly_rects(detections, idx_to_class, confidence_tag_meta):
labels = []
for classId, confidence, box in detections:
xmin = box[0] - box[2] / 2
ymin = box[1] - box[3] / 2
xmax = box[0] + box[2] / 2
ymax = box[1] + box[3] / 2
rect = sly.Rectangle(round(ymin), round(xmin), round(ymax), round(xmax))
label = sly.Label(rect, idx_to_class[classId])
confidence_tag = sly.Tag(confidence_tag_meta, value=round(float(confidence), 4))
label = label.add_tag(confidence_tag)
labels.append(label)
return labels
def test_from_supervisely():
import supervisely_lib as sly
data_path = "./data/test_supervisely/from_to"
if os.path.exists(data_path):
shutil.rmtree(data_path)
project_name = "pixel_project"
project_path = os.path.join(data_path, project_name)
project = sly.Project(project_path, sly.OpenMode.CREATE)
init_meta = project.meta
project.meta._project_type = "images"
project_ds = project.create_dataset(project_name)
img = np.ones((30, 30, 3))
project_ds.add_item_np("pixel.jpeg", img)
item_path, item_ann_path = project_ds.get_item_paths("pixel.jpeg")
ann = sly.Annotation.load_json_file(item_ann_path, project.meta)
bbox_class = sly.ObjClass(name="_bbox", geometry_type=sly.Rectangle)
meta_with_bboxes = project.meta.add_obj_classes([bbox_class])
bbox_label = sly.Label(
geometry=sly.Rectangle(0, 0, 10, 10),
obj_class=meta_with_bboxes.obj_classes.get("_bbox"),
)
ann_with_bboxes = ann.add_labels([bbox_label])
project_ds.set_ann("pixel.jpeg", ann_with_bboxes)
project.set_meta(meta_with_bboxes)
trans = hub.Dataset.from_supervisely(project)
dataset = trans.store(os.path.join(data_path, "pixel_dataset_bbox"))
project_back = dataset.to_supervisely(
os.path.join(data_path, "pixel_project_bbox_back"))
project.set_meta(init_meta)
poly_class = sly.ObjClass(name="_poly", geometry_type=sly.Polygon)
meta_with_poly = project.meta.add_obj_classes([poly_class])
points = [[0, 0], [0, 10], [10, 0], [10, 10]]
point_loc_points = [
sly.geometry.point_location.PointLocation(*point) for point in points
]
poly_label = sly.Label(
geometry=sly.Polygon(exterior=point_loc_points, interior=[]),
obj_class=meta_with_poly.obj_classes.get("_poly"),
)
ann_with_polys = ann.add_labels([poly_label])
project_ds.set_ann("pixel.jpeg", ann_with_polys)
project.set_meta(meta_with_poly)
trans = hub.Dataset.from_supervisely(project)
dataset = trans.store(os.path.join(data_path, "pixel_dataset_poly"))
project_back = dataset.to_supervisely(
os.path.join(data_path, "pixel_project_poly_back"))
def inference_image_path(image_path, context, state, app_logger):
app_logger.debug("Input path", extra={"path": image_path})
rect = None
if "rectangle" in state:
top, left, bottom, right = state["rectangle"]
rect = sly.Rectangle(top, left, bottom, right)
settings = state.get("settings", {})
for key, value in default_settings.items():
if key not in settings:
app_logger.warn(
"Field {!r} not found in inference settings. Use default value {!r}"
.format(key, value))
debug_visualization = settings.get("debug_visualization",
default_settings["debug_visualization"])
conf_thres = settings.get("conf_thres", default_settings["conf_thres"])
iou_thres = settings.get("iou_thres", default_settings["iou_thres"])
augment = settings.get("augment", default_settings["augment"])
image = sly.image.read(image_path) # RGB image
if rect is not None:
canvas_rect = sly.Rectangle.from_size(image.shape[:2])
results = rect.crop(canvas_rect)
if len(results) != 1:
return {
"message": "roi rectangle out of image bounds",
"roi": state["rectangle"],
"img_size": {
"height": image.shape[0],
"width": image.shape[1]
}
}
rect = results[0]
image = sly.image.crop(image, rect)
ann_json = inference(model,
half,
device,
imgsz,
stride,
image,
meta,
conf_thres=conf_thres,
iou_thres=iou_thres,
augment=augment,
debug_visualization=debug_visualization)
return ann_json
def inference(model, half, device, imgsz, stride, image: np.ndarray, meta: sly.ProjectMeta, conf_thres=0.25, iou_thres=0.45,
augment=False, agnostic_nms=False, debug_visualization=False) -> sly.Annotation:
names = model.module.names if hasattr(model, 'module') else model.names
img0 = image # RGB
# Padded resize
img = letterbox(img0, new_shape=imgsz, stride=stride)[0]
img = img.transpose(2, 0, 1) # to 3x416x416
img = np.ascontiguousarray(img)
img = torch.from_numpy(img).to(device)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
inf_out = model(img, augment=augment)[0]
# Apply NMS
labels = []
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres, agnostic=agnostic_nms)
for i, det in enumerate(output):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape).round()
for *xyxy, conf, cls in reversed(det):
top, left, bottom, right = int(xyxy[1]), int(xyxy[0]), int(xyxy[3]), int(xyxy[2])
rect = sly.Rectangle(top, left, bottom, right)
obj_class = meta.get_obj_class(names[int(cls)])
tag = sly.Tag(meta.get_tag_meta(CONFIDENCE), round(float(conf), 4))
label = sly.Label(rect, obj_class, sly.TagCollection([tag]))
labels.append(label)
height, width = img0.shape[:2]
ann = sly.Annotation(img_size=(height, width), labels=labels)
if debug_visualization is True:
# visualize for debug purposes
vis = np.copy(img0)
ann.draw_contour(vis, thickness=2)
sly.image.write("vis.jpg", vis)
return ann.to_json()
def convert_geometry(x_center, y_center, ann_width, ann_height, img_width,
img_height):
x_center = float(x_center)
y_center = float(y_center)
ann_width = float(ann_width)
ann_height = float(ann_height)
px_x_center = x_center * img_width
px_y_center = y_center * img_height
px_ann_width = ann_width * img_width
px_ann_height = ann_height * img_height
left = px_x_center - (px_ann_width / 2)
right = px_x_center + (px_ann_width / 2)
top = px_y_center - (px_ann_height / 2)
bottom = px_y_center + (px_ann_height / 2)
return sly.Rectangle(top, left, bottom, right)
def visualize_dets(img_, output_, save_path_, names_, meta_):
labels = []
for i, det in enumerate(output_):
if det is not None and len(det):
for *xyxy, conf, cls in reversed(det):
left, top, right, bottom = int(xyxy[0]), int(xyxy[1]), int(
xyxy[2]), int(xyxy[3])
rect = sly.Rectangle(top, left, bottom, right)
obj_class = meta_.get_obj_class(names_[int(cls)])
tag = sly.Tag(meta_.get_tag_meta("confidence"),
round(float(conf), 4))
label = sly.Label(rect, obj_class, sly.TagCollection([tag]))
labels.append(label)
width, height = img_.size
ann = sly.Annotation(img_size=(height, width), labels=labels)
vis = np.copy(img_)
ann.draw_contour(vis, thickness=2)
sly.image.write(save_path_, vis)
return vis
def main_split_textline(ini, common_info, logger=None):
# Init. path variables
global box_color, rst_dir_name
vars = {}
for key, val in ini.items():
vars[key] = cs.replace_string_from_dict(val, common_info)
except_dir_names = vars['except_dir_names'].replace(' ', '').split(',')
img_mode = vars['img_mode']
link_, copy_, border_, save_detect_box_img_, save_refine_box_img_ = \
cs.string_to_boolean(vars['link_']), cs.string_to_boolean(vars['copy_']), cs.string_to_boolean(vars['border_']), \
cs.string_to_boolean(vars['save_detect_box_img_']), cs.string_to_boolean(vars['save_refine_box_img_'])
# Init. CRAFT
gpu_ = cs.string_to_boolean(vars['cuda'])
device = torch.device('cuda' if (
torch.cuda.is_available() and gpu_) else 'cpu')
ko_model_dir, math_model_dir = cg.get_model_dir(root_dir=vars['ko_model_path'], model_file=vars['ko_model_name']), \
cg.get_model_dir(root_dir=vars['math_model_path'], model_file=vars['math_model_name'])
ko_detector = get_detector(os.path.join(ko_model_dir,
vars['ko_model_name']),
device,
quantize=False)
math_detector = get_detector(os.path.join(math_model_dir,
vars['math_model_name']),
device,
quantize=False)
easyocr_ini = cg.get_ini_parameters(
os.path.join(_this_folder_, vars['ocr_ini_fname']))
craft_params = load_craft_parameters(easyocr_ini['CRAFT'])
project = sly.Project(directory=vars['textline_dataset_path'],
mode=sly.OpenMode.READ,
block_directories=except_dir_names)
# Preprocess datasets
if link_:
link_or_copy_datasets(src_dir_path=vars['textline_dataset_path'],
dst_dir_path=vars['refine_dataset_path'],
dir_names=project.datasets.keys(),
except_dir_names=except_dir_names,
tgt_dir_name='img/',
mode=LINK,
logger=logger)
if copy_:
link_or_copy_datasets(src_dir_path=vars['textline_dataset_path'],
dst_dir_path=vars['refine_dataset_path'],
dir_names=project.datasets.keys(),
except_dir_names=except_dir_names,
tgt_dir_name='ann/',
mode=COPY,
logger=logger)
# Load and split textlines
for dataset in project:
sly.logger.info('Processing dataset: {}/{}'.format(
project.name, dataset.name))
for item_idx, item_name in enumerate(dataset):
item_paths = dataset.get_item_paths(item_name)
ann, json_data = sly.Annotation.load_json_file(
item_paths.ann_path, project.meta)
raw_img = sly.image.read(item_paths.img_path)
draw_detect_img, draw_refine_img = raw_img.copy(), raw_img.copy()
# Draw textline box
for label in ann.labels:
if label.obj_class.name == textline:
label.geometry.draw_contour(
draw_detect_img,
color=label.obj_class.color,
config=label.obj_class.geometry_config,
thickness=3)
label.geometry.draw_contour(
draw_refine_img,
color=label.obj_class.color,
config=label.obj_class.geometry_config,
thickness=3)
draw_detect_img = ((draw_detect_img.astype(np.uint16) +
raw_img.astype(np.uint16)) / 2).astype(
np.uint8)
draw_refine_img = ((draw_refine_img.astype(np.uint16) +
raw_img.astype(np.uint16)) / 2).astype(
np.uint8)
# Get textline ground truths
gt_objs = []
for idx, label in reversed(list(enumerate(ann.labels))):
if label.obj_class.name != textline:
crop_img = raw_img[
label.geometry.top:label.geometry.bottom,
label.geometry.left:label.geometry.right]
if crop_img.size == 0:
continue
crop_box, ret_ = ip.get_binary_area_coordinates_by_threshold(
crop_img, min_thresh=127, max_thresh=255)
if ret_:
crop_box_obj = ic.Box(crop_box)
proc_box = ic.calc_global_box_pos_in_box(
g_box=[
label.geometry.left, label.geometry.right,
label.geometry.top, label.geometry.bottom
],
box=crop_box_obj.rect2,
format='rect2')
min_x, max_x, min_y, max_y = proc_box
# debug_img = raw_img[min_y:max_y, min_x:max_x]
# print('Prev geo. : ', ann.labels[idx].geometry.left, ann.labels[idx].geometry.right, ann.labels[idx].geometry.top, ann.labels[idx].geometry.bottom)
# Remove label
ann = ann.delete_label(label)
# update coordinates
crop_labels = label.crop(
sly.Rectangle(min_y, min_x, max_y,
max_x)) # top, left, bottom, right,
for crop_label in crop_labels:
ann = ann.add_label(crop_label)
# print('Next geo. : ', ann.labels[idx].geometry.left, ann.labels[idx].geometry.right, ann.labels[idx].geometry.top, ann.labels[idx].geometry.bottom)
continue
# Remove textline object
ann = ann.delete_label(label)
if (label.geometry.right - label.geometry.left) <= 0 or (
label.geometry.bottom - label.geometry.top) <= 0:
continue
gt_box = ic.Box(
box=[[label.geometry.left, label.geometry.top],
[label.geometry.right, label.geometry.bottom]])
gt_obj = object.Object(name=textline,
box_obj=gt_box,
description=label.description.strip())
gt_objs.append(gt_obj)
# Get predict results
pred_objs = []
for detector in [ko_detector, math_detector]:
tgt_class = ko if (detector is ko_detector) else (math if (
detector is math_detector) else 'None')
# # Make border
border_margin = 0
if border_:
border_color = ig.WHITE
border_margin = 30
raw_img = cv2.copyMakeBorder(raw_img,
border_margin,
border_margin,
border_margin,
border_margin,
cv2.BORDER_CONSTANT,
value=border_color)
boxes = get_textbox(
detector,
raw_img,
canvas_size=craft_params['canvas_size'],
mag_ratio=craft_params['mag_ratio'],
text_threshold=craft_params['text_threshold'],
link_threshold=craft_params['link_threshold'],
low_text=craft_params['low_text'],
poly=False,
device=device,
optimal_num_chars=True)
if border_:
boxes = [
np.array([
box[0] - border_margin, box[1] - border_margin,
box[2] - border_margin, box[3] - border_margin,
box[4] - border_margin, box[5] - border_margin,
box[6] - border_margin, box[7] - border_margin
]) for box in boxes
]
horizontal_list, _ = group_text_box(
boxes, craft_params['slope_ths'],
craft_params['ycenter_ths'], craft_params['height_ths'],
craft_params['width_ths'], craft_params['add_margin'])
for h_box in horizontal_list:
pred_box = ic.Box(
box=[[h_box[0], h_box[2]], [h_box[1], h_box[3]]])
pred_obj = object.Object(name=tgt_class,
box_obj=pred_box,
description='')
pred_objs.append(pred_obj)
if tgt_class == ko:
box_color = ig.BROWN
if tgt_class == math:
box_color = ig.MAGENTA
draw_detect_img = ip.draw_box_on_img(draw_detect_img,
pred_box.flat_box,
color=box_color,
thickness=2)
# Save result image
ko_model_epoch, math_model_epoch = vars['ko_model_name'].split('_')[-1].replace('.pth', ''), \
vars['math_model_name'].split('_')[-1].replace('.pth', '')
rst_dir_name = f'{ko}_' + ko_model_epoch + '_' + f'{math}_' + math_model_epoch
rst_dir_path = os.path.join(vars['rst_path'], rst_dir_name,
'draw_box')
if save_detect_box_img_:
cg.folder_exists(rst_dir_path, create_=True)
ig.imwrite(
draw_detect_img,
os.path.join(rst_dir_path, f'[{img_mode}] ' + item_name))
# Compare GT. & PRED.
refine_gts = refine_ground_truths_by_predict_values(
gt_objs, pred_objs, raw_img) # test input : GTS, PREDS
# Draw refined boxes & texts
if save_refine_box_img_:
for (rf_box, rf_text, rf_class) in refine_gts:
rf_rect2 = ic.convert_rect4_to_rect2(rf_box)
x_min, x_max, y_min, y_max = rf_rect2
box = (x_min, y_min, x_max, y_max)
if rf_class == ko:
box_color = ig.BROWN
if rf_class == math:
box_color = ig.MAGENTA
# Draw boxes
draw_refine_img = ip.draw_box_on_img(draw_refine_img,
box,
color=box_color,
thickness=3)
# Draw texts (for 한글)
pil_img = Image.fromarray(draw_refine_img)
draw = ImageDraw.Draw(pil_img)
font = cg.KOR_FONT
margin_x, margin_y = 10, 45
draw.text(xy=((x_min + 1 + margin_x),
(y_min + 1 + margin_y)),
text=rf_text,
font=font,
fill=box_color)
draw_refine_img = np.array(pil_img)
rst_dir_path = os.path.join(vars['rst_path'], rst_dir_name,
'refine_box')
cg.folder_exists(rst_dir_path, create_=True)
ig.imwrite(
draw_refine_img,
os.path.join(rst_dir_path, f'[{img_mode}] ' + item_name))
# Insert refine_gts to json
if len(ann.labels) > 0:
obj_id = ann.labels[-1].geometry.sly_id + 1
else:
obj_id = 0
refine_json_data, refine_obj_id = update_json_from_results(
ann.to_json(), obj_id, [ko, math], refine_gts)
# Save refined json
rst_ann_fname = item_paths.ann_path.replace(
vars['textline_dataset_path'], vars['refine_dataset_path'])
with open(rst_ann_fname, 'w', encoding='utf-8') as f:
json.dump(refine_json_data, f, ensure_ascii=False, indent=4)
sly.logger.info('[{}/{}] Refined json path : {}'.format(
item_idx + 1, len(dataset), rst_ann_fname))
logger.info(" # {} in {} mode finished.".format(_this_basename_, OP_MODE))
return True