def block_detection(input_img_path,
output_root,
num=0,
resize_by_height=800,
show=False):
start = time.clock()
name = input_img_path.split('\\')[-1][:-4]
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
# *** Step 2 *** block processing: detect block -> calculate hierarchy -> detect components in block
blocks = blk.block_division(grey)
blocks = blk.block_add_bkg(blocks, org, grey.shape, show=show)
blk.block_hierarchy(blocks)
file.save_blocks(pjoin(output_root, name + '.json'), blocks)
draw.draw_region(blocks,
grey.shape,
show=show,
write_path=pjoin(output_root, name + '_blk.png'))
cv2.imwrite(pjoin(output_root, name + '.png'), org)
print("[Compo Detection Completed in %.3f s] %d %s" %
(time.clock() - start, num, input_img_path))
def compo_detection(input_img_path, output_root, resize_by_height=600):
start = time.clock()
print("Compo Detection for %s" % input_img_path)
name = input_img_path.split('\\')[-1][:-4]
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary_org = pre.preprocess(org, write_path=pjoin(output_root, name + '_binary.png'))
# *** Step 2 *** block processing: detect block -> detect components in block
blocks_corner = blk.block_division(grey, write_path=pjoin(output_root, name + '_block.png'))
compo_in_blk_boundary, compo_in_blk_corner, compo_in_blk_class = processing_block(org, binary_org, blocks_corner, cnn)
# *** Step 3 *** non-block processing: erase blocks from binary -> detect left components
binary_non_block = blk.block_erase(binary_org, blocks_corner)
compo_non_blk_boundary, compo_non_blk_corner, compo_non_blk_class = processing(org, binary_non_block, cnn, True)
# *** Step 4 *** merge results
# compos_boundary = compo_in_blk_boundary + compo_non_blk_boundary
compos_corner = compo_in_blk_corner + compo_non_blk_corner
compos_class = compo_in_blk_class + compo_non_blk_class
# *** Step 5 *** save results: save text label -> save drawn image
draw.draw_bounding_box_class(org, compos_corner, compos_class, write_path=pjoin(output_root, name + '_ip.png'))
file.save_corners_json(pjoin(output_root, name + '_ip.json'), compos_corner, compos_class)
print("[Compo Detection Completed in %.3f s]" % (time.clock() - start))
def ctpn(input_path_img, output_root, resize_by_height=600):
start = time.clock()
print(('Text Detection for {:s}'.format(input_path_img)))
img, _ = pre.read_img(input_path_img, resize_by_height)
img, scale = resize_im(img,
scale=TextLineCfg.SCALE,
max_scale=TextLineCfg.MAX_SCALE)
blobs, im_scales = _get_blobs(img, None)
if cfg.TEST.HAS_RPN:
im_blob = blobs['data']
blobs['im_info'] = np.array(
[[im_blob.shape[1], im_blob.shape[2], im_scales[0]]],
dtype=np.float32)
cls_prob, box_pred = sess.run([output_cls_prob, output_box_pred],
feed_dict={input_img: blobs['data']})
rois, _ = proposal_layer(cls_prob,
box_pred,
blobs['im_info'],
'TEST',
anchor_scales=cfg.ANCHOR_SCALES)
scores = rois[:, 0]
boxes = rois[:, 1:5] / im_scales[0]
textdetector = TextDetector()
boxes = textdetector.detect(boxes, scores[:, np.newaxis], img.shape[:2])
name = input_path_img.split('\\')[-1][:-4]
output_path_label = os.path.join(output_root, name + '_ocr.txt')
output_path_img = os.path.join(output_root, name + '_ocr.png')
draw_boxes(img, boxes, scale, output_path_label, output_path_img)
print('OCR Complete in %.3f s' % (time.clock() - start))
def main():
save = True
show = False
bad_num = 0
none_tree = 0
num = 0
start = 19558 # start point
end = 100000
img_root = 'E:\\Mulong\\Datasets\\gui\\rico\\combined\\all\\'
block_root = 'E:\\Mulong\\Datasets\\gui\\rico\\subtree\\rico-block\\json\\'
tree_root = 'E:\\Mulong\\Datasets\\gui\\rico\\subtree\\rico-tree-filtered\\widget-layout\\'
subtree_root = 'E:\\Mulong\\Datasets\\gui\\rico\\subtree\\rico-subtree\\widget-layout\\'
for index in range(start, end):
img_path = img_root + str(index) + '.jpg'
block_path = block_root + str(index) + '.json'
tree_path = tree_root + str(index) + '.json'
subtree_path = subtree_root + str(index) + '.json'
if not os.path.exists(block_path) or not os.path.exists(tree_path):
continue
start_time = time.clock()
img, _ = pre.read_img(img_path, resize_height=2560)
blocks = Block.load_blocks(block_path)
resize_block(blocks, det_height=800, tgt_height=2560, bias=0)
board_block = draw.draw_bounding_box(img, blocks, line=5)
try:
tree = Tree.load_tree(tree_path)
except:
bad_num += 1
print('*** F**k Json: %d %s ***' % (bad_num, tree_path))
continue
if tree is not None:
segments = segment_subtree(blocks, tree, img)
else:
none_tree += 1
print('*** Tree is None: %d %s ***' % (none_tree, tree_path))
continue
if show:
board_tree = np.full((2560, 1440, 3), 255,
dtype=np.uint8) # used for draw new labels
Tree.draw_tree(tree, board_tree, 0)
cv2.imshow('block', cv2.resize(board_block, (300, 500)))
cv2.imshow('tree', cv2.resize(board_tree, (300, 500)))
cv2.waitKey()
cv2.destroyAllWindows()
Tree.view_segments(segments, img)
if save:
jfile = open(subtree_path, 'w')
json.dump(segments, jfile, indent=4)
print('[%.3fs]: %d %s' % (time.clock() - start_time, num, img_path))
num += 1
def incorporate(img_path,
label_text,
labels_compo,
label_merge,
resize_by_height=600,
is_clip=False,
clip_path=None):
name = img_path.split('\\')[-1][:-4]
compo_path = pjoin(labels_compo, name + '_ip.json')
text_path = pjoin(label_text, name + '_ocr.txt')
img, _ = pre.read_img(img_path, resize_by_height)
compo_f = open(compo_path, 'r')
text_f = open(text_path, 'r')
compos = json.load(compo_f)
corners_compo = []
compos_class = []
corners_text = []
for compo in compos['compos']:
corners_compo.append([
compo['column_min'], compo['row_min'], compo['column_max'],
compo['row_max']
])
compos_class.append(compo['class'])
for line in text_f.readlines():
if len(line) > 1:
corners_text.append([int(c) for c in line[:-1].split(',')])
corners_text = refine_text(img, corners_text, 20, 10)
corners_compo_new, compos_class_new = nms(img, corners_compo, compos_class,
corners_text)
board = draw_bounding_box_class(img, corners_compo_new, compos_class_new)
output_path_label = pjoin(label_merge, name + '_merged.txt')
output_path_img = pjoin(label_merge, name + '_merged.png')
save_label_txt(img_path, corners_compo_new, compos_class_new,
output_path_label)
cv2.imwrite(output_path_img, board)
print('Merge Complete and Save to', output_path_img)
if is_clip:
save_clipping(img, corners_compo_new, compos_class_new, compo_index,
clip_path)
end = 100000
img_root = 'E:\\Mulong\\Datasets\\rico\\combined\\'
block_root = 'E:\\Temp\\rico-block\\'
tree_root = 'E:\\Temp\\rico-tree\\'
subtree_root = 'E:\\Temp\\rico-subtree\\'
for index in range(start, end):
img_path = img_root + str(index) + '.jpg'
block_path = block_root + str(index) + '.json'
tree_path = tree_root + str(index) + '.json'
subtree_path = subtree_root + str(index) + '.json'
if not os.path.exists(block_path) or not os.path.exists(tree_path):
continue
start_time = time.clock()
img, _ = pre.read_img(img_path, resize_height=2560)
block_img = cv2.imread(block_root + str(index) + '_blk.png')
blocks = Block.load_blocks(block_path)
resize_block(blocks, det_height=800, tgt_height=2560, bias=0)
board_block = draw.draw_bounding_box(img, blocks, line=5)
try:
tree = Tree.load_tree(tree_path)
except:
bad_num += 1
print('*** F**k Json: %d %s ***' %(bad_num, tree_path))
continue
segments = segment_subtree(blocks, tree)
if show:
def compo_detection(input_img_path, output_root, uied_params=None,
resize_by_height=600, block_pad=4,
classifier=None, show=False):
if uied_params is None:
uied_params = {'param-grad':5, 'param-block':5, 'param-minarea':150}
else:
uied_params = json.loads(uied_params)
print(uied_params)
start = time.clock()
name = input_img_path.split('/')[-1][:-4]
ip_root = file.build_directory(pjoin(output_root, "ip"))
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary = pre.binarization(org, grad_min=int(uied_params['param-grad']))
# *** Step 2 *** element detection
det.rm_line(binary, show=show)
# det.rm_line_v_h(binary, show=show)
uicompos = det.component_detection(binary)
# *** Step 4 *** results refinement
# uicompos = det.rm_top_or_bottom_corners(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, name + '_all.json'), uicompos)
# uicompos = det.merge_text(uicompos, org.shape)
draw.draw_bounding_box(org, uicompos, show=show, name='no-merge')
uicompos = det.merge_intersected_corner(uicompos, org)
Compo.compos_update(uicompos, org.shape)
Compo.compos_containment(uicompos)
draw.draw_bounding_box(org, uicompos, show=show, name='no-nesting')
# *** Step 5 ** nesting inspection
uicompos += nesting_inspection(org, grey, uicompos)
uicompos = det.compo_filter(uicompos, min_area=int(uied_params['param-minarea']))
Compo.compos_update(uicompos, org.shape)
draw.draw_bounding_box(org, uicompos, show=show, name='ip-nesting', write_path=pjoin(ip_root, 'result.jpg'))
# *** Step 5 *** Image Inspection: recognize image -> remove noise in image -> binarize with larger threshold and reverse -> rectangular compo detection
# if classifier is not None:
# classifier['Image'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_in_large_img(uicompos, org)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# det.detect_compos_in_img(uicompos, binary_org, org)
# draw.draw_bounding_box(org, uicompos, show=show)
# if classifier is not None:
# classifier['Noise'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_compos(uicompos)
# *** Step 6 *** element classification: all category classification
if classifier is not None:
classifier['Elements'].predict(seg.clipping(org, uicompos), uicompos)
draw.draw_bounding_box_class(org, uicompos, show=show, name='cls', write_path=pjoin(ip_root, 'result.jpg'))
Compo.compos_update(uicompos, org.shape)
draw.draw_bounding_box(org, uicompos, show=show, name='final', write_path=pjoin(output_root, 'result.jpg'))
file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
file.save_corners_json(pjoin(output_root, 'compo.json'), uicompos)
seg.dissemble_clip_img_fill(pjoin(output_root, 'clips'), org, uicompos)
print("[Compo Detection Completed in %.3f s] %s" % (time.clock() - start, input_img_path))
if show:
cv2.destroyAllWindows()
pass
img_file = 'E:\\Mulong\\Datasets\\rico\\combined\\10019.jpg'
resize_height = 800
cv2.namedWindow('control')
cv2.createTrackbar('resize_height', 'control', 800, 1600, nothing)
cv2.createTrackbar('grad_min', 'control', 0, 255, nothing)
cv2.createTrackbar('kernel', 'control', 1, 10, nothing)
while 1:
resize_height = cv2.getTrackbarPos('resize_height', 'control')
grad_min = cv2.getTrackbarPos('grad_min', 'control')
kernel = cv2.getTrackbarPos('kernel', 'control')
org, gray = pre.read_img(img_file, resize_height)
org = cv2.medianBlur(org, 1)
binary = pre.preprocess(org, grad_min)
# det.line_removal(binary, 8)
# morph = cv2.morphologyEx(binary, cv2.MORPH_DILATE, (5, 5))
# morph = cv2.morphologyEx(morph, cv2.MORPH_DILATE, (5, 5))
# morph = cv2.morphologyEx(morph, cv2.MORPH_DILATE, (5, 5))
cv2.imshow('org', org)
cv2.imshow('bin', binary)
cv2.imshow('no_line', binary)
cv2.waitKey(10)
def compo_detection(input_img_path,
output_root,
num=0,
resize_by_height=600,
block_pad=4,
classifier=None,
show=False,
write_img=True):
start = time.clock()
name = input_img_path.split('\\')[-1][:-4]
ip_root = file.build_directory(pjoin(output_root, "ip"))
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary = pre.binarization(
org,
show=show,
write_path=pjoin(ip_root, name + '_binary.png') if write_img else None)
binary_org = binary.copy()
# *** Step 2 *** block processing: detect block -> calculate hierarchy -> detect components in block
blocks = blk.block_division(
grey,
org,
show=show,
write_path=pjoin(ip_root, name + '_block.png') if write_img else None)
blk.block_hierarchy(blocks)
uicompos_in_blk = processing_block(org, binary, blocks, block_pad)
# *** Step 3 *** non-block part processing: remove lines -> erase blocks from binary -> detect left components
det.rm_line(binary, show=show)
blk.block_bin_erase_all_blk(binary, blocks, block_pad)
uicompos_not_in_blk = det.component_detection(binary)
uicompos = uicompos_in_blk + uicompos_not_in_blk
# *** Step 4 *** results refinement: remove top and bottom compos -> merge words into line
uicompos = det.rm_top_or_bottom_corners(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, name + '_all.json'), uicompos)
uicompos = det.merge_text(uicompos, org.shape)
draw.draw_bounding_box(org, uicompos, show=show)
# uicompos = det.merge_intersected_corner(uicompos, org.shape)
Compo.compos_containment(uicompos)
# draw.draw_bounding_box(org, uicompos, show=show, write_path=pjoin(ip_root, name + '_ip.png') if write_img else None)
# # *** Step 5 *** Image Inspection: recognize image -> remove noise in image -> binarize with larger threshold and reverse -> rectangular compo detection
# if classifier is not None:
# classifier['Image'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_in_large_img(uicompos, org)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# det.detect_compos_in_img(uicompos, binary_org, org)
# draw.draw_bounding_box(org, uicompos, show=show)
# if classifier is not None:
# classifier['Noise'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_compos(uicompos)
# *** Step 6 *** element classification: all category classification
if classifier is not None:
classifier['Elements'].predict(seg.clipping(org, uicompos), uicompos)
draw.draw_bounding_box_class(org,
uicompos,
show=show,
write_path=pjoin(ip_root,
name + '_cls.png'))
# uicompos = det.compo_filter(uicompos, org)
draw.draw_bounding_box(org, uicompos, show=show)
file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
print("[Compo Detection Completed in %.3f s] %d %s" %
(time.clock() - start, num, input_img_path))
# Record run time
open('time.txt', 'a').write(str(round(time.clock() - start, 3)) + '\n')
if show:
cv2.destroyAllWindows()
