def merge_intersected_corner(compos, org, max_gap=(0, 0), max_ele_height=25):
changed = False
new_compos = []
Compo.compos_update(compos, org.shape)
for i in range(len(compos)):
merged = False
cur_compo = compos[i]
for j in range(len(new_compos)):
relation = cur_compo.compo_relation(new_compos[j], max_gap)
# print(relation)
# draw.draw_bounding_box(org, [cur_compo, new_compos[j]], name='b-merge', show=True)
if relation == 1 or relation == -1 or \
(relation == 2 and new_compos[j].height < max_ele_height and cur_compo.height < max_ele_height):
new_compos[j].compo_merge(cur_compo)
cur_compo = new_compos[j]
# draw.draw_bounding_box(org, [new_compos[j]], name='a-merge', show=True)
merged = True
changed = True
# break
if not merged:
new_compos.append(compos[i])
if not changed:
return compos
else:
return merge_intersected_corner(new_compos, org, max_gap, max_ele_height)
def nesting_inspection(org, grey, compos, ffl_block):
'''
Inspect all big compos through block division by flood-fill
:param ffl_block: gradient threshold for flood-fill
:return: nesting compos
'''
nesting_compos = []
for i, compo in enumerate(compos):
if compo.height > 50:
replace = False
clip_org = compo.compo_clipping(org)
clip_grey = compo.compo_clipping(grey)
n_compos = blk.block_division(clip_grey,
org,
grad_thresh=ffl_block,
show=False)
Compo.cvt_compos_relative_pos(n_compos, compo.bbox.col_min,
compo.bbox.row_min)
for n_compo in n_compos:
if n_compo.redundant:
compos[i] = n_compo
replace = True
break
if not replace:
nesting_compos += n_compos
return nesting_compos
def merge_intersected_corner(compos,
org,
is_merge_contained_ele,
max_gap=(0, 0),
max_ele_height=25):
'''
:param is_merge_contained_ele: if true, merge compos nested in others
:param max_gap: (horizontal_distance, vertical_distance) to be merge into one line/column
:param max_ele_height: if higher than it, recognize the compo as text
:return:
'''
changed = False
new_compos = []
Compo.compos_update(compos, org.shape)
for i in range(len(compos)):
merged = False
cur_compo = compos[i]
for j in range(len(new_compos)):
relation = cur_compo.compo_relation(new_compos[j], max_gap)
# print(relation)
# draw.draw_bounding_box(org, [cur_compo, new_compos[j]], name='b-merge', show=True)
# merge compo[i] to compo[j] if
# 1. compo[j] contains compo[i]
# 2. compo[j] intersects with compo[i] with certain iou
# 3. is_merge_contained_ele and compo[j] is contained in compo[i]
if relation == 1 or \
relation == 2 or \
(is_merge_contained_ele and relation == -1):
# (relation == 2 and new_compos[j].height < max_ele_height and cur_compo.height < max_ele_height) or\
new_compos[j].compo_merge(cur_compo)
cur_compo = new_compos[j]
# draw.draw_bounding_box(org, [new_compos[j]], name='a-merge', show=True)
merged = True
changed = True
# break
if not merged:
new_compos.append(compos[i])
if not changed:
return compos
else:
return merge_intersected_corner(new_compos, org,
is_merge_contained_ele, max_gap,
max_ele_height)
def nesting_inspection(org, grey, compos):
nesting_compos = []
for i, compo in enumerate(compos):
if compo.height > 50:
replace = False
clip_org = compo.compo_clipping(org)
clip_grey = compo.compo_clipping(grey)
n_compos = blk.block_division(clip_grey, org, show=False)
Compo.cvt_compos_relative_pos(n_compos, compo.bbox.col_min,
compo.bbox.row_min)
for n_compo in n_compos:
if n_compo.redundant:
compos[i] = n_compo
replace = True
break
if not replace:
nesting_compos += n_compos
# cv2.imshow('clip', clip_org)
# cv2.waitKey()
return nesting_compos
def processing_block(org, binary, blocks, block_pad):
image_shape = org.shape
uicompos_all = []
for block in blocks:
# *** Step 2.1 *** check: examine if the block is valid layout block
if block.block_is_top_or_bottom_bar(image_shape,
C.THRESHOLD_TOP_BOTTOM_BAR):
continue
if block.block_is_uicompo(image_shape, C.THRESHOLD_COMPO_MAX_SCALE):
uicompos_all.append(block)
# *** Step 2.2 *** binary map processing: erase children block -> clipping -> remove lines(opt)
binary_copy = binary.copy()
for i in block.children:
blocks[i].block_erase_from_bin(binary_copy, block_pad)
block_clip_bin = block.compo_clipping(binary_copy)
# det.line_removal(block_clip_bin, show=True)
# *** Step 2.3 *** component extraction: detect components in block binmap -> convert position to relative
uicompos = det.component_detection(block_clip_bin)
Compo.cvt_compos_relative_pos(uicompos, block.bbox.col_min,
block.bbox.row_min)
uicompos_all += uicompos
return uicompos_all
def compo_detection(input_img_path,
output_root,
uied_params,
resize_by_height=600,
batch=False,
classifier=None,
show=False):
start = time.time()
name = input_img_path.split('/')[-1][:-4]
ip_root = file.build_directory(pjoin(output_root, "ip"))
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary = pre.binarization(org,
grad_min=int(uied_params['min-grad']),
show=show)
# *** Step 2 *** element detection
det.rm_line(binary, show=show)
# det.rm_line_v_h(binary, show=show)
uicompos = det.component_detection(binary,
min_obj_area=int(
uied_params['min-ele-area']))
file.save_corners_json(pjoin(ip_root, name + '_all.json'), uicompos)
draw.draw_bounding_box(org, uicompos, show=show, name='components')
# *** Step 3 *** results refinement
# uicompos = det.rm_top_or_bottom_corners(uicompos, org.shape)
# uicompos = det.merge_text(uicompos, org.shape)
uicompos = det.merge_intersected_corner(
uicompos,
org,
is_merge_contained_ele=uied_params['merge-contained-ele'],
max_gap=(0, 0),
max_ele_height=25)
Compo.compos_update(uicompos, org.shape)
Compo.compos_containment(uicompos)
draw.draw_bounding_box(org, uicompos, show=show, name='merged')
# *** Step 4 ** nesting inspection: treat the big compos as block and check if they have nesting element
uicompos += nesting_inspection(org,
grey,
uicompos,
ffl_block=uied_params['ffl-block'])
uicompos = det.compo_filter(uicompos,
min_area=int(uied_params['min-ele-area']))
Compo.compos_update(uicompos, org.shape)
draw.draw_bounding_box(org,
uicompos,
show=show,
name='nesting compo',
write_path=pjoin(ip_root, 'result.jpg'))
draw.draw_bounding_box(org,
uicompos,
write_path=pjoin(output_root, 'result.jpg'))
# *** Step 5 *** Image Inspection: recognize image -> remove noise in image
# -> binarize with larger threshold and reverse -> rectangular compo detection
# if classifier is not None:
# classifier['Image'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_in_large_img(uicompos, org)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# det.detect_compos_in_img(uicompos, binary, org)
# draw.draw_bounding_box(org, uicompos, show=show)
# if classifier is not None:
# classifier['Noise'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_compos(uicompos)
# *** Step 6 *** element classification: all category classification
if classifier is not None:
classifier['Elements'].predict(seg.clipping(org, uicompos), uicompos)
draw.draw_bounding_box_class(org,
uicompos,
show=show,
name='cls',
write_path=pjoin(ip_root, 'result.jpg'))
draw.draw_bounding_box_class(org,
uicompos,
write_path=pjoin(output_root,
'result.jpg'))
Compo.compos_update(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
file.save_corners_json(pjoin(output_root, 'compo.json'), uicompos)
# seg.dissemble_clip_img_fill(pjoin(output_root, 'clips'), org, uicompos)
if not batch:
print("[Compo Detection Completed in %.3f s] %s" %
(time.time() - start, input_img_path))
if show:
cv2.destroyAllWindows()
def compo_detection(input_img_path,
output_root,
uied_params,
resize_by_height=600,
classifier=None,
show=False,
wai_key=10):
start = time.clock()
name = input_img_path.split('/')[-1][:-4]
ip_root = file.build_directory(pjoin(output_root, "ip"))
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary = pre.binarization(org,
grad_min=int(uied_params['min-grad']),
show=show,
wait_key=wai_key)
# *** Step 2 *** element detection
det.rm_line(binary, show=show, wait_key=wai_key)
# det.rm_line_v_h(binary, show=show)
uicompos = det.component_detection(binary,
min_obj_area=int(
uied_params['min-ele-area']))
# draw.draw_bounding_box(org, uicompos, show=show, name='components', wait_key=wai_key)
# *** Step 3 *** results refinement
uicompos = det.merge_intersected_corner(
uicompos,
org,
is_merge_contained_ele=uied_params['merge-contained-ele'],
max_gap=(0, 0),
max_ele_height=25)
Compo.compos_update(uicompos, org.shape)
Compo.compos_containment(uicompos)
# draw.draw_bounding_box(org, uicompos, show=show, name='merged', wait_key=wai_key)
# *** Step 4 ** nesting inspection: treat the big compos as block and check if they have nesting element
uicompos += nesting_inspection(org,
grey,
uicompos,
ffl_block=uied_params['ffl-block'])
uicompos = det.compo_filter(uicompos,
min_area=int(uied_params['min-ele-area']))
Compo.compos_update(uicompos, org.shape)
draw.draw_bounding_box(org,
uicompos,
show=show,
name='merged compo',
write_path=pjoin(ip_root, 'result.jpg'),
wait_key=wai_key)
# *** Step 6 *** element classification: all category classification
if classifier is not None:
classifier['Elements'].predict(seg.clipping(org, uicompos), uicompos)
draw.draw_bounding_box_class(org,
uicompos,
show=show,
name='cls',
write_path=pjoin(ip_root, 'result.jpg'))
draw.draw_bounding_box_class(org,
uicompos,
write_path=pjoin(output_root,
'result.jpg'))
seg.dissemble_clip_img_fill(pjoin(output_root, 'new'), org, uicompos)
if classifier is None:
Compo.compos_update(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
file.save_corners_json(pjoin(output_root, 'compo.json'), uicompos)
else:
Compo.compos_update(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, "clf_" + name + '.json'),
uicompos)
file.save_corners_json(pjoin(output_root, 'clf_compo.json'), uicompos)
if show:
cv2.destroyAllWindows()
print("[Compo Detection Completed in %.3f s] %s" %
(time.clock() - start, input_img_path))
def compo_detection(input_img_path,
output_root,
uied_params,
resize_by_height=800,
classifier=None,
show=False,
wai_key=0):
start = time.clock()
name = input_img_path.split(
'/')[-1][:-4] if '/' in input_img_path else input_img_path.split(
'\\')[-1][:-4]
ip_root = file.build_directory(pjoin(output_root, "ip"))
# *** Step 1 *** pre-processing: read img -> get binary map
org, grey = pre.read_img(input_img_path, resize_by_height)
binary = pre.binarization(org, grad_min=int(uied_params['min-grad']))
# *** Step 2 *** element detection
det.rm_line(binary, show=show, wait_key=wai_key)
uicompos = det.component_detection(binary,
min_obj_area=int(
uied_params['min-ele-area']))
# *** Step 3 *** results refinement
uicompos = det.compo_filter(uicompos,
min_area=int(uied_params['min-ele-area']),
img_shape=binary.shape)
uicompos = det.merge_intersected_compos(uicompos)
det.compo_block_recognition(binary, uicompos)
if uied_params['merge-contained-ele']:
uicompos = det.rm_contained_compos_not_in_block(uicompos)
Compo.compos_update(uicompos, org.shape)
Compo.compos_containment(uicompos)
# *** Step 4 ** nesting inspection: check if big compos have nesting element
uicompos += nesting_inspection(org,
grey,
uicompos,
ffl_block=uied_params['ffl-block'])
Compo.compos_update(uicompos, org.shape)
draw.draw_bounding_box(org,
uicompos,
show=show,
name='merged compo',
write_path=pjoin(ip_root, name + '.jpg'),
wait_key=wai_key)
# *** Step 5 *** image inspection: recognize image -> remove noise in image -> binarize with larger threshold and reverse -> rectangular compo detection
# if classifier is not None:
# classifier['Image'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_in_large_img(uicompos, org)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# det.detect_compos_in_img(uicompos, binary_org, org)
# draw.draw_bounding_box(org, uicompos, show=show)
# if classifier is not None:
# classifier['Noise'].predict(seg.clipping(org, uicompos), uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show)
# uicompos = det.rm_noise_compos(uicompos)
# *** Step 6 *** element classification: all category classification
# if classifier is not None:
# classifier['Elements'].predict([compo.compo_clipping(org) for compo in uicompos], uicompos)
# draw.draw_bounding_box_class(org, uicompos, show=show, name='cls', write_path=pjoin(ip_root, 'result.jpg'))
# draw.draw_bounding_box_class(org, uicompos, write_path=pjoin(output_root, 'result.jpg'))
# *** Step 7 *** save detection result
Compo.compos_update(uicompos, org.shape)
file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
print(
"[Compo Detection Completed in %.3f s] Input: %s Output: %s" %
(time.clock() - start, input_img_path, pjoin(ip_root, name + '.json')))