def compo_in_img(org, bin, corners):
def reverse(img):
rec, b = cv2.threshold(img, 1, 255, cv2.THRESH_BINARY_INV)
return b
corners_compo = []
pad = 2
for corner in corners:
(top_left, bottom_right) = corner
(col_min, row_min) = top_left
(col_max, row_max) = bottom_right
col_min = max(col_min - pad, 0)
col_max = min(col_max + pad, org.shape[1])
row_min = max(row_min - pad, 0)
row_max = min(row_max + pad, org.shape[0])
clip_bin = bin[row_min:row_max, col_min:col_max]
clip_bin = reverse(clip_bin)
boundary_all, boundary_rec, boundary_nonrec = det.boundary_detection(
clip_bin,
C.THRESHOLD_OBJ_MIN_AREA,
C.THRESHOLD_OBJ_MIN_PERIMETER, # size of area
C.THRESHOLD_LINE_THICKNESS, # line check
C.THRESHOLD_REC_MIN_EVENNESS_STRONG,
C.THRESHOLD_IMG_MAX_DENT_RATIO) # rectangle check
corners_rec = det.get_corner(boundary_rec)
corners_rec = utils.corner_cvt_relative_position(
corners_rec, col_min, row_min)
corners_compo += corners_rec
draw.draw_bounding_box(org, corners_compo, show=True)
return corners_compo
def processing(org, binary, clf, main=True):
if main:
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_rec, boundary_non_rec = det.boundary_detection(binary, write_boundary=True)
corners_rec = det.get_corner(boundary_rec)
corners_non_rec = det.get_corner(boundary_non_rec)
# *** Step 3 *** data processing: identify blocks and compos from rectangles -> identify irregular compos
corners_block, corners_img, corners_compo = det.block_or_compo(org, binary, corners_rec)
det.compo_irregular(org, corners_non_rec, corners_img, corners_compo)
# *** Step 4 *** classification: clip and classify the components candidates -> ignore noises -> refine img
compos = seg.clipping(org, corners_compo)
compos_class = clf.predict(compos)
corners_compo, compos_class = det.strip_img(corners_compo, compos_class, corners_img)
# *** Step 5 *** result refinement
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# *** Step 6 *** recursive inspection: search components nested in components
corners_block, corners_img, corners_compo, compos_class = det.compo_in_img(processing, org, binary, clf, corners_img, corners_block, corners_compo, compos_class)
# *** Step 7 *** ocr check and text detection from cleaned image
if is_ocr:
corners_block, _ = det.rm_text(org, corners_block, ['block' for i in range(len(corners_block))])
corners_img, _ = det.rm_text(org, corners_img, ['img' for i in range(len(corners_img))])
corners_compo, compos_class = det.rm_text(org, corners_compo, compos_class)
# *** Step 8 *** merge overlapped components
# corners_img = det.rm_img_in_compo(corners_img, corners_compo)
corners_img, _ = det.merge_corner(org, corners_img, ['img' for i in range(len(corners_img))], is_merge_nested_same=True)
corners_compo, compos_class = det.merge_corner(org, corners_compo, compos_class, is_merge_nested_same=True)
return corners_block, corners_img, corners_compo, compos_class
# *** used for img inspection ***
# only consider rectangular components
else:
boundary_rec, boundary_non_rec = det.boundary_detection(binary)
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img, corners_compo = det.block_or_compo(org, binary, corners_rec)
compos = seg.clipping(org, corners_compo)
compos_class = clf.predict(compos)
corners_compo, compos_class = det.strip_img(corners_compo, compos_class, corners_img)
return corners_block, corners_compo, compos_class
def processing(org, binary, main=True):
if main:
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_rec, boundary_non_rec = det.boundary_detection(binary,
show=False)
corners_rec = det.get_corner(boundary_rec)
corners_non_rec = det.get_corner(boundary_non_rec)
# *** Step 3 *** data processing: identify blocks and compos from rectangles -> identify irregular compos
corners_block, corners_img, corners_compo = det.block_or_compo(
org, binary, corners_rec)
det.compo_irregular(org, corners_non_rec, corners_img, corners_compo)
corners_img, _ = det.rm_text(org, corners_img,
['img' for i in range(len(corners_img))])
# *** Step 4 *** classification: clip and classify the components candidates -> ignore noises -> refine img
compos = seg.clipping(org, corners_compo)
compos_class = CNN.predict(compos)
# corners_compo, compos_class = det.compo_filter(org, corners_compo, compos_class, is_icon)
corners_compo, compos_class = det.strip_img(corners_compo,
compos_class, corners_img)
# *** Step 5 *** result refinement
if is_merge:
corners_img, _ = det.merge_corner(
corners_img, ['img' for i in range(len(corners_img))])
corners_block, _ = det.rm_text(
org, corners_block, ['block' for i in range(len(corners_block))])
corners_img, _ = det.rm_text(org, corners_img,
['img' for i in range(len(corners_img))])
corners_compo, compos_class = det.rm_text(org, corners_compo,
compos_class)
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# *** Step 6 *** text detection from cleaned image
img_clean = draw.draw_bounding_box(org,
corners_img,
color=(255, 255, 255),
line=-1)
corners_word = ocr.text_detection(org, img_clean)
corners_text = ocr.text_merge_word_into_line(org, corners_word)
# *** Step 7 *** img inspection: search components in img element
if is_img_inspect:
corners_block, corners_img, corners_compo, compos_class = det.compo_in_img(
processing, org, binary, corners_img, corners_block,
corners_compo, compos_class)
return corners_block, corners_img, corners_compo, compos_class, corners_text
# *** used for img inspection ***
# only consider rectangular components
else:
boundary_rec, boundary_non_rec = det.boundary_detection(binary)
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img, corners_compo = det.block_or_compo(
org, binary, corners_rec)
compos = seg.clipping(org, corners_compo)
compos_class = CNN.predict(compos)
corners_compo, compos_class = det.compo_filter(org, corners_compo,
compos_class, is_icon)
corners_compo, compos_class = det.strip_img(corners_compo,
compos_class, corners_img)
corners_block, _ = det.rm_text(
org, corners_block, ['block' for i in range(len(corners_block))])
corners_compo, compos_class = det.rm_text(org, corners_compo,
compos_class)
return corners_block, corners_compo, compos_class
# initialization
start = time.clock()
is_detect_line = False
is_merge_img = False
is_shrink_img = False
is_ocr = True
is_segment = False
is_save = True
# *** Step 1 *** pre-processing: gray, gradient, binary
org, gray = pre.read_img('input/5.png', (0, 600)) # cut out partial img
binary = pre.preprocess(gray, 1)
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_all, boundary_rec, boundary_nonrec = det.boundary_detection(binary)
# get corner of boundaries
corners_rec = det.get_corner(boundary_rec)
corners_nonrec = det.get_corner(boundary_nonrec)
# *** Step 3 *** process data: identify blocks and imgs from rectangles -> identify compos -> identify irregular imgs
# identify rectangular block and rectangular img from rectangular shapes
corners_block, corners_img = det.img_or_block(org, binary, corners_rec)
# identify potential buttons and input bars
corners_block, corners_compo = det.uicomponent_or_block(org, corners_block)
# shrink images with extra borders
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# identify irregular-shape img from irregular shapes
corners_img += det.img_irregular(org, corners_nonrec)
# ignore too large and highly likely text areas
# *** Step 2 *** line detection: for better boundary detection
if is_detect_line:
line_h, line_v = det.line_detection(bin, C.THRESHOLD_LINE_MIN_LENGTH_H,
C.THRESHOLD_LINE_MIN_LENGTH_V,
C.THRESHOLD_LINE_THICKNESS)
bin_no_line = det.rm_line(bin, [line_h, line_v])
binary = bin_no_line
else:
binary = bin
# *** Step 3 *** object detection: get connected areas -> get boundary -> get corners
boundary_all, boundary_rec, boundary_nonrec = det.boundary_detection(
binary,
C.THRESHOLD_OBJ_MIN_AREA,
C.THRESHOLD_OBJ_MIN_PERIMETER, # size of area
C.THRESHOLD_LINE_THICKNESS, # line check
C.THRESHOLD_REC_MIN_EVENNESS,
C.THRESHOLD_IMG_MAX_DENT_RATIO) # rectangle check
# get corner of boundaries
corners_rec = det.get_corner(boundary_rec)
corners_nonrec = det.get_corner(boundary_nonrec)
# *** Step 4 *** process data: identify blocks and imgs from rectangles -> identify compos -> identify irregular imgs
# identify rectangular block and rectangular img from rectangular shapes
corners_block, corners_img = det.img_or_block(
org, binary, corners_rec, C.THRESHOLD_BLOCK_MAX_BORDER_THICKNESS,
C.THRESHOLD_BLOCK_MAX_CROSS_POINT) # block check
# identify potential buttons and input bars
corners_block, corners_compo = det.uicomponent_or_block(
org, corners_block, C.THRESHOLD_UICOMPO_MAX_HEIGHT,
print(input_path)
print(time.ctime())
out_img_draw = pyjoin(C.ROOT_IMG_DRAWN, index + '.png')
out_img_clean = pyjoin(C.ROOT_IMG_CLEAN, index + '.png')
out_img_gradient = pyjoin(C.ROOT_IMG_GRADIENT, index + '.png')
out_img_segment = pyjoin(C.ROOT_IMG_SEGMENT, index)
out_label = pyjoin(C.ROOT_LABEL, index)
# *** Step 1 *** pre-processing: gray, gradient, binary
org, gray = pre.read_img(input_path, (0, 2600)) # cut out partial img
if org is None or gray is None: continue
binary = pre.preprocess(gray, 1)
# *** Step 2 *** processing: get connected areas -> get boundary -> rectangle check
boundary_rec, boundary_all = det.boundary_detection(
binary, C.THRESHOLD_MIN_OBJ_AREA, C.THRESHOLD_MIN_REC_PARAMETER,
C.THRESHOLD_MIN_REC_EVENNESS, C.THRESHOLD_MAX_LINE_THICKNESS,
C.THRESHOLD_MIN_LIN_LENGTH, C.THRESHOLD_MAX_IMG_DENT_RATIO)
# get corner of boundaries -> img or block check
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img = det.block_or_img(
binary, corners_rec, C.THRESHOLD_MAX_BLOCK_BORDER_THICKNESS,
C.THRESHOLD_MAX_BLOCK_CROSS_POINT)
# refine img component
corners_img = det.img_refine2(corners_img, C.THRESHOLD_MAX_IMG_EDGE_RATIO,
C.THRESHOLD_MUST_IMG_HEIGHT,
C.THRESHOLD_MUST_IMG_WIDTH)
# *** Step 3 *** post-processing: remove img elements from original image and segment into smaller size
img_clean = draw.draw_bounding_box(corners_img, org, (255, 255, 255), -1)
seg.segment_img(img_clean, 600, out_img_segment, 0)
# draw results
C = Config()
input_root = C.IMG_ROOT
output_root = C.OUTPUT_ROOT
is_save = True
is_show = False
start = time.clock()
# pre-processing: gray, gradient, binary
org, gray = pre.read_img('input/1.png', (0, 3000)) # cut out partial img
binary = pre.preprocess(gray, 1)
# processing: get connected areas -> get boundary -> rectangle check -> get corner of boundaries -> img or frame check -> refine img component
boundary_rec, boundary_non_rec = det.boundary_detection(
binary, C.THRESHOLD_MIN_OBJ_AREA, C.THRESHOLD_MIN_REC_PARAMETER,
C.THRESHOLD_MIN_REC_EVENNESS, C.THRESHOLD_MIN_LINE_THICKNESS)
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img = det.block_or_img(binary, corners_rec,
C.THRESHOLD_MAX_BORDER_THICKNESS)
corners_img = det.img_refine2(corners_img, C.THRESHOLD_MAX_EDGE_RATIO)
# remove img elements and segment into smaller size
img_clean = draw.draw_bounding_box(corners_img, org, (255, 255, 255), -1)
seg.segment_img(img_clean, 600, 'output/segment')
# draw results
draw_bounding = draw.draw_bounding_box(corners_block, org, (0, 255, 0))
draw_bounding = draw.draw_bounding_box(corners_img, draw_bounding, (0, 0, 255))
draw_boundary = draw.draw_boundary(boundary_rec, org.shape)
