def compo_in_img(org, bin, corners):
def reverse(img):
rec, b = cv2.threshold(img, 1, 255, cv2.THRESH_BINARY_INV)
return b
corners_compo = []
pad = 2
for corner in corners:
(top_left, bottom_right) = corner
(col_min, row_min) = top_left
(col_max, row_max) = bottom_right
col_min = max(col_min - pad, 0)
col_max = min(col_max + pad, org.shape[1])
row_min = max(row_min - pad, 0)
row_max = min(row_max + pad, org.shape[0])
clip_bin = bin[row_min:row_max, col_min:col_max]
clip_bin = reverse(clip_bin)
boundary_all, boundary_rec, boundary_nonrec = det.boundary_detection(
clip_bin,
C.THRESHOLD_OBJ_MIN_AREA,
C.THRESHOLD_OBJ_MIN_PERIMETER, # size of area
C.THRESHOLD_LINE_THICKNESS, # line check
C.THRESHOLD_REC_MIN_EVENNESS_STRONG,
C.THRESHOLD_IMG_MAX_DENT_RATIO) # rectangle check
corners_rec = det.get_corner(boundary_rec)
corners_rec = utils.corner_cvt_relative_position(
corners_rec, col_min, row_min)
corners_compo += corners_rec
draw.draw_bounding_box(org, corners_compo, show=True)
return corners_compo
def processing(org, binary, clf, main=True):
if main:
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_rec, boundary_non_rec = det.boundary_detection(binary, write_boundary=True)
corners_rec = det.get_corner(boundary_rec)
corners_non_rec = det.get_corner(boundary_non_rec)
# *** Step 3 *** data processing: identify blocks and compos from rectangles -> identify irregular compos
corners_block, corners_img, corners_compo = det.block_or_compo(org, binary, corners_rec)
det.compo_irregular(org, corners_non_rec, corners_img, corners_compo)
# *** Step 4 *** classification: clip and classify the components candidates -> ignore noises -> refine img
compos = seg.clipping(org, corners_compo)
compos_class = clf.predict(compos)
corners_compo, compos_class = det.strip_img(corners_compo, compos_class, corners_img)
# *** Step 5 *** result refinement
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# *** Step 6 *** recursive inspection: search components nested in components
corners_block, corners_img, corners_compo, compos_class = det.compo_in_img(processing, org, binary, clf, corners_img, corners_block, corners_compo, compos_class)
# *** Step 7 *** ocr check and text detection from cleaned image
if is_ocr:
corners_block, _ = det.rm_text(org, corners_block, ['block' for i in range(len(corners_block))])
corners_img, _ = det.rm_text(org, corners_img, ['img' for i in range(len(corners_img))])
corners_compo, compos_class = det.rm_text(org, corners_compo, compos_class)
# *** Step 8 *** merge overlapped components
# corners_img = det.rm_img_in_compo(corners_img, corners_compo)
corners_img, _ = det.merge_corner(org, corners_img, ['img' for i in range(len(corners_img))], is_merge_nested_same=True)
corners_compo, compos_class = det.merge_corner(org, corners_compo, compos_class, is_merge_nested_same=True)
return corners_block, corners_img, corners_compo, compos_class
# *** used for img inspection ***
# only consider rectangular components
else:
boundary_rec, boundary_non_rec = det.boundary_detection(binary)
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img, corners_compo = det.block_or_compo(org, binary, corners_rec)
compos = seg.clipping(org, corners_compo)
compos_class = clf.predict(compos)
corners_compo, compos_class = det.strip_img(corners_compo, compos_class, corners_img)
return corners_block, corners_compo, compos_class
def processing(org, binary, main=True):
if main:
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_rec, boundary_non_rec = det.boundary_detection(binary,
show=False)
corners_rec = det.get_corner(boundary_rec)
corners_non_rec = det.get_corner(boundary_non_rec)
# *** Step 3 *** data processing: identify blocks and compos from rectangles -> identify irregular compos
corners_block, corners_img, corners_compo = det.block_or_compo(
org, binary, corners_rec)
det.compo_irregular(org, corners_non_rec, corners_img, corners_compo)
corners_img, _ = det.rm_text(org, corners_img,
['img' for i in range(len(corners_img))])
# *** Step 4 *** classification: clip and classify the components candidates -> ignore noises -> refine img
compos = seg.clipping(org, corners_compo)
compos_class = CNN.predict(compos)
# corners_compo, compos_class = det.compo_filter(org, corners_compo, compos_class, is_icon)
corners_compo, compos_class = det.strip_img(corners_compo,
compos_class, corners_img)
# *** Step 5 *** result refinement
if is_merge:
corners_img, _ = det.merge_corner(
corners_img, ['img' for i in range(len(corners_img))])
corners_block, _ = det.rm_text(
org, corners_block, ['block' for i in range(len(corners_block))])
corners_img, _ = det.rm_text(org, corners_img,
['img' for i in range(len(corners_img))])
corners_compo, compos_class = det.rm_text(org, corners_compo,
compos_class)
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# *** Step 6 *** text detection from cleaned image
img_clean = draw.draw_bounding_box(org,
corners_img,
color=(255, 255, 255),
line=-1)
corners_word = ocr.text_detection(org, img_clean)
corners_text = ocr.text_merge_word_into_line(org, corners_word)
# *** Step 7 *** img inspection: search components in img element
if is_img_inspect:
corners_block, corners_img, corners_compo, compos_class = det.compo_in_img(
processing, org, binary, corners_img, corners_block,
corners_compo, compos_class)
return corners_block, corners_img, corners_compo, compos_class, corners_text
# *** used for img inspection ***
# only consider rectangular components
else:
boundary_rec, boundary_non_rec = det.boundary_detection(binary)
corners_rec = det.get_corner(boundary_rec)
corners_block, corners_img, corners_compo = det.block_or_compo(
org, binary, corners_rec)
compos = seg.clipping(org, corners_compo)
compos_class = CNN.predict(compos)
corners_compo, compos_class = det.compo_filter(org, corners_compo,
compos_class, is_icon)
corners_compo, compos_class = det.strip_img(corners_compo,
compos_class, corners_img)
corners_block, _ = det.rm_text(
org, corners_block, ['block' for i in range(len(corners_block))])
corners_compo, compos_class = det.rm_text(org, corners_compo,
compos_class)
return corners_block, corners_compo, compos_class
start = time.clock()
is_detect_line = False
is_merge_img = False
is_shrink_img = False
is_ocr = True
is_segment = False
is_save = True
# *** Step 1 *** pre-processing: gray, gradient, binary
org, gray = pre.read_img('input/5.png', (0, 600)) # cut out partial img
binary = pre.preprocess(gray, 1)
# *** Step 2 *** object detection: get connected areas -> get boundary -> get corners
boundary_all, boundary_rec, boundary_nonrec = det.boundary_detection(binary)
# get corner of boundaries
corners_rec = det.get_corner(boundary_rec)
corners_nonrec = det.get_corner(boundary_nonrec)
# *** Step 3 *** process data: identify blocks and imgs from rectangles -> identify compos -> identify irregular imgs
# identify rectangular block and rectangular img from rectangular shapes
corners_block, corners_img = det.img_or_block(org, binary, corners_rec)
# identify potential buttons and input bars
corners_block, corners_compo = det.uicomponent_or_block(org, corners_block)
# shrink images with extra borders
if is_shrink_img:
corners_img = det.img_shrink(org, binary, corners_img)
# identify irregular-shape img from irregular shapes
corners_img += det.img_irregular(org, corners_nonrec)
# ignore too large and highly likely text areas
corners_img = det.img_refine(org, corners_img)
# merge overlapped corners, and remove nested corners
def block_division(grey, show=False):
'''
:param grey: grey-scale of original image
:return: corners: list of [(top_left, bottom_right)]
-> top_left: (column_min, row_min)
-> bottom_right: (column_max, row_max)
'''
def flood_fill_bfs(img, x_start, y_start, mark):
'''
Identify the connected region based on the background color
:param img: grey-scale image
:param x_start: row coordinate of start position
:param y_start: column coordinate of start position
:param mark: record passed points
:return: region: list of connected points
'''
def neighbor(x, y):
for i in range(x - 1, x + 2):
if i < 0 or i >= img.shape[0]: continue
for j in range(y - 1, y + 2):
if j < 0 or j >= img.shape[1]: continue
if mark[i, j] == 0 and abs(img[i, j] - img[x, y]) < 8:
stack.append([i, j])
mark[i, j] = 255
stack = [[x_start, y_start]] # points waiting for inspection
region = [[x_start, y_start]] # points of this connected region
mark[x_start, y_start] = 255 # drawing broad
while len(stack) > 0:
point = stack.pop()
region.append(point)
neighbor(point[0], point[1])
return region
blocks = []
mask = np.zeros((grey.shape[0], grey.shape[1]), dtype=np.uint8)
broad = np.zeros((grey.shape[0], grey.shape[1], 3), dtype=np.uint8)
row, column = grey.shape[0], grey.shape[1]
for x in range(row):
for y in range(column):
if mask[x, y] == 0:
region = flood_fill_bfs(grey, x, y, mask)
# ignore small regions
if len(region) < 500:
continue
# get the boundary of this region
boundary = util.boundary_get_boundary(region)
# ignore lines
if util.boundary_is_line(boundary, 5):
continue
# ignore non-rectangle as blocks must be rectangular
if not util.boundary_is_rectangle(boundary, 0.66, 0.25):
continue
blocks.append(boundary)
draw_region(region, broad)
if show:
cv2.imshow('broad', broad)
cv2.waitKey()
blocks_corner = det.get_corner(blocks)
return blocks_corner