def EstimateLines(image, left_margin, right_margin, min_peak_distance,
orig_img, container):
line_slices = []
window = 250
lw_sizes = []
peak_sizes = []
## debug output
imgLines = orig_img.copy()
for i in range(left_margin + 1, right_margin):
roi = image[:, max(left_margin, i - window):i]
intensity_hist, peaks, intensity_hist_s = findpeaks(
roi, min_peak_distance)
"""if (i%500 == 0):
plt.figure(figsize=(10,5))
plt.xticks(range(0, image.shape[0], int(image.shape[0]/10)))
plt.xlim([0, image.shape[0]])
plt.plot(intensity_hist)
plt.plot(peaks, intensity_hist[peaks], "x")
plt.show()
"""
## start recording vertical slice peaks, later used for threading
peak_sizes.append(len(peaks))
if (len(peaks) > 1):
lw = getlinewidth(peaks)
lw_sizes.append(lw)
else:
print('### no peaks! y=', i)
line_slices.append(peaks)
for _, p in enumerate(peaks):
## draw a broad band 9px wide
imgLines[p - 4, i] = [255, 0, 0]
imgLines[p - 3, i] = [255, 0, 0]
imgLines[p - 2, i] = [255, 0, 0]
imgLines[p - 1, i] = [255, 0, 0]
imgLines[p, i] = [255, 0, 0]
imgLines[p + 1, i] = [255, 0, 0]
imgLines[p + 2, i] = [255, 0, 0]
imgLines[p + 3, i] = [255, 0, 0]
imgLines[p + 4, i] = [255, 0, 0]
writedebugimage(imgLines, container, 'lines-with-smudge')
logging.info(f'length of peak_sizes : {len(peak_sizes)}')
return np.max(peak_sizes), np.median(peak_sizes), np.mean(
lw_sizes), np.std(lw_sizes), np.min(lw_sizes), line_slices
def ThresholdWithOTSU(image, container, orig_img):
thirdH = int(image.shape[0] / 3)
thirdW = int(image.shape[1] / 3)
roi = image[thirdH:2 * thirdH, thirdW:2 * thirdW]
#histr = cv2.calcHist([image],[0],None,[256],[0,256])
#plt.plot(histr)
#plt.show()
histr = cv2.calcHist([roi], [0], None, [256], [0, 256])
#plt.plot(histr)
#plt.show()
img2 = image.copy()
ot, roit = cv2.threshold(roi, 0, 255, cv2.THRESH_OTSU)
_, ti = cv2.threshold(img2, ot, 255, cv2.THRESH_BINARY_INV)
writedebugimage(ti, container, 'raw-otsu')
contours, _ = cv2.findContours(ti, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
mask = np.ones(image.shape[:2], dtype="uint8") * 255
mask_remove = orig_img.copy()
mask_retain = orig_img.copy()
# loop over the contours
for c in contours:
# if the contour is bad, draw it on the mask
if is_good_contour_for_line_segmentation(c, ti) == True:
r = cv2.boundingRect(c)
cv2.rectangle(mask, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]),
color=0,
thickness=-1)
cv2.drawContours(mask_retain, [c],
0,
color=(0, 255, 0),
thickness=1)
else:
#r = cv2.boundingRect(c)
cv2.drawContours(mask_remove, [c],
0,
color=(0, 0, 255),
thickness=-1)
writedebugimage(mask_retain, container, 'retaining-this')
writedebugimage(mask_remove, container, 'removing-this')
# remove the contours from the image and show the resulting images
writedebugimage(255 - mask, container, 'contour-removal-mask')
## AARGH! someone please simplify this! X is the correct form, it has already been inverted so text is white.
x = cv2.bitwise_and(ti, ti, mask=255 - mask)
return x
def ThresholdSauvola(img, container, orig_img):
## based on observation using these values of k,r work best (default k=0.2)
thresh_sauvola = threshold_sauvola(img.copy(), window_size=31, k=0.1, r=45)
binary_sauvola = img > thresh_sauvola
image = binary_sauvola.astype('uint8') * 255
## Sauvola is edge sensitive and prone to creating thick borders around image. Try remove them based on size
contours, _ = cv2.findContours(255 - image.copy(), cv2.RETR_LIST,
cv2.CHAIN_APPROX_NONE)
mask = np.ones(img.shape[:2], dtype="uint8") * 255
mask_remove = orig_img.copy()
mask_retain = orig_img.copy()
# loop over the contours
for c in contours:
# if the contour is bad, draw it on the mask
if is_good_contour_for_line_segmentation(c,
255 - image.copy()) == True:
r = cv2.boundingRect(c)
cv2.rectangle(mask, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]),
color=0,
thickness=-1)
cv2.drawContours(mask_retain, [c],
0,
color=(0, 255, 0),
thickness=1)
else:
#r = cv2.boundingRect(c)
cv2.drawContours(mask_remove, [c],
0,
color=(0, 0, 255),
thickness=-1)
writedebugimage(mask_retain, container, 'retaining-this')
writedebugimage(mask_remove, container, 'removing-this')
# remove the contours from the image and show the resulting images
writedebugimage(255 - mask, container, 'contour-removal-mask')
## AARGH! someone please simplify this! X is the correct form, it has already been inverted so text is white.
x = cv2.bitwise_and(255 - image.copy(), 255 - mask, mask=255 - mask)
return x
def main(event: func.EventGridEvent, context: func.Context):
cwd = context.function_directory
session_id = str(context.invocation_id)
logging.info(f'session_id: {session_id}')
if (event.event_type != "Microsoft.Storage.BlobCreated"):
logging.info('Received delete event. Skip processing %s',
event.subject)
return
container, filename, manuscriptid, file_url = parse_input(event)
if (container == None):
logging.info('Error parsing event. Skip processing %s',
event.get_json())
return
if (filename.endswith(".jpg") != True):
logging.info('Received non jpg file. Skip processing %s',
event.get_json())
return
## download the file locally.
logging.info('Received container %s, filename %s, file_url %s', container,
filename, file_url)
orig_img = download_image(container, filename, session_id, manuscriptid)
## Steps 0-4 are preprocessing - nlmeans denoising, bilateral, medianblur, gray scale
grey_img = PreProcessImage(orig_img, container)
if (IsLowContrast(grey_img)):
bw_img = ThresholdSauvola(grey_img, container, orig_img)
writedebugimage(bw_img, container, 'sauvola-4')
else:
bw_img = ThresholdWithOTSU(grey_img, container, orig_img)
writedebugimage(bw_img, container, 'otsu-4')
# %%
## find approx left and right margins.
left_margin, right_margin = FindMargins(bw_img)
logging.info('left margin=%s, right margin=%s', left_margin, right_margin)
## find approx linewidth assuming straight lines. This does not account for skew but creates baseline for more precise algo
## later
peakpositions, linewidth, straight_line_contours = DoStraightLines(
bw_img, left_margin, right_margin)
num_straight_lines = len(peakpositions)
logging.info('#straightlines=%s, linewidth estimate=%s',
num_straight_lines, linewidth)
# %%
## Try to account for line skew. If all goes well then line_contours has accurage peak info else peakpositions has peak info
img2 = DoHorizontalSmudge(bw_img.copy())
writedebugimage(img2, container, 'horizontal-smudge-45')
## get intensity profile of moving window starting from left margin
min_peak_distance = int(0.75 * linewidth)
num_lines, num_lines_median, mean_linewidth, min_linewidth, std_linewidth, line_data = EstimateLines(
img2.copy(), left_margin, right_margin, min_peak_distance, orig_img,
container)
logging.info(
'Approx1: #maxlines=%s, mean linewidth=%s, min linewidth=%s, stddev linewidth=%s, #medianlines=%s'
.format(num_lines, mean_linewidth, min_linewidth, std_linewidth,
num_lines_median))
# %%
line_contours = []
#lines should not differ much from straight line estimate - if it does fall back to using straight lines!
if (abs(num_lines - num_straight_lines) <= 5):
num_straight_lines = int(num_lines)
line_contours = CreateLineContours(line_data, num_straight_lines,
left_margin, right_margin)
else:
print('Using straight line algo!')
line_contours = straight_line_contours
img3 = orig_img.copy()
cv2.polylines(img3, line_contours, False, (255, 0, 0))
writedebugimage(img3, container, 'line-contours-5')
## Generate approx line bottoms for front end to draw
approx_lines = np.array([
np.percentile(np.array([x[0][1] for x in m]), 75)
for m in line_contours
])
aw = getlinewidth(approx_lines)
approx_line_bottoms = approx_lines.astype('uint') + int(aw / 2)
#print(approx_line_bottoms)
lines_filename = 'lines-' + str(manuscriptid) + '.csv'
upload_csv(approx_line_bottoms, container, lines_filename)
logging.info('uploaded approx lines for FE')
## Line determination done, we are ready to run contouring to detect blocks of text
contours = GetContours(bw_img)
logging.info(f'total contours: {len(contours)}')
## hold contour, rect, hull, centroid in one place
contourdatalist = []
for (i, c) in enumerate(contours):
# compute bounding box of contour
rect = cv2.boundingRect(c)
hull = cv2.convexHull(c)
m = cv2.moments(c)
if (m["m00"] != 0):
cx = int(m["m10"] / m["m00"])
cy = int(m["m01"] / m["m00"])
contourdatalist.append((c, rect, hull, (cx, cy)))
else:
## TODO debug why these werent caught earlier? minAreaRect issue or external vs all issue?
print('zero moment rect!', rect)
## split any rect spanning lines into smaller rects
cdlist, s, c, d = PreProcessBoundingRects1(contourdatalist, linewidth,
bw_img, orig_img, container)
logging.info('Done PreprocessingBR1')
## remove very small rects or fully contained in another
cdlist = PreProcessBoundingRects2(cdlist)
logging.info('Done PreprocessingBR2')
oi1 = orig_img.copy()
bi1 = orig_img.copy()
bi1[:, :, :] = 255
EmitLinesAndRects(line_contours, cdlist, oi1, bi1, (255, 0, 0),
(0, 255, 0))
writedebugimage(oi1, container, 'lines-rects')
writedebugimage(bi1, container, 'only-lines-rects')
oi2 = orig_img.copy()
bi2 = orig_img.copy()
bi2[:, :, :] = 255
EmitLinesAndConvexHulls(line_contours, cdlist, oi2, bi2, (255, 0, 0),
(0, 255, 0))
writedebugimage(oi2, container, 'lines-hulls')
writedebugimage(bi2, container, 'only-lines-hulls')
## assign each rectangle to a line
linerects = ArrangeLineByLine(cdlist, line_contours)
logging.info('Done line assignments')
## merge certain rectangles after line numbering assigned
logging.info(f'Calling BR3 with linewidth {linewidth}')
updatedlinerects = []
for _, lr in enumerate(linerects):
mergedlr = PreProcessBR3(lr, linewidth)
updatedlinerects.append(mergedlr)
## final cleanup to trim rects with skewed aspect ratio or lying outside margins
updatedlinerects = PostProcessBoundingRects4(updatedlinerects, linewidth,
left_margin, right_margin)
logging.info('Done PreprocessingBR4')
oi3 = orig_img.copy()
bi3 = orig_img.copy()
bi3[:, :, :] = 255
numchar = 0
for (i, rc) in enumerate(updatedlinerects):
for (j, r) in enumerate(rc):
cv2.rectangle(oi3, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]),
((i % 2) * 255, ((i + 1) % 2) * 255, 0), 1)
cv2.rectangle(bi3, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]),
((i % 2) * 255, ((i + 1) % 2) * 255, 0), 1)
numchar = numchar + 1
cv2.polylines(oi3, line_contours, False, (127, 0, 127))
cv2.polylines(bi3, line_contours, False, (127, 0, 127))
logging.info(
f'Stats:Contours {len(contours)}, good rects {s}, compound rects {c}, chars {numchar}'
)
writedebugimage(oi3, container, 'final')
writedebugimage(bi3, container, 'final-rects')
## This prints the bounding rect coordinates. 2D array so csv print needs some more parsing.
rects_filename = 'rectangles-' + str(manuscriptid) + '.csv'
upload_csv(updatedlinerects, container, rects_filename)
logging.debug('Uploaded bounding rects. Done!')
return
def ShowBoundingRects(cdlist, oi, title, container):
for (j, cd) in enumerate(cdlist):
r = cd[1]
cv2.rectangle(oi, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]),
(255, 0, 255), 1)
writedebugimage(oi, container, title)