def generate_yatzy_sheet(img, num_rows_in_grid=19, max_num_cols=20):
img = resize_to_right_ratio(img)
# Step 1
img_adaptive_binary = get_adaptive_binary_image(img)
cv_utils.show_window('img_adaptive_binary', img_adaptive_binary)
# Step2 and 3, Find the biggest contour and rotate it
img_yatzy_sheet, img_binary_yatzy_sheet = get_rotated_yatzy_sheet(
img, img_adaptive_binary)
# Step 4, Get a painted grid with vertical / horizontal lines
img_binary_grid, img_binary_only_numbers = get_yatzy_grid(
img_binary_yatzy_sheet)
# Step 5, Get every yatzy grid cell as a sorted bounding rect in order to later locate numbers to correct cell
yatzy_cells_bounding_rects, grid_bounding_rect = get_yatzy_cells_bounding_rects(
img_binary_grid, num_rows_in_grid, max_num_cols)
# Get the area of the yatzy grid from different versions of raw img
img_binary_only_numbers = cv_utils.get_bounding_rect_content(
img_binary_only_numbers, grid_bounding_rect)
img_binary_yatzy_sheet = cv_utils.get_bounding_rect_content(
img_binary_yatzy_sheet, grid_bounding_rect)
img_yatzy_sheet = cv_utils.get_bounding_rect_content(
img_yatzy_sheet, grid_bounding_rect)
return img_yatzy_sheet, img_binary_yatzy_sheet, img_binary_only_numbers, yatzy_cells_bounding_rects
def get_rotated_image_from_contour(img, contour):
rotated_rect = cv2.minAreaRect(contour)
# Get the center x,y and width and height.
x_center = int(rotated_rect[0][0])
y_center = int(rotated_rect[0][1])
width = int(rotated_rect[1][0])
height = int(rotated_rect[1][1])
angle_degrees = rotated_rect[2]
if (width > height):
temp_height = height
height = width
width = temp_height
angle_degrees = 90 + angle_degrees
# Reassign rotated rect with updated values
rotated_rect = ((x_center, y_center), (width, height), angle_degrees)
# Find the 4 (x,y) coordinates for the rotated rectangle, order: bl, tl,tr, br
rect_box_points = cv2.boxPoints(rotated_rect)
img_debug_contour = img.copy()
cv2.drawContours(img_debug_contour, [contour], 0, (0, 0, 255), 3)
cv_utils.show_window('biggest_contour', img_debug_contour)
img_debug = img.copy()
cv2.drawContours(img_debug, [np.int0(rect_box_points)], 0, (0, 0, 255), 3)
cv_utils.show_window('min_area_rect_original_image', img_debug)
# Prepare for rotation transformation
src_pts = rect_box_points.astype("float32")
dst_pts = np.array(
[
[0, height - 1], # Bottom Left
[0, 0], # Top Left
[width - 1, 0], # Top Right
],
dtype="float32")
# Affine rotation transformation
ROTATION_MAT = cv2.getAffineTransform(src_pts[:3], dst_pts)
return cv2.warpAffine(img, ROTATION_MAT, (width, height))
def get_yatzy_grid(img_binary_sheet):
""" Returns a binary image with a grid and the input image containing only horizontal/vertical lines.
Args:
img_binary_sheet ((rows,col) array): binary image
Returns:
img_binary_grid: an image containing painted vertically and horizontally lines
img_binary_sheet_only_digits: an image containing only(mostly) handwritten digits
"""
height, width = img_binary_sheet.shape
img_binary_sheet_morphed = img_binary_sheet.copy()
# Now we have the binary image with adaptive threshold.
# We need to do some morphylogy operations in order to strengthen thin lines, remove noise, and also handwritten stuff.
# We only want the horizontal / vertical pixels left before we start identifying the grid. See http://homepages.inf.ed.ac.uk/rbf/HIPR2/morops.htm
# CLOSING: (dilate -> erode) will fill in background (black) regions with White. Imagine sliding struct element
# in the background pixel, if it cannot fit the background completely(touching the foreground), fill this pixel with white
# OPENING: ALL FOREGROUND PIXELS(white) that can fit the structelement will be white, else black.
# Erode -> Dilate
# Erosion: If the structuring element can fit inside the forground pixel(white), then keep white, else set to black
# Dilation: For every background pixel(black), if one of the foreground(white) pixels are present, set this background (black) to foreground.
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
img_binary_sheet_morphed = cv2.morphologyEx(img_binary_sheet_morphed,
cv2.MORPH_DILATE, kernel)
cv_utils.show_window('morph_dilate_binary_img', img_binary_sheet_morphed)
sheet_binary_grid_horizontal = img_binary_sheet_morphed.copy()
sheet_binary_grid_vertical = img_binary_sheet_morphed.copy()
# We use relative length for the structuring line in order to be dynamic for multiple sizes of the sheet.
structuring_line_size = int(width / 5.0)
# Try to remove all vertical stuff in the image,
element = cv2.getStructuringElement(cv2.MORPH_RECT,
(structuring_line_size, 1))
sheet_binary_grid_horizontal = cv2.morphologyEx(
sheet_binary_grid_horizontal, cv2.MORPH_OPEN, element)
# Try to remove all horizontal stuff in image, Morph OPEN: Keep everything that fits structuring element i.e vertical lines
element = cv2.getStructuringElement(cv2.MORPH_RECT,
(1, structuring_line_size))
sheet_binary_grid_vertical = cv2.morphologyEx(sheet_binary_grid_vertical,
cv2.MORPH_OPEN, element)
# Concatenate the vertical/horizontal lines into grid
img_binary_sheet_morphed = cv2.add(sheet_binary_grid_vertical,
sheet_binary_grid_horizontal)
cv_utils.show_window("morph_keep_only_horizontal_lines",
sheet_binary_grid_horizontal)
cv_utils.show_window("morph_keep_only_vertical_lines",
sheet_binary_grid_vertical)
cv_utils.show_window("concatenate_vertical_horizontal",
img_binary_sheet_morphed)
"""
Time to get a solid grid, from what we see above, the grid is still not fully filled (sometimes)
since the paper is not fully straight on the table etc. For this we use Hough Transform
Hough transform identifies points (x,y) on the same line.
"""
# We ideally should choose np.pi / 2 for the Theta accumulator, since we only want lines in 90 degrees and 0 degrees.
rho_accumulator = 1
angle_accumulator = np.pi / 2
# Min vote for defining a line
threshold_accumulator_votes = int(width / 2)
# Find lines in the image according to the Hough Algorithm
grid_lines = cv2.HoughLines(img_binary_sheet_morphed, rho_accumulator,
angle_accumulator, threshold_accumulator_votes)
img_binary_grid = np.zeros(img_binary_sheet_morphed.shape,
dtype=img_binary_sheet_morphed.dtype)
# Since we can have multiple lines for same grid line, we merge nearby lines
grid_lines = merge_nearby_lines(grid_lines)
draw_lines(grid_lines, img_binary_grid)
# Since all sheets does not have outerborders. We draw a rectangle around the
outer_border = np.array([
[1, height - 1], # Bottom Left
[1, 1], # Top Left
[width - 1, 1], # Top Right
[width - 1, height - 1] # Bottom Right
])
cv2.drawContours(img_binary_grid, [outer_border], 0, (255, 255, 255), 3)
# Remove the grid from the binary image an keep only the digits.
img_binary_sheet_only_digits = cv2.bitwise_and(
img_binary_sheet, 255 - img_binary_sheet_morphed)
cv_utils.show_window("yatzy_grid_binary_lines", img_binary_grid)
return img_binary_grid, img_binary_sheet_only_digits
if args.debug:
cv_utils.set_debug(bool(args.debug))
if args.img_path:
img_path = args.img_path
print("Reading image from path", img_path)
input_img = cv2.imread(img_path)
img_yatzy_sheet, img_binary_yatzy_sheet, img_binary_only_numbers, yatzy_cells_bounding_rects = yatzy_sheet.generate_yatzy_sheet(
input_img, num_rows_in_grid=num_rows)
# Debugging step
img_yatzy_cells = img_yatzy_sheet.copy()
cv_utils.draw_bounding_rects(img_yatzy_cells, yatzy_cells_bounding_rects)
cv_utils.show_window('img_yatzy_cells', img_yatzy_cells)
digit_contours = cv_utils.get_external_contours(img_binary_only_numbers)
# Iterate over contours and predict numbers if the contours belong to a yatzy cell
for i, cnt in enumerate(digit_contours):
digit_bounding_rect = cv2.boundingRect(cnt)
x, y, w, h = digit_bounding_rect
# Identify if and to which yatzy cell this bounding rect belongs to
yatzy_cell = yatzy_sheet.validate_and_find_yatzy_cell(
yatzy_cells_bounding_rects, digit_bounding_rect)
if yatzy_cell is None:
continue