def __init__(self, board_width, resource_path, start_path):
self.board_width = board_width
self.cube_size = 1
self.start_path = start_path
self.resource_path = resource_path
optional_path = "3d_cube_with_ar/"
required_path = "font/white_rabbit_numbers.jpg"
self.font = cv.imread("%s%s%s" %
(start_path, optional_path, required_path))
font_six_points = [[16, 20], [80, 80]]
blue_tint = np.empty(self.font.shape, dtype=np.uint8)
blue_tint[:, :] = (155, 155, 0)
self.font[16:16 + 80,
20:20 + 80] = cv.subtract(self.font[16:16 + 80, 20:20 + 80],
blue_tint[16:16 + 80, 20:20 + 80])
self.font_inv = cv.bitwise_not(self.font)
font_one_points = [[16, 155], [80, 80]]
font_two_points = [[20, 155 + 80 + 50], [80, 80]]
font_three_points = [[20, 155 + (80 + 50) * 2], [80, 80]]
font_four_points = [[22, 155 + (80 + 50) * 3 - 5], [80, 80]]
font_five_points = [[22, 155 + (80 + 50) * 4 - 10], [80, 80]]
self.font_point = np.array([
font_one_points, font_two_points, font_three_points,
font_four_points, font_five_points, font_six_points
])
self.draw_cube = DrawCube(resource_path)
self.common_core = CommonCore(None)
def GET(self, grade):
grade = parse_int_param(grade, 5)
common_core = CommonCore()
data = common_core.get_grade(grade)
if not data:
logging.warn('Cannot find data for grade %d' % grade)
return error_response(404)
try:
grade_json = encode_json(data)
return grade_json
except Exception, e:
logging.error(e)
return error_response(500)
# 2 should work and disagree with 3
# 3 should work and disagree with 2
# 4 should work and disagree with 5
# 5 should work and disagree with 4
# 6 should work and agree with 7
# 7 should work and agree with 6
pictures = [['opencv_frame_0_0.png', 'opencv_frame_0_1.png'],
['opencv_frame_7_0.png', 'opencv_frame_7_1.png'],
['opencv_frame_7_1.png', 'opencv_frame_7_0.png'],
['opencv_frame_5_0.png', 'opencv_frame_5_1.png']]
board_width = 7
board_size = (board_width, board_width)
draw_cube = DrawCube(resources_path)
common_core = CommonCore(board_size)
ar_input = ArInput(board_width, resources_path, '')
board_rotation_f1 = BoardRotation()
board_rotation_f2 = BoardRotation()
def analyze_image_and_add_ar(frame_1, frame_2):
# frame_1 is suggestion and display frame, frame_2 is confirm frame
img_1 = frame_1
chessboard_img_1, corners1_1, ret_1 = common_core.find_chessboard_corners_1(
img_1)
# fail program if points not found
if not ret_1:
frame_wait_cnt = 10
print('1st checkerboard NOT detected, wait %s frames' %
class ArInput:
def __init__(self, board_width, resource_path, start_path):
self.board_width = board_width
self.cube_size = 1
self.start_path = start_path
self.resource_path = resource_path
optional_path = "3d_cube_with_ar/"
required_path = "font/white_rabbit_numbers.jpg"
self.font = cv.imread("%s%s%s" %
(start_path, optional_path, required_path))
font_six_points = [[16, 20], [80, 80]]
blue_tint = np.empty(self.font.shape, dtype=np.uint8)
blue_tint[:, :] = (155, 155, 0)
self.font[16:16 + 80,
20:20 + 80] = cv.subtract(self.font[16:16 + 80, 20:20 + 80],
blue_tint[16:16 + 80, 20:20 + 80])
self.font_inv = cv.bitwise_not(self.font)
font_one_points = [[16, 155], [80, 80]]
font_two_points = [[20, 155 + 80 + 50], [80, 80]]
font_three_points = [[20, 155 + (80 + 50) * 2], [80, 80]]
font_four_points = [[22, 155 + (80 + 50) * 3 - 5], [80, 80]]
font_five_points = [[22, 155 + (80 + 50) * 4 - 10], [80, 80]]
self.font_point = np.array([
font_one_points, font_two_points, font_three_points,
font_four_points, font_five_points, font_six_points
])
self.draw_cube = DrawCube(resource_path)
self.common_core = CommonCore(None)
def update_cube_size(self, new_cube_size):
self.cube_size = new_cube_size
def look_for_cube_size_v1(self, img, corners):
cube_return = self.cube_size
self.cube_size = (self.cube_size % 6) + 1
return cube_return
def look_for_cube_size_v2(self, img, corners2, draw_buttons=True):
hue, sat, val = cv.split(cv.cvtColor(img, cv.COLOR_BGR2HSV))
image_sums = np.zeros((6, 2), dtype=np.int32)
greatest_sum = 0
greatest_sum_roi_corners = -1
for square in range(6):
# acquire points from corners to use for ar buttons
first_bad_format = corners2[self.board_width * square, 0]
second_bad_format = corners2[self.board_width * square + 1, 0]
third_bad_format = corners2[self.board_width * (square + 1) + 1, 0]
fourth_bad_format = corners2[self.board_width * (square + 1), 0]
first = (first_bad_format[0], first_bad_format[1])
fourth = (fourth_bad_format[0], fourth_bad_format[1])
second = (int(first[0] * 2 - second_bad_format[0]),
int(first[1] * 2 - second_bad_format[1]))
third = (int(fourth[0] * 2 - third_bad_format[0]),
int(fourth[1] * 2 - third_bad_format[1]))
# put points into arrays for use below
roi_corners_for_mask = np.array([[first, second, third, fourth]],
dtype=np.int32)
roi_corners = np.array([[third], [second], [first], [fourth]],
dtype=np.int32)
# create mask with polygon
mask = np.zeros(sat.shape, dtype=np.uint8)
# channel_count = sat.shape[2]
channel_count = 1
ignore_mask_color = (255, ) * channel_count
cv.fillPoly(mask, roi_corners_for_mask, ignore_mask_color)
x_min, y_min, x_max, y_max = self.common_core.sort_points_for_extremes(
roi_corners_for_mask)
# smooth image
sat[y_min:y_max,
x_min:x_max] = cv.medianBlur(sat[y_min:y_max, x_min:x_max], 9)
val[y_min:y_max,
x_min:x_max] = cv.medianBlur(val[y_min:y_max, x_min:x_max], 9)
# black out all area outside roi_corners
masked_val_img = np.zeros(sat.shape, dtype=np.uint8)
masked_sat_img = np.zeros(sat.shape, dtype=np.uint8)
masked_val_img[y_min:y_max, x_min:x_max] = cv.bitwise_and(
val[y_min:y_max, x_min:x_max], mask[y_min:y_max, x_min:x_max])
masked_sat_img[y_min:y_max, x_min:x_max] = cv.bitwise_and(
sat[y_min:y_max, x_min:x_max], mask[y_min:y_max, x_min:x_max])
# threshold for high val and high sat
_, masked_val_img[y_min:y_max, x_min:x_max] = cv.threshold(
masked_val_img[y_min:y_max, x_min:x_max], 70, 255,
cv.THRESH_BINARY)
_, masked_sat_img[y_min:y_max, x_min:x_max] = cv.threshold(
masked_sat_img[y_min:y_max, x_min:x_max], 40, 255,
cv.THRESH_BINARY)
# combine high val and high sat
masked_val_sat_bin = masked_val_img
masked_val_sat_bin[y_min:y_max, x_min:x_max] = cv.bitwise_and(
masked_val_img[y_min:y_max, x_min:x_max],
masked_sat_img[y_min:y_max, x_min:x_max])
# count high val and high sat pixels
image__sum = (masked_val_sat_bin > 60).sum()
image_sums[square] = (square, image__sum)
# keep track of greatest sum roi_corners
if (image_sums[square][1] > greatest_sum):
greatest_sum = image_sums[square][1]
greatest_sum_roi_corners = roi_corners
greatest_masked_val_sat_bin = masked_val_sat_bin
# place button on square
if draw_buttons:
img = self.place_button_on_checkerboard(
img,
self.font,
self.font_point[square],
roi_corners,
mask=masked_val_sat_bin)
# sort sums
image_sums = sorted(image_sums, key=itemgetter(1))
# TODO unhard-code pixels of finger minimum
is_finger_detected = image_sums[-1][
1] > image_sums[-2][1] * 2 and image_sums[-1][1] > 500
square_with_greatest_detection = image_sums[-1][0]
if is_finger_detected:
button_pressed = square_with_greatest_detection + 1
return img, button_pressed, greatest_sum_roi_corners, greatest_masked_val_sat_bin
else:
return img, -1, None, None
def place_inverted_button_on_checkerboard(self, img, font_square,
greatest_masked_val_sat_bin,
greatest_sum_roi_corners):
img = self.place_button_on_checkerboard(
img,
self.font_inv,
self.font_point[font_square],
greatest_sum_roi_corners,
mask=greatest_masked_val_sat_bin)
return img
def place_button_on_checkerboard(self, img, font_img, font_point_square,
roi_corners, mask):
roi_corners_float = roi_corners.astype(np.float32)
img = self.draw_cube.add_image_to_base(img,
font_img,
roi_corners_float,
font_point_square[0],
font_point_square[1][0],
font_point_square[1][1],
mask=mask)
return img
def get_cube_size(self):
return self.cube_size
def __init__(self):
self.last_0th_checkerboard_point = [-1, -1]
self.common_core = CommonCore(None)
class BoardRotation:
def __init__(self):
self.last_0th_checkerboard_point = [-1, -1]
self.common_core = CommonCore(None)
def update_rotation(self, last_point):
self.last_0th_checkerboard_point = last_point
def get_rotated_corners(self, points):
rotation = 0
new_points = points
x_min, y_min, x_max, y_max = self.common_core.sort_points_for_extremes(
new_points)
distance_between_checkerboard_extremes = self.distance_between_points(
[x_min, y_min], [x_max, y_max])
if self.relative_0th_change(distance_between_checkerboard_extremes,
new_points[0, 0],
self.last_0th_checkerboard_point) > .2:
if self.relative_0th_change(distance_between_checkerboard_extremes,
new_points[6, 0],
self.last_0th_checkerboard_point) < .2:
rotation = 1
elif self.relative_0th_change(
distance_between_checkerboard_extremes, new_points[7 * 6,
0],
self.last_0th_checkerboard_point) < .2:
rotation = 3
elif self.relative_0th_change(
distance_between_checkerboard_extremes,
new_points[7 * 6 + 6,
0], self.last_0th_checkerboard_point) < .2:
rotation = 2
# TODO make these rearranges more efficient
if rotation == 1:
new_points = np.empty((49, 1, 2))
for i in range(7):
for k in range(7):
new_points[i + (6 - k) * 7] = points[i * 7 + k]
# 42, 35, 28, 21, 14, 7, 0, 43, 36
if rotation == 2:
new_points = np.empty((49, 1, 2))
for i in range(7):
for k in range(7):
new_points[(6 - i) * 7 + 6 - k] = points[i * 7 + k]
# 48, 47, 46
if rotation == 3:
new_points = np.empty((49, 1, 2))
for i in range(7):
for k in range(7):
new_points[6 + (k * 7) - i] = points[i * 7 + k]
# 6, 13, 20, 27, 34, 41, 48, 5, 12
self.last_0th_checkerboard_point = new_points[0, 0]
return new_points
def relative_0th_change(self, distance_between_checkerboard_extremes,
this_0th, last_0th):
distance_between_start_points = self.distance_between_points(
last_0th, this_0th)
relative_0th_change = distance_between_start_points / distance_between_checkerboard_extremes
return relative_0th_change
def distance_between_points(self, point_1, point_2):
distance_between_start_points = ((point_1[0] - point_2[0])**2 +
(point_1[1] - point_2[1])**2)**0.5
return distance_between_start_points
def __init__(self, resources_path):
self.bottom_cube_img = cv.imread(resources_path +
"to_project/devel_square.jpg")
self.common_core = CommonCore(None)
class DrawCube:
def __init__(self, resources_path):
self.bottom_cube_img = cv.imread(resources_path +
"to_project/devel_square.jpg")
self.common_core = CommonCore(None)
def draw_cube_pieces(self, corners, corners2, img, board_size, cube_size):
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
# Arrays to store object points and image points from all the images.
objp = np.zeros((board_size[1] * board_size[0], 3), np.float32)
objp[:, :2] = np.mgrid[0:board_size[0],
0:board_size[1]].T.reshape(-1, 2)
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.
objpoints.append(objp)
imgpoints.append(corners)
_, mtx, dist, _, _ = cv.calibrateCamera(objpoints, imgpoints,
(img.shape[1], img.shape[0]),
None, None)
# 3d Time
objp2 = np.zeros((board_size[1] * board_size[0], 3), np.float32)
objp2[:, :2] = np.mgrid[0:board_size[0],
0:board_size[1]].T.reshape(-1, 2)
edge_size = cube_size
x_offset = 0
y_offset = 0
z_offset = -0
axis3 = np.float32([
[x_offset, y_offset, z_offset],
[x_offset, y_offset + edge_size, z_offset],
[x_offset + edge_size, y_offset + edge_size, z_offset],
[x_offset + edge_size, y_offset, z_offset],
[x_offset, y_offset, z_offset - edge_size],
[x_offset, y_offset + edge_size, z_offset - edge_size],
[x_offset + edge_size, y_offset + edge_size, z_offset - edge_size],
[x_offset + edge_size, y_offset, z_offset - edge_size]
])
_, rvecs2, tvecs2 = cv.solvePnP(objp2, corners2, mtx, dist)
imgpts3, jac3 = cv.projectPoints(axis3, rvecs2, tvecs2, mtx, dist)
destination_points = np.array(
[imgpts3[0], imgpts3[3], imgpts3[2], imgpts3[1]])
# Read image to project and put it on image with same size as image to project to
project_img = self.bottom_cube_img
dev_rows, dev_cols, _ = project_img.shape
img = self.add_image_to_base(img, project_img, destination_points,
[0, 0], dev_cols, dev_rows)
img = self.draw_cube(img, imgpts3)
return img
def add_image_to_base(self,
img,
project_img,
dst_pts,
src_start_point,
src_width,
src_height,
mask=None):
# Put image passed in onto black image
img_to_use = np.zeros((img.shape[0], img.shape[1], 3), dtype=np.uint8)
mask_to_use = np.zeros((img.shape[0], img.shape[1], 1), dtype=np.uint8)
img_to_use[0:src_width, 0:src_height] = \
project_img[src_start_point[0]:src_start_point[0] + src_width, src_start_point[1]:src_start_point[1] + src_height]
# Create matrix to transform then warp
src_pts = np.array(
[[0, 0], [src_width, 0], [src_width, src_height], [0, src_height]],
dtype="float32")
matrix = cv.getPerspectiveTransform(src_pts, dst_pts)
warped_img = cv.warpPerspective(img_to_use, matrix,
(img.shape[1], img.shape[0]))
dst_pts_int = dst_pts.astype(int)
dst_pts_int[2][0][0] = math.ceil(dst_pts[2][0][0])
dst_pts_int[2][0][1] = math.ceil(dst_pts[2][0][1])
dst_pts_int[3][0][0] = math.ceil(dst_pts[3][0][0])
dst_pts_int[3][0][1] = math.ceil(dst_pts[3][0][1])
# Points for roi
x_min, y_min, x_max, y_max = self.common_core.sort_points_for_extremes(
dst_pts_int)
# create mask with white quadrangle around destination points
cv.fillConvexPoly(mask_to_use, dst_pts_int, 255)
mask_to_use_inv = cv.bitwise_not(mask_to_use)
if type(mask) != type(None):
mask_to_use_inv[y_min:y_max, x_min:x_max] = cv.bitwise_or(
mask[y_min:y_max, x_min:x_max], mask_to_use_inv[y_min:y_max,
x_min:x_max])
img[y_min:y_max,
x_min:x_max] = cv.bitwise_and(img[y_min:y_max, x_min:x_max],
img[y_min:y_max, x_min:x_max],
mask=mask_to_use_inv[y_min:y_max,
x_min:x_max])
mask_to_use_inv_inv = cv.bitwise_not(mask_to_use_inv)
warped_img[y_min:y_max, x_min:x_max] = cv.bitwise_and(
warped_img[y_min:y_max, x_min:x_max],
warped_img[y_min:y_max, x_min:x_max],
mask=mask_to_use_inv_inv[y_min:y_max, x_min:x_max])
img[y_min:y_max,
x_min:x_max] = cv.add(img[y_min:y_max, x_min:x_max],
warped_img[y_min:y_max, x_min:x_max])
return img
def draw_cube(self, img, imgpts):
imgpts = np.int32(imgpts).reshape(-1, 2)
# draw back left wall
back_pts_2 = np.empty((0, 2), dtype=int)
back_pts_2 = np.append(back_pts_2, [imgpts[0]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[4]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[5]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[1]], axis=0)
img = cv.drawContours(img, [back_pts_2], -1, (100, 100, 255), 10)
# draw back right wall
back_pts_2 = np.empty((0, 2), dtype=int)
back_pts_2 = np.append(back_pts_2, [imgpts[0]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[4]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[7]], axis=0)
back_pts_2 = np.append(back_pts_2, [imgpts[3]], axis=0)
img = cv.drawContours(img, [back_pts_2], -1, (255, 100, 100), -1)
# draw pillars in blue color
for i, j in zip(range(0, 4), range(4, 8)):
img = cv.line(img, tuple(imgpts[i]), tuple(imgpts[j]), (255, 0, 0),
3)
# draw top layer in red color
img = cv.drawContours(img, [imgpts[4:]], -1, (0, 0, 255), 3)
return img