def get_contour(rescaled_image):
MORPH = 9
CANNY = 84
HOUGH = 25
IM_HEIGHT, IM_WIDTH, _ = rescaled_image.shape
gray = cv2.cvtColor(rescaled_image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (MORPH, MORPH))
dilated = cv2.dilate(gray, kernel)
edged = cv2.Canny(dilated, 0, CANNY)
test_corners = get_corners(edged)
approx_contours = []
if len(test_corners) >= 4:
quads = []
for quad in itertools.combinations(test_corners, 4):
points = np.array(quad)
points = transform.order_points(points)
points = np.array([[p] for p in points], dtype="int32")
quads.append(points)
quads = sorted(quads, key=cv2.contourArea, reverse=True)[:5]
quads = sorted(quads, key=angle_range)
approx = quads[0]
if is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
(_, cnts, hierarchy) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:5]
for c in cnts:
approx = cv2.approxPolyDP(c, 80, True)
if is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
break
if not approx_contours:
TOP_RIGHT = (IM_WIDTH, 0)
BOTTOM_RIGHT = (IM_WIDTH, IM_HEIGHT)
BOTTOM_LEFT = (0, IM_HEIGHT)
TOP_LEFT = (0, 0)
screenCnt = np.array([[TOP_RIGHT], [BOTTOM_RIGHT], [BOTTOM_LEFT],
[TOP_LEFT]])
else:
screenCnt = max(approx_contours, key=cv2.contourArea)
return screenCnt.reshape(4, 2)
def run_on_image(self, image):
detect_passport_predictions = self.predictor(image)
mask = detect_passport_predictions["instances"].pred_masks.numpy()[0]
mask = np.array(mask, dtype=np.uint8) * 255
_, contours, _ = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
rect = cv2.minAreaRect(contours[0])
box = cv2.boxPoints(rect)
box = np.int0(box)
passport = four_point_transform(image, order_points(box))
return passport
def get_image(im, id):
#resize image to fit screen
r = 1000.0 / im.shape[1]
dim = (1000, int(im.shape[0] * r))
resized = cv2.resize(im, dim, interpolation=cv2.INTER_AREA)
#convert image to grayscale
grayscaled = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)
(x, y, dst_norm_scaled) = cornerHarris_demo(125, grayscaled)
xmin = int(round(x - (resized.shape[0] * .06)))
xmax = int(round(x + (resized.shape[0] * .06)))
ymin = int(round(y - (resized.shape[0] * .06)))
ymax = int(round(y + (resized.shape[0] * .06)))
cropped = resized[xmin:xmax, ymin:ymax]
gray_cropped = cv2.cvtColor(cropped, cv2.COLOR_BGR2GRAY)
pts = get_corners(gray_cropped, id)
ordered = transform.order_points(pts)
warped = transform.four_point_transform(gray_cropped, ordered)
return warped
def getChessContour(img):
blurred = cv2.GaussianBlur(img, (5, 5), 0.0)
unsharp = cv2.addWeighted(img, 3, blurred, -2, 0, img)
img = unsharp
M, ideal_grid, grid_next, grid_good, spts = findChessboard(img)
# View
if M is not None:
M, _ = generateNewBestFit((ideal_grid+8)*32, grid_next, grid_good) # generate mapping for warping image
img_warp = cv2.warpPerspective(img, M, (17*32, 17*32), flags=cv2.WARP_INVERSE_MAP)
best_lines_x, best_lines_y = getBestLines(img_warp)
min_x, max_x, min_y, max_y = best_lines_x[0], best_lines_x[-1], best_lines_y[0], best_lines_y[-1]
width, height = max_x - min_x, max_y - min_y
min_x -= width/6
max_x += width/6
min_y -= height/6
max_y += height/6
x,y = np.meshgrid([min_x, max_x], [min_y, max_y])
xy = np.vstack([x.flatten(), y.flatten()]).T.astype(np.float32)
xy = np.expand_dims(xy,0)
xy_unwarp = cv2.perspectiveTransform(xy, M)
pts = order_points(xy_unwarp[0,:,:])
return [np.array(pts, dtype=np.int32)]
T_marker = np.array([markerLength, markerLength, 0.0])
A = np.array([0.0, 0.0, 0.0]) + T_marker
B = np.array([0.4 + markerSeparation, 0.0, 0.0]) + T_marker
C = np.array([0.4 + markerSeparation, 0.4 + markerSeparation, 0.0
]) + T_marker
D = np.array([0.0, 0.4 + markerSeparation, 0.0]) + T_marker
### Find Transformatio Matrix ###
rotM = np.zeros(shape=(3, 3))
cv2.Rodrigues(rvec, rotM, jacobian=0)
### Map to image coordinate ###
pts, jac = cv2.projectPoints(
np.float32([A, B, C, D]).reshape(-1, 3), rvec, tvec,
cameraMatrix, dist)
pts = np.array([tuple(pts[i].ravel()) for i in range(4)],
dtype="float32")
pts = order_points(pts)
### Draw axis ###
for point in [A, B, C, D]:
cv2.aruco.drawAxis(image=frame,
cameraMatrix=cameraMatrix,
distCoeffs=dist,
rvec=rvec,
tvec=tvec + np.dot(point, rotM.T),
length=0.1)
### Draw work space ###
# drawBox(frame, rvec, tvec + np.dot(A, rotM.T), size=0.4 + markerSeparation)
## Fill Marker ##
for corner, id in zip(markerCorners, markerIds):
points = [(int(point[0]), int(point[1])) for point in corner[0]]
ids = id[0]
#Finding contours from main image
contours, hierarchy = cv2.findContours(binary_image_gray.copy(), cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
for c in contours:
peri = cv2.arcLength(c, True)
approx = cv2.approxPolyDP(c, 0.02 * peri, True)
area = cv2.contourArea(c)
#Selecting the upper rectangle
if ((len(approx) == 4) & (area > 1000000) & (area < 2000000)):
cv2.drawContours(img, [c], -1, (0, 255, 0), 3)
box = cv2.minAreaRect(c)
box = cv2.cv.BoxPoints(box)
box = np.array(box, dtype="int")
rect = transform.order_points(box)
x1, y1 = rect[0]
x2, y2 = rect[2]
hig = x2 - x1
wid = y2 - y1
#Getting transformation from image (avoid rotations)
warped = transform.four_point_transform(binary_image_gray, box)
warped2 = transform.four_point_transform(final, box)
warped3 = transform.four_point_transform(img, box)
arr = []
number = 0
#Finding countours again from the transformed image
contours2, hierarchy2 = cv2.findContours(warped2.copy(), cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
if cv.contourArea( cont ) < 20000 : continue # print(cv.contourArea( cont )) arc_len = cv.arcLength( cont, True ) approx = cv.approxPolyDP( cont, 0.1 * arc_len, True ) if ( len( approx ) != 4 ): continue IS_FOUND = True box = np.array([ approx[0][0], approx[1][0], approx[2][0], approx[3][0]], dtype = "float32") box = order_points(box) col2 = col row2 = col*RATIO des = np.float32([[0,0],[row2,0],[row2,col2],[0,col2]]) M = cv.getPerspectiveTransform(box, des) dst = cv.warpPerspective(img,M,(int(row2),col2)) # cv.drawContours( img, [approx], -1, ( 255, 0, 0 ), 2 ) for i in range(4): # cv.circle(img, (box[i][0], box[i][1]), 3, (0,255,0), 3) cv.putText(img, str(i), (box[i][0], box[i][1]), cv.FONT_HERSHEY_PLAIN, 2.0, (0, 0, 255), 2) # print(ymin, ymax, xmin, xmax)
def main():
# cap = cv2.VideoCapture(cv2.CAP_DSHOW)
# codec = 0x47504A4D # MJPG
# cap.set(cv2.CAP_PROP_FPS, 30.0)
# cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('m','j','p','g'))
# cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M','J','P','G'))
# cap.set(3, 1920)
# cap.set(4, 1080)
cap = cv2.VideoCapture("../K.mp4")
img_list = []
valid_list = []
bg_list = []
ret, img = cap.read()
frame_counter = 0
N = 1000
parameters = cv2.aruco.DetectorParameters_create()
# parameters(doCornerRefinement=True)
# markerLength=0.039 # real
# markerSeparation=0.0975 # real
markerLength = 0.04
markerSeparation = 0.01
# board = cv2.aruco.GridBoard_create(markersX=10, markersY=10, markerLength=0.039, markerSeparation=0.0975, dictionary=dictionary) # real
board = cv2.aruco.GridBoard_create(markersX=10,
markersY=10,
markerLength=markerLength,
markerSeparation=markerSeparation,
dictionary=dictionary)
backSub = cv2.createBackgroundSubtractorMOG2(history=30,
varThreshold=16,
detectShadows=False)
# backSub = cv2.createBackgroundSubtractorKNN(history=30, dist2Threshold=1000.0, detectShadows=True)
for i in range(N):
ret, frame = cap.read()
original = frame.copy()
markerCorners, markerIds, _ = cv2.aruco.detectMarkers(
frame, dictionary, parameters=parameters)
if markerIds is not None:
ret, _, _ = cv2.aruco.estimatePoseBoard(corners=markerCorners,
ids=markerIds,
board=board,
cameraMatrix=cameraMatrix,
distCoeffs=dist,
rvec=rvec,
tvec=tvec)
if ret:
# cv2.aruco.drawAxis(image=frame, cameraMatrix=cameraMatrix, distCoeffs=dist, rvec=rvec, tvec=tvec, length=0.1) # origin
T_marker = np.array([markerLength, markerLength, 0.0])
A = np.array([0.0, 0.0, 0.0]) + T_marker
B = np.array([0.4 + markerSeparation, 0.0, 0.0]) + T_marker
C = np.array([
0.4 + markerSeparation, 0.4 + markerSeparation, 0.0
]) + T_marker
D = np.array([0.0, 0.4 + markerSeparation, 0.0]) + T_marker
### Find Transformatio Matrix ###
rotM = np.zeros(shape=(3, 3))
cv2.Rodrigues(rvec, rotM, jacobian=0)
### Map to image coordinate ###
pts, jac = cv2.projectPoints(
np.float32([A, B, C, D]).reshape(-1, 3), rvec, tvec,
cameraMatrix, dist)
pts = np.array([tuple(pts[i].ravel()) for i in range(4)],
dtype="float32")
pts = order_points(pts)
### Draw axis ###
for point in [A, B, C, D]:
cv2.aruco.drawAxis(image=frame,
cameraMatrix=cameraMatrix,
distCoeffs=dist,
rvec=rvec,
tvec=tvec + np.dot(point, rotM.T),
length=0.1)
### Draw work space ###
# drawBox(frame, rvec, tvec + np.dot(A, rotM.T), size=0.4 + markerSeparation)
## Fill Marker ##
for corner, id in zip(markerCorners, markerIds):
points = [(int(point[0]), int(point[1]))
for point in corner[0]]
ids = id[0]
pts1 = np.array(points, np.int32)
cv2.fillPoly(frame, [pts1], 255)
## Perspective Crop ##
warped = four_point_transform(original, pts)
warped = cv2.resize(warped, (800, 800))
valid_mask = four_point_transform(
np.ones(original.shape[:2], dtype="uint8") * 255, pts)
valid_mask = cv2.resize(valid_mask, (800, 800))
cv2.imshow("Warped", warped)
cv2.imshow("Valid", valid_mask)
cv2.waitKey(1)
img_list.append(warped)
valid_list.append(valid_mask)
frame_counter = 0
for i in range(N):
index = random.randint(0, len(img_list) - 1)
warped = img_list[index]
valid_mask = valid_list[index]
frame_counter += 1
if frame_counter > 500 and random.randint(0, N - frame_counter) < 200:
warped_final = bg_list[random.randint(0, len(bg_list) - 1)]
else:
if frame_counter < 200:
mean_canvas = np.ones(warped.shape, dtype="uint8") * 200
mean_canvas = cv2.bitwise_or(mean_canvas,
mean_canvas,
mask=cv2.bitwise_not(valid_mask))
else:
mean_canvas = cv2.bitwise_or(bg,
bg,
mask=cv2.bitwise_not(valid_mask))
if frame_counter % 8 == 0: bg_list.append(bg)
valid_mask = cv2.bitwise_or(warped, warped, mask=valid_mask)
warped_final = cv2.bitwise_or(mean_canvas, valid_mask)
cv2.imshow('Passed', warped_final)
fgMask = backSub.apply(cv2.GaussianBlur(warped_final, (5, 5), 0))
cv2.imshow('FG Mask', fgMask)
bg = backSub.getBackgroundImage()
cv2.imshow('BG', bg)
# cv2.imshow("Preview", frame)
# cv2.imshow("marker33", markerImage)
key = cv2.waitKey(1)
if key == 27:
break
if key == ord('m'):
mode = not mode
if key == ord(' '):
cv2.imwrite(str(i) + ".jpg", warped)
i += 1
# cv2.imwrite("Real5.png", bg)
cv2.waitKey(0)
cap.release()
cv2.destroyAllWindows()
return bg
def scan():
if not request.stream:
return "", 400
head = request.stream.read(1024)
mime_detector = magic.Magic(mime=True)
mime = mime_detector.from_buffer(head)
if not mime.startswith("image/"):
return "", 400
buf = head + request.stream.read()
nparr = np.frombuffer(buf, np.uint8)
image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
ratio = image.shape[0] / 500.0
orig = image.copy()
image = imutils.resize(image, height=500)
IM_HEIGHT, IM_WIDTH, _ = image.shape
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9))
dilated = cv2.dilate(gray, kernel)
edged = cv2.Canny(dilated, 0, 84)
lsd = cv2.createLineSegmentDetector()
lines = lsd.detect(edged)[0]
corners = []
if lines is not None:
lines = lines.squeeze().astype(np.int32).tolist()
horizontal_lines_canvas = np.zeros(edged.shape, dtype=np.uint8)
vertical_lines_canvas = np.zeros(edged.shape, dtype=np.uint8)
for line in lines:
x1, y1, x2, y2 = line
if abs(x2 - x1) > abs(y2 - y1):
(x1, y1), (x2, y2) = sorted(((x1, y1), (x2, y2)),
key=lambda pt: pt[0])
cv2.line(horizontal_lines_canvas, (max(x1 - 5, 0), y1),
(min(x2 + 5, edged.shape[1] - 1), y2), 255, 2)
else:
(x1, y1), (x2, y2) = sorted(((x1, y1), (x2, y2)),
key=lambda pt: pt[1])
cv2.line(vertical_lines_canvas, (x1, max(y1 - 5, 0)),
(x2, min(y2 + 5, edged.shape[0] - 1)), 255, 2)
lines = []
contours = cv2.findContours(horizontal_lines_canvas, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)[1]
contours = sorted(contours,
key=lambda c: cv2.arcLength(c, True),
reverse=True)[:2]
horizontal_lines_canvas = np.zeros(edged.shape, dtype=np.uint8)
for contour in contours:
contour = contour.reshape((contour.shape[0], contour.shape[2]))
min_x = np.amin(contour[:, 0], axis=0) + 2
max_x = np.amax(contour[:, 0], axis=0) - 2
left_y = int(np.average(contour[contour[:, 0] == min_x][:, 1]))
right_y = int(np.average(contour[contour[:, 0] == max_x][:, 1]))
lines.append((min_x, left_y, max_x, right_y))
cv2.line(horizontal_lines_canvas, (min_x, left_y),
(max_x, right_y), 1, 1)
corners.append((min_x, left_y))
corners.append((max_x, right_y))
contours = cv2.findContours(vertical_lines_canvas, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)[1]
contours = sorted(contours,
key=lambda c: cv2.arcLength(c, True),
reverse=True)[:2]
vertical_lines_canvas = np.zeros(edged.shape, dtype=np.uint8)
for contour in contours:
contour = contour.reshape((contour.shape[0], contour.shape[2]))
min_y = np.amin(contour[:, 1], axis=0) + 2
max_y = np.amax(contour[:, 1], axis=0) - 2
top_x = int(np.average(contour[contour[:, 1] == min_y][:, 0]))
bottom_x = int(np.average(contour[contour[:, 1] == max_y][:, 0]))
lines.append((top_x, min_y, bottom_x, max_y))
cv2.line(vertical_lines_canvas, (top_x, min_y), (bottom_x, max_y),
1, 1)
corners.append((top_x, min_y))
corners.append((bottom_x, max_y))
corners_y, corners_x = np.where(horizontal_lines_canvas +
vertical_lines_canvas == 2)
corners += zip(corners_x, corners_y)
test_corners = []
for c in corners:
if predicate(test_corners, c):
test_corners.append(c)
approx_contours = []
if len(test_corners) >= 4:
quads = []
for quad in itertools.combinations(test_corners, 4):
points = np.array(quad)
points = transform.order_points(points)
points = np.array([[p] for p in points], dtype="int32")
quads.append(points)
quads = sorted(quads, key=cv2.contourArea, reverse=True)[:5]
quads = sorted(quads, key=angle_range)
approx = quads[0]
if is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
(_, cnts, hierarchy) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:5]
for c in cnts:
approx = cv2.approxPolyDP(c, 80, True)
if is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
break
screenCnt = None
if not approx_contours:
TOP_RIGHT = (IM_WIDTH, 0)
BOTTOM_RIGHT = (IM_WIDTH, IM_HEIGHT)
BOTTOM_LEFT = (0, IM_HEIGHT)
TOP_LEFT = (0, 0)
screenCnt = np.array([[TOP_RIGHT], [BOTTOM_RIGHT], [BOTTOM_LEFT],
[TOP_LEFT]])
else:
screenCnt = max(approx_contours, key=cv2.contourArea)
screenCnt = screenCnt.reshape(4, 2)
warped = transform.four_point_transform(orig, screenCnt * ratio)
gray = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
sharpen = cv2.GaussianBlur(gray, (0, 0), 3)
sharpen = cv2.addWeighted(gray, 1.5, sharpen, -0.5, 0)
thresh = cv2.adaptiveThreshold(sharpen, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY, 21, 15)
scan = cv2.imencode(mimetypes.guess_extension(mime), thresh)[1]
return send_file(io.BytesIO(scan),
mimetype=mime,
as_attachment=True,
attachment_filename="Scan")
def get_contour(self, rescaled_image):
"""
Returns a numpy array of shape (4, 2) containing the vertices of the four corners
of the document in the image. It considers the corners returned from get_corners()
and uses heuristics to choose the four corners that most likely represent
the corners of the document. If no corners were found, or the four corners represent
a quadrilateral that is too small or convex, it returns the original four corners.
"""
# these constants are carefully chosen
MORPH = 9
CANNY = 84
HOUGH = 25
IM_HEIGHT, IM_WIDTH, _ = rescaled_image.shape
# convert the image to grayscale and blur it slightly
gray = cv2.cvtColor(rescaled_image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7,7), 0)
# dilate helps to remove potential holes between edge segments
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(MORPH,MORPH))
dilated = cv2.morphologyEx(gray, cv2.MORPH_CLOSE, kernel)
# find edges and mark them in the output map using the Canny algorithm
edged = cv2.Canny(dilated, 0, CANNY)
test_corners = self.get_corners(edged)
approx_contours = []
if len(test_corners) >= 4:
quads = []
for quad in itertools.combinations(test_corners, 4):
points = np.array(quad)
points = transform.order_points(points)
points = np.array([[p] for p in points], dtype = "int32")
quads.append(points)
# get top five quadrilaterals by area
quads = sorted(quads, key=cv2.contourArea, reverse=True)[:5]
# sort candidate quadrilaterals by their angle range, which helps remove outliers
quads = sorted(quads, key=self.angle_range)
approx = quads[0]
if self.is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
# for debugging: uncomment the code below to draw the corners and countour found
# by get_corners() and overlay it on the image
# cv2.drawContours(rescaled_image, [approx], -1, (20, 20, 255), 2)
# plt.scatter(*zip(*test_corners))
# plt.imshow(rescaled_image)
# plt.show()
# also attempt to find contours directly from the edged image, which occasionally
# produces better results
(cnts, hierarchy) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:5]
# loop over the contours
for c in cnts:
# approximate the contour
approx = cv2.approxPolyDP(c, 80, True)
if self.is_valid_contour(approx, IM_WIDTH, IM_HEIGHT):
approx_contours.append(approx)
break
# If we did not find any valid contours, just use the whole image
if not approx_contours:
TOP_RIGHT = (IM_WIDTH, 0)
BOTTOM_RIGHT = (IM_WIDTH, IM_HEIGHT)
BOTTOM_LEFT = (0, IM_HEIGHT)
TOP_LEFT = (0, 0)
screenCnt = np.array([[TOP_RIGHT], [BOTTOM_RIGHT], [BOTTOM_LEFT], [TOP_LEFT]])
else:
screenCnt = max(approx_contours, key=cv2.contourArea)
return screenCnt.reshape(4, 2)
