def fine_tune_transform(feature1, feature2, init_pair_idx):
ind = []
k = 1
while len(ind) < 0.6 * min(len(feature1["pts"]), len(feature2["pts"])) and k < 10:
# Step 1. Randomly choose 20 points evenly distributed on the image
rand_pts = np.random.rand(20, 2) * (np.amax(feature1["pts"], axis=0) - np.amin(feature1["pts"], axis=0)) * \
np.array([1, 0.8]) + np.amin(feature1["pts"], axis=0)
# Step 2. Find nearest points from feature1
dist_mat = spd.cdist(rand_pts, feature1["pts"][init_pair_idx[:, 0]])
tmp_ind = np.argmin(dist_mat, axis=1)
# Step 3. Use these points to find a homography
tf = cv2.findHomography(feature1["pts"][init_pair_idx[tmp_ind, 0]], feature2["pts"][init_pair_idx[tmp_ind, 1]],
method=cv2.RANSAC, ransacReprojThreshold=5)
# Then use the transform find more matched points
pts12 = cv2.perspectiveTransform(np.array([[p] for p in feature1["pts"]], dtype="float32"), tf[0])[:, 0, :]
dist_mat = spd.cdist(pts12, feature2["pts"])
num1, num2 = dist_mat.shape
idx12 = np.argsort(dist_mat, axis=1)
tmp_ind = np.argwhere(np.array([dist_mat[i, idx12[i, 0]] for i in range(num1)]) < 5)
if len(tmp_ind) > len(ind):
ind = tmp_ind
logging.debug("len(ind) = %d, len(feature) = %d", len(ind), min(len(feature1["pts"]), len(feature2["pts"])))
k += 1
pair_idx = np.hstack((ind, idx12[ind, 0]))
tf = cv2.findHomography(feature1["pts"][pair_idx[:, 0]], feature2["pts"][pair_idx[:, 1]],
method=cv2.RANSAC, ransacReprojThreshold=5)
return tf, pair_idx
def locate(self, visible_markers):
"""
- find overlapping set of surface markers and visible_markers
- compute homography (and inverse) based on this subset
"""
if not self.defined:
self.build_correspondance(visible_markers)
else:
marker_by_id = dict([(m['id'],m) for m in visible_markers])
visible_ids = set(marker_by_id.keys())
requested_ids = set(self.markers.keys())
overlap = visible_ids & requested_ids
self.detected_markers = len(overlap)
if len(overlap)>=min(2,len(requested_ids)):
self.detected = True
yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] )
uv = np.array( [self.markers[i].uv_coords for i in overlap] )
yx.shape=(-1,1,2)
uv.shape=(-1,1,2)
# print 'uv',uv
# print 'yx',yx
self.m_to_screen,mask = cv2.findHomography(uv,yx)
self.m_from_screen,mask = cv2.findHomography(yx,uv)
else:
self.detected = False
self.m_from_screen = None
self.m_to_screen = None
def answer_caching_request(self, marker_cache, frame_index):
visible_markers = marker_cache[frame_index]
# cache point had not been visited
if visible_markers == False:
return False
# cache point had been visited
marker_by_id = dict([(m["id"], m) for m in visible_markers])
visible_ids = set(marker_by_id.keys())
requested_ids = set(self.markers.keys())
overlap = visible_ids & requested_ids
detected_markers = len(overlap)
if len(overlap) >= min(2, len(requested_ids)):
yx = np.array([marker_by_id[i]["verts_norm"] for i in overlap])
uv = np.array([self.markers[i].uv_coords for i in overlap])
yx.shape = (-1, 1, 2)
uv.shape = (-1, 1, 2)
m_to_screen, mask = cv2.findHomography(uv, yx)
m_from_screen, mask = cv2.findHomography(yx, uv)
return {
"m_to_screen": m_to_screen,
"m_from_screen": m_from_screen,
"detected_markers": len(overlap),
"gaze_on_srf": self.gaze_on_srf_by_frame_idx(frame_index, m_from_screen),
}
else:
# surface not found
return None
def draw_matches(window_name, kp_pairs1, kp_pairs2, electricImg, acousticImg,
mainImg, electricColorImg, acousticColorImg, mainColorImg):
mkp1_1, mkp2_1 = zip(*kp_pairs1)
p1_1 = np.float32([kp.pt for kp in mkp1_1])
p2_1 = np.float32([kp.pt for kp in mkp2_1])
mkp1_2, mkp2_2 = zip(*kp_pairs2)
p1_2 = np.float32([kp.pt for kp in mkp1_2])
p2_2 = np.float32([kp.pt for kp in mkp2_2])
if len(kp_pairs1) >= 5:
H1, status1 = cv2.findHomography(p1_1, p2_1, cv2.RANSAC, 5.0)
else:
H1, status1 = None, None
if len(kp_pairs2) >= 5:
H2, status2 = cv2.findHomography(p1_2, p2_2, cv2.RANSAC, 5.0)
else:
H2, status2 = None, None
if len(p1_1):
explore_match(window_name, kp_pairs1, kp_pairs2, electricImg,
acousticImg, mainImg, electricColorImg, acousticColorImg,
mainColorImg, status1, H1, status2, H2)
def point_corresp_mod(pointcorr_name,current_elevation,homo_filename,elevdiff,gthomo_filename,shift_gt_homo):
elevprop = []
for ce in current_elevation:
elevprop.append(float(ce)/1.5)
pct = open(pointcorr_name,'r')
fullextract = pct.readlines()
pclines = fullextract[-6::]
video_lines = []
worldPts = []
temp_holder = []
#Extract latest point correspondences
for j in range(0,2):
temp_holder.append(pclines[j].split())
#Extract world points
for k in range(0,4):
worldPts.append([float(temp_holder[0][k]),float(temp_holder[1][k])])
worldPts2 = np.float32(worldPts)
#Prepare video point arrays
for x in range(2,6):
video_lines.append(pclines[x].split())
for y in range(0,4):
video_lines[x-2][y] = video_lines[x-2][y].split('e+')
point_arrays = []
#each point:
# [[X0, Y0]
# [X1, Y1]]
for a in range (0,4):
point_arrays.append([[float(video_lines[0][a][0])*(10**float(video_lines[0][a][1])),
float(video_lines[1][a][0])*(10**float(video_lines[1][a][1]))],
[float(video_lines[2][a][0])*(10**float(video_lines[2][a][1])),
float(video_lines[3][a][0])*(10**float(video_lines[3][a][1]))]])
curr_videoPts = []
curr_vidlower = []
for i in range (0,4):
delta_x = point_arrays[i][1][0] - point_arrays[i][0][0]
delta_y = point_arrays[i][1][1] - point_arrays[i][0][1]
a = math.sqrt((float(elevdiff)**2) / (1 + ((delta_x**2)/(delta_y**2))))
b = a * (delta_x**2)/(delta_y**2)
curr_videoPts.append([point_arrays[i][0][0] + (delta_x * elevprop[i]),point_arrays[i][0][1] + (delta_y * elevprop[i])])
curr_vidlower.append([point_arrays[i][0][0] + ((1-b)*delta_x * elevprop[i]),point_arrays[i][0][1] + ((1-a) * delta_y * elevprop[i])])
curr_videoPts2 = np.float32(curr_videoPts)
currlower = np.float32(curr_vidlower)
homography, mask = cv2.findHomography(np.array(curr_videoPts2), np.array(worldPts2))
homography2, fail = cv2.findHomography(np.array(currlower), np.array(worldPts2))
np.savetxt(homo_filename,homography)
if shift_gt_homo == 1:
np.savetxt(gthomo_filename,homography2)
else:
np.savetxt(gthomo_filename,homography)
def get_transform_matrix(v0, v1):
src = []
des = []
for i in range(len(v0[0])):
src.append([v0[0][i], v0[1][i]])
des.append([v1[0][i], v1[1][i]])
src = numpy.float32(src)
des = numpy.float32(des)
return cv2.findHomography(src, des)[0], cv2.findHomography(des, src)[0]
def multiRansac(oKeys, nKeys): h = 0 length, _, _ = oKeys.shape while h < 1000: rndIdx = np.random.choice(length, 4) k1, k2 = oKeys[rndIdx], nKeys[rndIdx] h1 = cv2.findHomography(k1,k2) h2 = cv2.findHomography(k2,k1) import ipdb; ipdb.set_trace()
def _find_homographies(points_A, points_B):
points_A = points_A.reshape((-1, 1, 2))
points_B = points_B.reshape((-1, 1, 2))
B_to_A, mask = cv2.findHomography(
points_A, points_B, method=cv2.RANSAC, ransacReprojThreshold=100
)
A_to_B, mask = cv2.findHomography(points_B, points_A)
return A_to_B, B_to_A
def remove_homography_from_points(kp0,kp1,H=None,method=cv2.RANSAC,thresh=1): # Before: 1
if H is None:
H,inliers = cv2.findHomography(kp0,kp1,method=method,ransacReprojThreshold=thresh)
if H is None:
# Try again, with the first match removed
H,inliers = cv2.findHomography(kp0[1:],kp1[1:],method=method,ransacReprojThreshold=thresh)
inliers = np.r_[[[0]],inliers]
else:
inliers = np.ones(kp0.shape[0])
Hinv = np.linalg.inv(H)
kp1_ = cv2.perspectiveTransform(kp1[np.newaxis,:,:],Hinv)
return kp0,kp1_[0,:,:],H,Hinv,inliers
def compute_homography(kp1, kp2, outlier_algorithm=cv2.RANSAC, reproj_threshold=5.0):
"""
Given two arrays of keypoints compute a homography
Parameters
----------
kp1 : ndarray
(n, 2) of coordinates from the source image
kp2 : ndarray
(n, 2) of coordinates from the destination image
outlier_algorithm : object
The openCV algorithm to use for outlier detection
reproj_threshold : float
The RANSAC reprojection threshold
Returns
-------
transformation_matrix : ndarray
The 3x3 transformation matrix
mask : ndarray
Boolean array of the outliers
"""
transformation_matrix, mask = cv2.findHomography(kp1,
kp2,
outlier_algorithm,
reproj_threshold)
mask = mask.astype(bool)
return transformation_matrix, mask
def track(self, frame):
'''Returns a list of detected TrackedTarget objects'''
self.frame_points, frame_descrs = self.detect_features(frame)
if len(self.frame_points) < MIN_MATCH_COUNT:
return []
matches = self.matcher.knnMatch(frame_descrs, k = 2)
matches = [m[0] for m in matches if len(m) == 2 and m[0].distance < m[1].distance * 0.75]
if len(matches) < MIN_MATCH_COUNT:
return []
matches_by_id = [[] for _ in xrange(len(self.targets))]
for m in matches:
matches_by_id[m.imgIdx].append(m)
tracked = []
for imgIdx, matches in enumerate(matches_by_id):
if len(matches) < MIN_MATCH_COUNT:
continue
target = self.targets[imgIdx]
p0 = [target.keypoints[m.trainIdx].pt for m in matches]
p1 = [self.frame_points[m.queryIdx].pt for m in matches]
p0, p1 = np.float32((p0, p1))
H, status = cv2.findHomography(p0, p1, cv2.RANSAC, 3.0)
status = status.ravel() != 0
if status.sum() < MIN_MATCH_COUNT:
continue
p0, p1 = p0[status], p1[status]
x0, y0, x1, y1 = target.rect
quad = np.float32([[x0, y0], [x1, y0], [x1, y1], [x0, y1]])
quad = cv2.perspectiveTransform(quad.reshape(1, -1, 2), H).reshape(-1, 2)
track = TrackedTarget(target=target, p0=p0, p1=p1, H=H, quad=quad)
tracked.append(track)
tracked.sort(key = lambda t: len(t.p0), reverse=True)
return tracked
def matching(kp1,des1,img1,kp2_ext,des2_ext):
global MIN_MATCH_COUNT
kp2 = kp2_ext
des2 = des2_ext
if len(kp1) <= MIN_MATCH_COUNT*2 or len(kp2) <= MIN_MATCH_COUNT*2:
return [], None
#FLANN_INDEX_KDTREE = 0
#index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
#search_params = dict(checks = 50)
#flann = cv2.FlannBasedMatcher(index_params, search_params)
#matches = flann.knnMatch(des1,des2,k=2)
global FEATURE_DETECTOR
global CROSS_CHECK
if FEATURE_DETECTOR == "AKAZE":
if CROSS_CHECK:
bf = cv2.BFMatcher(cv2.NORM_HAMMING,crossCheck=True)
good = bf.match(des1,des2)
else:
bf = cv2.BFMatcher(cv2.NORM_HAMMING)
matches = bf.knnMatch(des1, des2, k=2)
good = []
for m,n in matches:
if m.distance < 0.8*n.distance:
good.append(m)
else:
if CROSS_CHECK:
bf = cv2.BFMatcher(crossCheck=True)
good = bf.match(des1,des2)
else:
bf = cv2.BFMatcher()
matches = bf.knnMatch(des1, des2, k=2)
good = []
for m,n in matches:
if m.distance < 0.8*n.distance:
good.append(m)
if len(good)>MIN_MATCH_COUNT:
src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
if mask is None:
return [], None
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)
return good, np.int32(dst)
return [], None
def process(self):
img1 = cv2.cvtColor(self.imgcv1, cv2.COLOR_BGR2GRAY) #queryimage # left image
img2 = cv2.cvtColor(self.imgcv2, cv2.COLOR_BGR2GRAY) #trainimage # right image
# find the keypoints and descriptors with SIFT
kp1, des1 = self.detector.detectAndCompute(img1,None)
kp2, des2 = self.detector.detectAndCompute(img2,None)
matches = self.flann.knnMatch(des1,des2,k=2)
pts1 = []
pts2 = []
# ratio test as per Lowe's paper
for i,(m,n) in enumerate(matches):
if m.distance < 0.8*n.distance:
pts2.append(kp2[m.trainIdx].pt)
pts1.append(kp1[m.queryIdx].pt)
pts1 = np.float32(pts1)
pts2 = np.float32(pts2)
M, mask = cv2.findHomography(pts1, pts2, cv2.RANSAC,5.0)
h,w = img1.shape
h/=2
w/=2
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)
cv2.polylines(img2,[np.int32(dst)],True,255,3)
#We select only inlier points
pts1 = pts1[mask.ravel()==1]
pts2 = pts2[mask.ravel()==1]
self.data=(M,pts1,pts2)
pts1i=np.int32(pts1)
pts2i=np.int32(pts2)
return drawpoints(img1,img2,pts1i,pts2i)
def matchKeypoints(self, kpsA, kpsB, featuresA, featuresB,
ratio, reprojThresh):
# compute the raw matches and initialize the list of actual
# matches
matcher = cv2.DescriptorMatcher_create("BruteForce")
rawMatches = matcher.knnMatch(featuresA, featuresB, 2)
matches = []
# loop over the raw matches
for m in rawMatches:
# ensure the distance is within a certain ratio of each
# other (i.e. Lowe's ratio test)
if len(m) == 2 and m[0].distance < m[1].distance * ratio:
matches.append((m[0].trainIdx, m[0].queryIdx))
# computing a homography requires at least 4 matches
if len(matches) > 4:
# construct the two sets of points
ptsA = np.float32([kpsA[i] for (_, i) in matches])
ptsB = np.float32([kpsB[i] for (i, _) in matches])
# compute the homography between the two sets of points
(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC,
reprojThresh)
# return the matches along with the homograpy matrix
# and status of each matched point
return (matches, H, status)
# otherwise, no homograpy could be computed
return None
def register_image(file_path, reference_form_path, output_path, result_writer, config_file):
reference = cv2.imread(reference_form_path, 0)
logging.info("read reference %s", reference_form_path)
orb = cv2.SIFT()
kp2, des2 = orb.detectAndCompute(reference, None)
image = cv2.imread(file_path, 0)
logging.info("read uploaded image %s", file_path)
kp1, des1 = orb.detectAndCompute(image, None)
logging.info("detected orb")
bf = cv2.BFMatcher(cv2.NORM_L2)
raw_matches = bf.knnMatch(des1, trainDescriptors=des2, k=2)
logging.info("knn matched")
matches = filter_matches(kp1, kp2, raw_matches)
mkp1, mkp2 = zip(*matches)
p1 = np.float32([kp.pt for kp in mkp1])
p2 = np.float32([kp.pt for kp in mkp2])
logging.info("starting RANSAC")
homography_transform, mask = cv2.findHomography(p1, p2, cv2.RANSAC, 5.0)
logging.info("RANSAC finished")
homography, transform = check_homography(homography_transform)
good_enough_match = check_match(homography, transform)
h, w = reference.shape
image_transformed = cv2.warpPerspective(image, homography_transform, (w, h))
logging.info("transformed image")
head, file_name = os.path.split(file_path)
transformed_image = write_transformed_image(image_transformed, homography, transform, good_enough_match, file_name,
output_path, result_writer)
logging.info("transformed %s", transformed_image)
return create_response(transformed_image, good_enough_match, config_file)
def homography():
dstPoints = np.asarray([[0.55,-0.5],[0.8,-0.5],[0.8,0],[0.55,0]],np.float32)
srcPoints1 = np.asarray([[482,148,1]],np.float32).T
srcPoints = np.asarray([[482,148],[406,133],[312,190],[399,222]],np.float32)
H = cv2.findHomography(srcPoints,dstPoints,0)[0]
return H
def two_view_reconstruction_plane_based(p1, p2, camera1, camera2, threshold):
"""Reconstruct two views from point correspondences lying on a plane.
Args:
p1, p2: lists points in the images
camera1, camera2: Camera models
threshold: reprojection error threshold
Returns:
rotation, translation and inlier list
"""
b1 = camera1.pixel_bearing_many(p1)
b2 = camera2.pixel_bearing_many(p2)
x1 = multiview.euclidean(b1)
x2 = multiview.euclidean(b2)
H, inliers = cv2.findHomography(x1, x2, cv2.RANSAC, threshold)
motions = multiview.motion_from_plane_homography(H)
motion_inliers = []
for R, t, n, d in motions:
inliers = _two_view_reconstruction_inliers(
b1, b2, R.T, -R.T.dot(t), threshold)
motion_inliers.append(inliers)
best = np.argmax(map(len, motion_inliers))
R, t, n, d = motions[best]
inliers = motion_inliers[best]
return cv2.Rodrigues(R)[0].ravel(), t, inliers
def _warp_keypoints(self, good_matches, key_frame, sh_frame):
"""Projects keypoints to the frontal plane
This method computes the homography matrix that is required to
project a list of keypoints to the frontal plane.
:param good_matches: list of good matches
:param key_frame: list of keypoints in the input (query) image
:param sh_frame: shape of the input (query) image
:returns: [Hinv, dst_size] homography matrix and size of resulting
image
"""
# bring object to frontoparallel plane: centered, up-right
dst_size = (sh_frame[1], sh_frame[0]) # cols,rows
scale_row = 1./self.sh_train[0]*dst_size[1]/2.
bias_row = dst_size[0]/4.
scale_col = 1./self.sh_train[1]*dst_size[0]*3/4.
bias_col = dst_size[1]/8.
# source points are the ones in the train image
src_points = [key_frame[good_matches[i].trainIdx].pt
for i in xrange(len(good_matches))]
# destination points are the ones in the query image
# off-set in space so that the image is centered
dst_points = [self.key_train[good_matches[i].queryIdx].pt
for i in xrange(len(good_matches))]
dst_points = [[x*scale_row+bias_row, y*scale_col+bias_col]
for x, y in dst_points]
# find homography
Hinv, _ = cv2.findHomography(np.array(src_points),
np.array(dst_points), cv2.RANSAC)
return [Hinv, dst_size]
def homographyFilter2(kp1, kp2, match, v=False):
xkp1 = [k.pt for k in kp1]
xkp2 = [k.pt for k in kp2]
if len(match) > 10:
query_pts = np.float32([xkp1[m[0]] for m in match]).reshape(-1, 1, 2)
train_pts = np.float32([xkp2[m[1]] for m in match]).reshape(-1, 1, 2)
M, mask = cv2.findHomography(query_pts, train_pts, cv2.RANSAC, 5.0)
if M is None:
if v == True:
print(' [o] Failed to find homography')
return []
matchesMask = mask.ravel().tolist()
if v == True:
m_len = len(match)
n_len = np.sum(matchesMask)
perc = n_len / m_len * 100.0
print(' [o] {} of {} matches kept ({:.2f}%)'.format(n_len, m_len, perc))
return matchesMask
else:
if v == True:
print(' [o] Not enough matches')
return []
def pano(image1,image2):
## Detect features and compute descriptors.
(keypoints1, descriptors1) = extract_features(image1, algorithm='SURF')
(keypoints2, descriptors2) = extract_features(image2, algorithm='SURF')
print len(keypoints1), "features detected in image1"
print len(keypoints2), "features detected in image2"
## Find corresponding features.
(points1, points2) = find_correspondences(keypoints1, descriptors1, keypoints2, descriptors2)
#print len(points1), "features matched"
## Visualise corresponding features.
correspondences = draw_correspondences(image1, image2, points1, points2)
cv2.imwrite("LR/correspondences.jpg", correspondences)
#cv2.imshow('correspondences',correspondences)
#print 'Wrote correspondences.jpg'
## Find homography between the views.
(homography, _) = cv2.findHomography(points2, points1)
## Calculate size and offset of merged panorama.
(size, offset) = calculate_size(image1.shape, image2.shape, homography)
## Finally combine images into a panorama.
panorama = merge_images(image1, image2, homography, size, offset, (points1, points2))
#print 'Wrote panorama.jpg'
#raw_input()
return panorama
def find_homography(image_1_kp, image_2_kp, matches):
image_1_points = np.zeros((len(matches), 1, 2), dtype=np.float32)
image_2_points = np.zeros((len(matches), 1, 2), dtype=np.float32)
for match_idx, match in enumerate(matches):
image_1_points[match_idx] = image_1_kp[match.queryIdx].pt
image_2_points[match_idx] = image_2_kp[match.trainIdx].pt
return cv2.findHomography(image_1_points, image_2_points, method=cv2.RANSAC, ransacReprojThreshold=5.0)[0]
def find_homography(kp1, des1, kp2, des2):
"""
Given a set of keypoints and descriptors finds the homography
"""
# Tenta fazer a melhor comparacao usando o algoritmo
matches = flann.knnMatch(des1, des2, k=2)
# store all the good matches as per Lowe's ratio test.
good = []
for m,n in matches:
if m.distance < 0.7*n.distance:
good.append(m)
if len(good)>MIN_MATCH_COUNT:
# Separa os bons matches na origem e no destino
src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
# Tenta achar uma trasformacao composta de rotacao, translacao e escala que situe uma imagem na outra
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
# Transforma os pontos da imagem origem para onde estao na imagem destino
dst = cv2.perspectiveTransform(pts,M)
return M
else:
# Caso em que nao houve matches o suficiente
return -1
def _detect_corner_points(self, key_frame, good_matches):
"""Detects corner points in an input (query) image
This method finds the homography matrix to go from the template
(train) image to the input (query) image, and finds the coordinates
of the good matches (from the train image) in the query image.
:param key_frame: keypoints of the query image
:param good_matches: list of good matches
:returns: coordinates of good matches in transformed query image
"""
# find homography using RANSAC
src_points = [self.key_train[good_matches[i].queryIdx].pt
for i in xrange(len(good_matches))]
dst_points = [key_frame[good_matches[i].trainIdx].pt
for i in xrange(len(good_matches))]
H, _ = cv2.findHomography(np.array(src_points), np.array(dst_points),
cv2.RANSAC)
# outline train image in query image
src_corners = np.array([(0, 0), (self.sh_train[1], 0),
(self.sh_train[1], self.sh_train[0]),
(0, self.sh_train[0])], dtype=np.float32)
dst_corners = cv2.perspectiveTransform(src_corners[None, :, :], H)
# convert to tuple
dst_corners = map(tuple, dst_corners[0])
return dst_corners
def find_homography(kp1, des1, kp2, des2):
"""
Given a set of keypoints and descriptors finds the homography
"""
# Tenta fazer a melhor comparacao usando o algoritmo
matches = flann.knnMatch(des1, des2, k=2)
# store all the good matches as per Lowe's ratio test.
good = []
for m,n in matches:
if m.distance < 0.7*n.distance:
good.append(m)
if len(good)>MIN_MATCH_COUNT:
src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)
return M
else:
return -1
def importExternalTextureFromImage(self, group, imageURL): projectedCorners = group['projectedCorners'] srcGroupMap = group['projectedPoints'] groupColors = group['colors'] groupColors.clear() image = cv2.imread(imageURL, cv2.CV_LOAD_IMAGE_COLOR) height = image.shape[0] width = image.shape[1] src = self.verticalReshape(projectedCorners) dst = self.verticalReshape([(0, 0), (width, 0), (width, height), (0, height)]) transformationMatrix, mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0) finalSrc = srcGroupMap.keys() finalDst = np.int32(cv2.perspectiveTransform(self.verticalReshape(finalSrc), transformationMatrix)) for i in range(len(finalDst)): dstCoord = tuple(finalDst[i].ravel()) if self.isOutOfFrame(dstCoord, width, height): continue srcCoord = finalSrc[i] for point in srcGroupMap[srcCoord]: groupColors[tuple(point)] = image[dstCoord[1]][dstCoord[0]] return
def matchKeypoints(kpsA, kpsB, featuresA, featuresB, ratio, repojThresh):
# compute the raw matches and intialize the list of actual matches
matcher = cv2.DescriptorMatcher_create("BruteForce")
rawMatches = matcher.knnMatch(featuresA, featuresB, 2)
# k=2 is used to indicate that the top two matches for each feature vector are returned
# Lowe's ratio test
# there can be false matches returned from the matching algorithm
# using lowe's ratio test we select only the matches within a certain threshold
matches = []
for m in rawMatches:
if len(m) == 2 and m[0].distance < m[1].distance * ratio:
matches.append((m[0].trainIdx, m[0].queryIdx))
# Step 3 - Use the RANSAC algorithm to estimate a homography matrix
# computing the homography matrix requires at least 4 matches
if len(matches) > 4:
# construct the two sets of points
ptsA = np.float32([kpsA[i] for (_, i) in matches])
ptsB = np.float32([kpsB[i] for (i, _) in matches])
# compute the homography between the two sets of points
(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, repojThresh)
# return the matches, homography matrix and status of each matched point
return (matches, H, status)
return None
def ransac():
threshold = 90 # seems somewhat reasonable here (not a small image)
best_sample = []
best_sample_params = []
best_sample_homography = []
for _ in range(iter_count):
cur_sample = random.sample(range(0, 16), 4)
cur_homography = cv2.findHomography(img_points[cur_sample], chessboard_points[cur_sample])[0]
converted_points = map(lambda x: from_homogeneous(np.dot(cur_homography, np.insert(x, 2, 1))), img_points)
error = chessboard_points - converted_points
filtered_points = filter(lambda (x, y): (x ** 2 + y ** 2) < threshold * threshold, error)
if len(filtered_points) > len(best_sample):
best_sample = np.copy(filtered_points)
best_sample_params = np.copy(cur_sample)
best_sample_homography = np.copy(cur_homography)
print('Best sample params: {}'.format(best_sample_params))
print('Best sample: \n{}\n'.format(best_sample))
dst = cv2.perspectiveTransform(np.array([img_points]), best_sample_homography)
proj_error = 0
for i in range(8):
proj_error += np.linalg.norm(dst[0][i] - chessboard_points[i])
proj_error /= 8
print('RANSAC error: {}\n'.format(proj_error)) # usually relatively small (<50) yet can be big
return best_sample_params, best_sample_homography
def filter_out_static(input_data, reprojection_threshold=2.0):
"""
:param input_data: input DataFrame of motion vectors of all frames.
:return: list of boolean, True when the input vector has moved
"""
frame_count = input_data['framenum'].max()
motion_all_mask = []
for frame_number in range(1, frame_count+1):
logging.info('loading frame {}'.format(frame_number))
#select motion vector for the current frame only
info_frame = input_data[input_data['framenum'] == frame_number]
if info_frame.empty: # happens for keyframes, just skip it
logging.info('no vector info for frame {}'.format(frame_number))
continue
# reformat source and destination coordinates
src_pts = np.float32(info_frame[['srcx', 'srcy']]).reshape(-1, 1, 2)
dst_pts = np.float32(info_frame[['dstx', 'dsty']]).reshape(-1, 1, 2)
# compute 2D transformation using RANSAC method
M, homography_mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, reprojection_threshold)
# object in motions are the ones that does not fit the homography
# a logical not should be enough (for first try)
motion_mask = np.logical_not(homography_mask).ravel().tolist()
# then accumulate the results
motion_all_mask += motion_mask
return motion_all_mask
def f2f(bg, curr, bgMask):
sift = cv2.SIFT(nOctaveLayers=3, contrastThreshold=0.05, edgeThreshold=10)
prev = bg.toImg()
N, M, _ = prev.shape
kp1, des1 = sift.detectAndCompute(prev, bgMask)
kp2, des2 = sift.detectAndCompute(curr, None)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)
good = []
for m, n in matches:
if m.distance < 0.75 * n.distance:
good.append(m)
if len(good) > 10:
src_pts = np.float32(
[kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
dst_pts = np.float32(
[kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
H, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 7.0)
print H
print len(kp1), len(kp2), len(good)
img4 = cv2.warpPerspective(curr, H, (M, N), flags=cv2.WARP_INVERSE_MAP)
mask4 = cv2.warpPerspective(np.zeros((720,1280), dtype=np.uint8), H, (M, N),
flags=cv2.WARP_INVERSE_MAP, borderValue=255)
mask4 = cv2.bitwise_not(mask4)
bg.add(img4, mask4)
cv2.bitwise_or(bgMask, mask4, dst=bgMask)
return H
def handleTree(self, group):
projectedCorners = group['projectedCorners']
srcGroupMap = group['projectedPoints']
groupColors = group['colors']
groupColors.clear()
image = cv2.imread("images/treeTexture.png", -1)
height = image.shape[0]
width = image.shape[1]
src = self.verticalReshape(projectedCorners)
dst = self.verticalReshape([(0, 0), (width, 0), (width, height), (0, height)])
transformationMatrix, mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0)
finalSrc = srcGroupMap.keys()
finalDst = np.int32(cv2.perspectiveTransform(self.verticalReshape(finalSrc),
transformationMatrix))
for i in range(len(finalDst)):
dstCoord = tuple(finalDst[i].ravel())
if self.isOutOfFrame(dstCoord, width, height):
continue
colorWithAlpha = image[dstCoord[1]][dstCoord[0]]
if(colorWithAlpha[3] < 1.0):
color = np.asarray([-1, -1, -1])
else:
color = np.asarray([colorWithAlpha[0],
colorWithAlpha[1],
colorWithAlpha[2]])
srcCoord = finalSrc[i]
for point in srcGroupMap[srcCoord]:
groupColors[tuple(point)] = color
return
def ransac(kpt_1, kpt_2, match):
data_1 = np.array([kp.pt for kp in kpt_1])[match[0], :]
data_2 = np.array([kp.pt for kp in kpt_2])[match[1], :]
_, matches = cv2.findHomography(data_1, data_2, cv2.RANSAC, 20.0)
return np.sum(matches)
# Press space to take screenshot
elif k % 256 == 32:
# SPACE pressed
img_name = "WindowName"
cv2.imshow(img_name, framefield)
print("{} written!".format(img_name))
img_counter += 1
cv2.setMouseCallback('WindowName', onMouse)
# Convert to numpy for other usages
posNp = np.array(posList)
points = np.array(posList)
# Calculate matrix H for coordinate projecting to 2D
htransf, status = cv2.findHomography(points, pts_dst)
# Reset webcams
clock.release()
field.release()
clockimages = cv2.VideoCapture(0)
hog = cv2.HOGDescriptor()
hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
# Open webcam video stream 0 for built-in webcam, 1 for external webcam
field = cv2.VideoCapture("/home/gustav/TIFX04/Arturs_kod/test2.mp4")
frame_width = int(field.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_height = int(field.get(cv2.CAP_PROP_FRAME_HEIGHT))
newWidth = 640.0
rat1 = newWidth / im_src.shape[1]
dim1 = (int(newWidth), int(im_src.shape[0] * rat1))
im_small = cv2.resize(im_src, dim1, interpolation=cv2.INTER_AREA)
# Four corners of the book in source image
pts_src = np.array([[57, 368], [98, 22], [585, 28], [626, 374]],
dtype=float)
# Read destination image.
im_dst = cv2.imread('destimg2.jpg')
# Four corners of the book in destination image.
pts_dst = np.array([[0, 0], [511, 0], [511, 639], [0, 639]], dtype=float)
# Calculate Homography
h, status = cv2.findHomography(pts_src, pts_dst)
# Warp source image to destination based on homography
im_out = cv2.warpPerspective(im_small, h,
(im_dst.shape[1], im_dst.shape[0]))
im_grey = cv2.cvtColor(im_out, cv2.COLOR_BGR2GRAY)
cv2.imwrite('img23.png', im_out)
# Match to template tiles
tileFiles = [
'tile000002.png', 'tile000004.png', 'tile000008.png', 'tile000016.png',
'tile000032.png', 'tile000064.png', 'tile000128.png', 'tile000256.png',
'tile000512.png', 'tile001024.png'
]
lineThicknessIdx = 1
def predict(self, frame, tracks):
"""
Predicts tracklet positions in the next frame and estimates camera motion.
Parameters
----------
frame : ndarray
The next frame.
tracks : List[Track]
List of tracks to predict.
Feature points of each track are updated in place.
Returns
-------
Dict[int, ndarray], ndarray
Returns a dictionary with track IDs as keys and predicted bounding
boxes of [x1, x2, y1, y2] as values, and a 3x3 homography matrix.
"""
# preprocess frame
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
frame_small = cv2.resize(frame_gray,
None,
fx=self.opt_flow_scale_factor[0],
fy=self.opt_flow_scale_factor[1])
# order tracks from closest to farthest
tracks.sort(reverse=True)
# detect target feature points
all_prev_pts = []
np.copyto(self.fg_mask, self.ones)
for track in tracks:
inside_tlbr = intersection(track.tlbr, self.frame_rect)
target_mask = crop(self.fg_mask, inside_tlbr)
target_area = mask_area(target_mask)
keypoints = self._rect_filter(track.keypoints, inside_tlbr,
self.fg_mask)
# only detect new keypoints when too few are propagated
if len(keypoints) < self.feature_density * target_area:
img = crop(self.prev_frame_gray, inside_tlbr)
feature_dist = self._estimate_feature_dist(
target_area, self.feat_dist_factor)
keypoints = cv2.goodFeaturesToTrack(img,
mask=target_mask,
minDistance=feature_dist,
**self.target_feat_params)
if keypoints is None:
keypoints = np.empty((0, 2), np.float32)
else:
keypoints = self._ellipse_filter(keypoints, track.tlbr,
inside_tlbr[:2])
# batch keypoints
all_prev_pts.append(keypoints)
# zero out target in foreground mask
target_mask[:] = 0
target_ends = list(
itertools.accumulate(
len(pts) for pts in all_prev_pts)) if all_prev_pts else [0]
target_begins = itertools.chain([0], target_ends[:-1])
# detect background feature points
prev_frame_small_bg = cv2.resize(self.prev_frame_gray,
None,
fx=self.bg_feat_scale_factor[0],
fy=self.bg_feat_scale_factor[1])
bg_mask_small = cv2.resize(self.fg_mask,
None,
fx=self.bg_feat_scale_factor[0],
fy=self.bg_feat_scale_factor[1],
interpolation=cv2.INTER_NEAREST)
keypoints = self.bg_feat_detector.detect(prev_frame_small_bg,
mask=bg_mask_small)
if len(keypoints) == 0:
self.bg_keypoints = np.empty((0, 2), np.float32)
LOGGER.warning('Camera motion estimation failed')
return {}, None
keypoints = np.float32([kp.pt for kp in keypoints])
keypoints = self._unscale_pts(keypoints, self.bg_feat_scale_factor)
bg_begin = target_ends[-1]
all_prev_pts.append(keypoints)
# match features using optical flow
all_prev_pts = np.concatenate(all_prev_pts)
scaled_prev_pts = self._scale_pts(all_prev_pts,
self.opt_flow_scale_factor)
all_cur_pts, status, err = cv2.calcOpticalFlowPyrLK(
self.prev_frame_small, frame_small, scaled_prev_pts, None,
**self.opt_flow_params)
status = self._get_status(status, err, self.max_error)
all_cur_pts = self._unscale_pts(all_cur_pts,
self.opt_flow_scale_factor, status)
# reuse preprocessed frame for next prediction
self.prev_frame_gray = frame_gray
self.prev_frame_small = frame_small
# estimate camera motion
homography = None
prev_bg_pts, matched_bg_pts = self._get_good_match(
all_prev_pts, all_cur_pts, status, bg_begin, -1)
if len(matched_bg_pts) < 4:
self.bg_keypoints = np.empty((0, 2), np.float32)
LOGGER.warning('Camera motion estimation failed')
return {}, None
homography, inlier_mask = cv2.findHomography(
prev_bg_pts,
matched_bg_pts,
method=cv2.RANSAC,
maxIters=self.ransac_max_iter,
confidence=self.ransac_conf)
self.prev_bg_keypoints, self.bg_keypoints = self._get_inliers(
prev_bg_pts, matched_bg_pts, inlier_mask)
if homography is None or len(self.bg_keypoints) < self.inlier_thresh:
self.bg_keypoints = np.empty((0, 2), np.float32)
LOGGER.warning('Camera motion estimation failed')
return {}, None
# estimate target bounding boxes
next_bboxes = {}
np.copyto(self.fg_mask, self.ones)
for begin, end, track in zip(target_begins, target_ends, tracks):
prev_pts, matched_pts = self._get_good_match(
all_prev_pts, all_cur_pts, status, begin, end)
prev_pts, matched_pts = self._fg_filter(prev_pts, matched_pts,
self.fg_mask, self.size)
if len(matched_pts) < 3:
track.keypoints = np.empty((0, 2), np.float32)
continue
# model motion as partial affine
affine_mat, inlier_mask = cv2.estimateAffinePartial2D(
prev_pts,
matched_pts,
method=cv2.RANSAC,
maxIters=self.ransac_max_iter,
confidence=self.ransac_conf)
if affine_mat is None:
track.keypoints = np.empty((0, 2), np.float32)
continue
est_tlbr = self._estimate_bbox(track.tlbr, affine_mat)
track.prev_keypoints, track.keypoints = self._get_inliers(
prev_pts, matched_pts, inlier_mask)
if (intersection(est_tlbr, self.frame_rect) is None
or len(track.keypoints) < self.inlier_thresh):
track.keypoints = np.empty((0, 2), np.float32)
continue
next_bboxes[track.trk_id] = est_tlbr
track.inlier_ratio = len(track.keypoints) / len(matched_pts)
# zero out predicted target in foreground mask
target_mask = crop(self.fg_mask, est_tlbr)
target_mask[:] = 0
return next_bboxes, homography
img_board_transformed = wrap_board2tile(img_tile, img_board, tile_corners,
board_corners)
img_tile_color_transformed = wrap_board2tile(
img_tile,
cv2.cvtColor(cv2.imread('tiles/board test.jpg'), cv2.COLOR_BGR2RGB),
tile_corners, board_corners)
# Display images
# cv2.imshow("Source Image", img_board)
# cv2.imshow("Destination Image", img_tile)
# cv2.imshow("Warped Source Image", img_board_transformed)
visualize_image(img_tile_color_transformed)
H, status = cv2.findHomography(board_corners, tile_corners, cv2.RANSAC, 5.0)
img_out = cv2.warpPerspective(img_board, H, ((int)(img_tile.shape[1]),
(int)(img_tile.shape[0])))
visualize_image(img_out, 'gray')
src = img_board
dst = img_tile
# Find the corners after the transform has been applied
height, width = src.shape[:2]
corners = np.array([[0, 0], [0, height - 1], [width - 1, height - 1],
[width - 1, 0]])
corners = cv2.perspectiveTransform(np.float32([corners]), H)[0]
# Find the bounding rectangle
bx, by, bwidth, bheight = cv2.boundingRect(corners)
image = cv2.imread(
"/home/dhanalaxmi/workspace/AI/Height_detection/image/object/hg.jpeg")
im = cv2.imread(
"/home/dhanalaxmi/workspace/AI/Height_detection/image/object/hg1.jpeg")
#Corner coordinates
image_corner = np.float32([[81, 447], [532, 447], [94, 756], [531, 758]])
cv2.circle(image, (81, 447), 8, (255, 0, 0), 5)
cv2.circle(image, (532, 447), 8, (255, 0, 0), 5)
cv2.circle(image, (94, 756), 8, (255, 0, 0), 5)
cv2.circle(image, (531, 758), 8, (255, 0, 0), 5)
plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
plt.show()
im_corner = np.float32([[43, 517], [288, 480], [77, 718], [322, 703]])
cv2.circle(im, (43, 517), 8, (255, 0, 0), 5)
cv2.circle(im, (288, 480), 8, (255, 0, 0), 5)
cv2.circle(im, (77, 718), 8, (255, 0, 0), 5)
cv2.circle(im, (322, 703), 8, (255, 0, 0), 5)
plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
plt.show()
#Homography matrix
h, mask = cv2.findHomography(im_corner, image_corner)
out = cv2.warpPerspective(im, h, (1000, 1000))
#Shoe the image
plt.imshow(cv2.cvtColor(out, cv2.COLOR_BGR2RGB))
plt.show()
while (1):
ret, QueryImgBGR = cam.read()
QueryImg = cv2.cvtColor(QueryImgBGR, cv2.COLOR_BGR2GRAY)
queryKP, queryDesc = detector.detectAndCompute(QueryImg, None)
matches = flann.knnMatch(queryDesc, trainDesc, k=2)
goodMatch = []
for m, n in matches:
if (m.distance < 0.75 * n.distance):
goodMatch.append(m)
if (len(goodMatch) > MIN_MATCH_COUNT):
tp = []
qp = []
for m in goodMatch:
tp.append(trainKP[m.trainIdx].pt)
qp.append(queryKP[m.queryIdx].pt)
tp, qp = np.float32((tp, qp))
H, status = cv2.findHomography(tp, qp, cv2.RANSAC, 3.0)
h, w = trainImg.shape
trainingBorder = np.float32([[[0, 0], [0, h - 1], [w - 1, h - 1],
[w - 1, 0]]])
queryBorder = cv2.perspectiveTransform(trainingBorder, H)
cv2.polylines(QueryImgBGR, [np.int32(queryBorder)], True, (0, 255, 0),
5)
print("Match Found")
else:
print("Not enough matches - %d%d" % (len(goodMatch), MIN_MATCH_COUNT))
cv2.imshow('result', QueryImgBGR)
cv2.waitKey(10)
img_left = cv2.cvtColor(imgA, cv2.COLOR_BGR2GRAY)
img_right = cv2.cvtColor(imgB, cv2.COLOR_BGR2GRAY)
# find the keypoints and descriptors with ORB - alternative to SIFT
orb = cv2.ORB_create(
nfeatures=5000
) #ablation study of top# of matches doesnt yield much visual difference between 1000-5000
locs1, des1 = orb.detectAndCompute(img_left, None)
locs2, des2 = orb.detectAndCompute(img_right, None)
# match features - https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_feature2d/py_matcher/py_matcher.html
bfmatch = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
matches = bfmatch.match(des1, des2)
sort_matches = sorted(matches, key=lambda x: x.distance)
matched_locs1 = np.array(
[locs1[match.queryIdx].pt for match in sort_matches[:700]])
matched_locs2 = np.array(
[locs2[match.trainIdx].pt for match in sort_matches[:700]]
) #ablation study of top# of matches doesnt yield much visual difference between 300-700
# compute homography - https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_feature2d/py_feature_homography/py_feature_homography.html
H, mask = cv2.findHomography(matched_locs1, matched_locs2, cv2.RANSAC, 5.0)
# compose composite image
composite_pano = compositeH(H, img_left, img_right)
composite_pano = cv2.cvtColor(composite_pano, cv2.COLOR_GRAY2RGB)
plt.imshow(composite_pano)
plt.show()
def metric_paf(Y_true, Y_pred, PATCH_SIZE, base_four_points,
perturbed_base_four_points):
# Compute the True H using Y_true
assert (Y_true.shape == Y_pred.shape
), "the shape of gt and pred should be the same"
batch_size = Y_true.shape[0]
mace_b = []
for i in range(batch_size):
Y_true_in_loop = Y_true[i, :, :, :]
Y_pred_in_loop = Y_pred[i, :, :, :]
base_four_points_in_loop = base_four_points[i, :]
perturbed_base_four_points_in_loop = perturbed_base_four_points[i, :]
# define corners of image patch
top_left_point = (base_four_points_in_loop[0],
base_four_points_in_loop[1])
bottom_left_point = (base_four_points_in_loop[2],
base_four_points_in_loop[3])
bottom_right_point = (base_four_points_in_loop[4],
base_four_points_in_loop[5])
top_right_point = (base_four_points_in_loop[6],
base_four_points_in_loop[7])
four_points = [
top_left_point, bottom_left_point, bottom_right_point,
top_right_point
]
perturbed_top_left_point = (perturbed_base_four_points_in_loop[0],
perturbed_base_four_points_in_loop[1])
perturbed_bottom_left_point = (perturbed_base_four_points_in_loop[2],
perturbed_base_four_points_in_loop[3])
perturbed_bottom_right_point = (perturbed_base_four_points_in_loop[4],
perturbed_base_four_points_in_loop[5])
perturbed_top_right_point = (perturbed_base_four_points_in_loop[6],
perturbed_base_four_points_in_loop[7])
perturbed_four_points = [
perturbed_top_left_point, perturbed_bottom_left_point,
perturbed_bottom_right_point, perturbed_top_right_point
]
predicted_pf_x1 = Y_pred_in_loop[:, :, 0]
predicted_pf_y1 = Y_pred_in_loop[:, :, 1]
pf_x1_img_coord = predicted_pf_x1
pf_y1_img_coord = predicted_pf_y1
y_patch_grid, x_patch_grid = np.mgrid[0:config.PATCH_SIZE,
0:config.PATCH_SIZE]
patch_coord_x = x_patch_grid + top_left_point[0]
patch_coord_y = y_patch_grid + top_left_point[1]
points_branch1 = np.vstack(
(patch_coord_x.flatten(), patch_coord_y.flatten())).transpose()
mapped_points_branch1 = points_branch1 + np.vstack(
(pf_x1_img_coord.flatten(),
pf_y1_img_coord.flatten())).transpose()
original_points = np.vstack((points_branch1))
mapped_points = np.vstack((mapped_points_branch1))
H_predicted = cv2.findHomography(np.float32(original_points),
np.float32(mapped_points), cv2.RANSAC,
10)[0]
predicted_delta_four_point = cv2.perspectiveTransform(
np.asarray([four_points], dtype=np.float32),
H_predicted).squeeze() - np.asarray(perturbed_four_points)
result = np.mean(np.linalg.norm(predicted_delta_four_point, axis=1))
mace_b.append(result)
__mace = np.mean(mace_b)
return __mace
def compute_homo_transformation(query_img, train_img, f=None):
# initiate SIFT detector
sift = cv2.xfeatures2d.SIFT_create()
# find the keypoints and descriptors with SIFT
kp_query, des_query = sift.detectAndCompute(query_img, None)
kp_train, des_train = sift.detectAndCompute(train_img, None)
n_features_query = len(kp_query)
n_features_train = len(kp_train)
if f:
f.write('Number of features for query image: {}\n'.format(
n_features_query))
f.write('Number of features for query image: {}\n'.format(
n_features_train))
# convert image colors to grey so that sift features can be drawn on top
gray_query_img = cv2.cvtColor(query_img, cv2.COLOR_BGR2GRAY)
gray_train_img = cv2.cvtColor(train_img, cv2.COLOR_BGR2GRAY)
# draw sift features on gray image
draw_kp_params = dict(outImage=np.array([]),
flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
img_query_features = cv2.drawKeypoints(gray_query_img, kp_query,
**draw_kp_params)
img_train_features = cv2.drawKeypoints(gray_train_img, kp_train,
**draw_kp_params)
# BFMatcher with variable k parameter
bf = cv2.BFMatcher()
matches = bf.knnMatch(des_query, des_train, k=2)
# get number of matches
n_matches = len(matches)
# filter only good matches using ratio test
matches = filter(lambda f: f[0].distance < 0.7 * f[1].distance, matches)
matches = map(lambda f: [f[0]], matches)
# sort matches by distance
matches = sorted(matches, key=lambda f: f[0].distance)
# get number of matches and number of good matches
n_good_matches = len(matches)
if f:
f.write("Number of matches: {}\n".format(n_matches))
f.write("Number of good matches: {}\n".format(n_good_matches))
# draw first 20 matches
draw_params = dict(matchColor=(0, 255, 0),
singlePointColor=None,
outImg=np.array([]),
flags=2)
img_matches_before = cv2.drawMatchesKnn(query_img, kp_query, train_img,
kp_train, matches[:20],
**draw_params)
# extract points in the matches
matched_points_query = np.vstack(
[kp_query[match[0].queryIdx].pt for match in matches])
matched_points_train = np.vstack(
[kp_train[match[0].trainIdx].pt for match in matches])
# find homography between query and train image using RANSAC algorithm
h, status = cv2.findHomography(matched_points_query, matched_points_train,
cv2.RANSAC)
# write
if f:
f.write('h Matrix:\n')
f.write(str(h))
f.write('\n')
# calculate boundaries of query image after homography transformation
corners = get_corner_coords(query_img.shape)
transformed_corners = cv2.perspectiveTransform(np.float32([corners]),
h).astype(np.int32)
train_img_with_transformed_corners = cv2.polylines(train_img,
transformed_corners,
True, (246, 148, 59), 4)
# caculate key points after homography transformation
kp_warp = cv2.perspectiveTransform(
np.array([[[kp.pt[0], kp.pt[1]] for kp in kp_query]]), h)[0]
kp_warp = np.array(
[cv2.KeyPoint(pt[0], pt[1], 1) for pt in kp_warp.tolist()])
# filter matched key points
matches_after_homo = list(
filter(
lambda match: dist_kp(kp_warp[match[0].queryIdx], kp_train[match[
0].trainIdx]) < 1, matches))
# get total number of matches consitent with the computed homography
n_consistent_matches = len(matches_after_homo)
if f:
f.write("Number of matches consistent with homography: {}\n".format(
n_consistent_matches))
# show top 10 matches after homography transformation
img_matches_after = cv2.drawMatchesKnn(query_img, kp_query,
train_img_with_transformed_corners,
kp_train, matches_after_homo[:10],
**draw_params)
return img_query_features, img_train_features, img_matches_before, img_matches_after, h
def main():
# Initializing network
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
detector_graph = tf.Graph()
decoder_graph = tf.Graph()
with detector_graph.as_default():
detector_sess = tf.Session()
detector_model = tf.saved_model.loader.load(detector_sess,
[tag_constants.SERVING],
args.detector_model)
detector_input_name = detector_model.signature_def[
signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY].inputs['image'].name
detector_input = detector_graph.get_tensor_by_name(detector_input_name)
detector_output_name = detector_model.signature_def[
signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY].outputs['detections'].name
detector_output = detector_graph.get_tensor_by_name(
detector_output_name)
with decoder_graph.as_default():
decoder_sess = tf.Session()
decoder_model = tf.saved_model.loader.load(decoder_sess,
[tag_constants.SERVING],
args.decoder_model)
decoder_input_name = decoder_model.signature_def[
signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY].inputs['image'].name
decoder_input = decoder_graph.get_tensor_by_name(decoder_input_name)
decoder_output_name = decoder_model.signature_def[
signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY].outputs['decoded'].name
decoder_output = decoder_graph.get_tensor_by_name(decoder_output_name)
cap = cv2.VideoCapture(args.video)
bch = bchlib.BCH(BCH_POLYNOMIAL, BCH_BITS)
ret, frame = cap.read()
f_height, f_width = frame.shape[0:2]
if args.save_video is not None:
fourcc1 = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(args.save_video, fourcc1, 30.0,
(f_width, f_height))
ii = 0
while (True):
ret, frame = cap.read()
if frame is None:
break
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
detector_image_input_0 = cv2.resize(frame_rgb, (1024, 1024))
detector_image_input = np.expand_dims(
np.float32(detector_image_input_0), axis=0) / 255.0
# print('detector_image_input', detector_image_input)
output_image = detector_sess.run(
detector_output, feed_dict={detector_input: detector_image_input})
output_image = np.array(output_image[0, :, :, :])
output_image = x = np.argmax(output_image, axis=-1)
color_codes = np.array([[255, 255, 255], [0, 0, 0]])
out_vis_image = color_codes[output_image.astype(int)]
# writeImage(out_vis_image.astype(np.uint8), 'test_contour')
mask_im = cv2.resize(np.float32(out_vis_image), (f_width, f_height))
if args.visualize_detector:
mask_vis = mask_im.astype(np.uint8)
# print('args.visualize_detector', args.visualize_detector)
# print('frame.size', (f_width,f_height))
ii += 1
jj = 0
contours, _ = cv2.findContours(
cv2.cvtColor(mask_im, cv2.COLOR_BGR2GRAY).astype(np.uint8), 1, 2)
extrema = np.zeros((8, 2))
corners = np.zeros((4, 2))
for cnt in contours:
area = cv2.contourArea(cnt)
if area < 1000:
continue
hull = cv2.convexHull(cnt)
if len(hull) < 4:
continue
if args.visualize_detector:
cv2.polylines(mask_vis,
np.int32([corners]),
thickness=6,
color=(100, 100, 250),
isClosed=True)
extrema[0, :] = hull[np.argmax(hull[:, 0, 0]), 0, :]
extrema[1, :] = hull[np.argmax(hull[:, 0, 0] + hull[:, 0, 1]),
0, :]
extrema[2, :] = hull[np.argmax(hull[:, 0, 1]), 0, :]
extrema[3, :] = hull[np.argmax(-hull[:, 0, 0] + hull[:, 0, 1]),
0, :]
extrema[4, :] = hull[np.argmax(-hull[:, 0, 0]), 0, :]
extrema[5, :] = hull[np.argmax(-hull[:, 0, 0] - hull[:, 0, 1]),
0, :]
extrema[6, :] = hull[np.argmax(-hull[:, 0, 1]), 0, :]
extrema[7, :] = hull[np.argmax(hull[:, 0, 0] - hull[:, 0, 1]),
0, :]
extrema_lines = extrema - np.roll(extrema, shift=1, axis=0)
extrema_len = extrema_lines[:, 0]**2 + extrema_lines[:, 1]**2
line_idx = np.sort(extrema_len.argsort()[-4:])
for c in range(4):
p1 = extrema[line_idx[(c - 1) % 4], :]
p2 = extrema[(line_idx[(c - 1) % 4] - 1) % 8, :]
p3 = extrema[line_idx[c], :]
p4 = extrema[(line_idx[c] - 1) % 8, :]
corners[c, :] = get_intersect(p1, p2, p3, p4)
new_area = poly_area(corners)
if new_area / area > 1.5:
continue
corners = order_points(corners)
corners_full_res = corners
pts_dst = np.array([[0, 0], [399, 0], [399, 399], [0, 399]])
h, status = cv2.findHomography(corners_full_res, pts_dst)
try:
warped_im = cv2.warpPerspective(frame_rgb, h, (400, 400))
# print('warped_im', warped_im)
w_im = warped_im.astype(np.float32)
w_im /= 255.
except:
continue
print('----,', 'frame:', ii, 'index:', jj)
jj += 1
for im_rotation in range(4):
w_rotated = np.rot90(w_im, im_rotation)
recovered_secret = decoder_sess.run(
[decoder_output], feed_dict={decoder_input:
[w_rotated]})[0][0]
recovered_secret = list(recovered_secret)
recovered_secret = [int(i) for i in recovered_secret]
packet_binary = "".join(
[str(bit) for bit in recovered_secret[:96]])
footer = recovered_secret[96:]
if np.sum(footer) > 0:
continue
packet = bytes(
int(packet_binary[i:i + 8], 2)
for i in range(0, len(packet_binary), 8))
packet = bytearray(packet)
data, ecc = packet[:-bch.ecc_bytes], packet[-bch.ecc_bytes:]
bitflips = bch.decode_inplace(data, ecc)
if bitflips != -1:
print('Num bits corrected: ', bitflips)
try:
code = data.decode("utf-8")
print('----', code)
writeImage(warped_im, 'test', ii, jj)
writeImage(frame_rgb, 'test_raw', ii, jj)
writeImage(out_vis_image.astype(np.uint8),
'test_contour', ii, jj)
writeImage(detector_image_input_0, 'test_raw_1024', ii,
jj)
except:
continue
color = (100, 250, 100)
cv2.polylines(frame,
np.int32([corners]),
thickness=6,
color=color,
isClosed=True)
font = cv2.FONT_HERSHEY_SIMPLEX
im = cv2.putText(
frame, code,
tuple((corners[0, :] + np.array([0, -15])).astype(
np.int)), font, 1, (0, 0, 0), 2, cv2.LINE_AA)
if args.save_video is not None:
out.write(frame)
else:
cv2.imshow('frame', frame)
if args.visualize_detector:
cv2.imshow('detector_mask', mask_vis)
cv2.waitKey(1)
cap.release()
if args.save_video:
out.release()
matches = feature_matcher.knnMatch(des, des2, k=2)
# store all the good matches as per Lowe's ratio test.
good = []
for m, n in matches:
if m.distance < 0.7 * n.distance:
good.append(m)
if len(good) > MIN_MATCH_COUNT:
src_pts = np.float32([kp[m.queryIdx].pt
for m in good]).reshape(-1, 1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt
for m in good]).reshape(-1, 1, 2)
# estimating homography
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,
ransac_outlier_thresh)
matchesMask = mask.ravel().tolist()
print(f" estimated homography :\n{M}\n")
real_homog = np.loadtxt(f"{homg_dir}/H1to{img2_num}p",
delimiter=',')
print(f" real homography :\n{real_homog}\n")
# calculating error based on frobenius norm
error = ((M - real_homog)**2).sum()
print(f"error in estimation : {error}")
h, w = gray.shape
pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1],
[w - 1, 0]]).reshape(-1, 1, 2)
dst = cv2.perspectiveTransform(pts, M)
rawMatches = matcher.knnMatch(desA, desB, 2)
matches = []
H = np.ones((3, 3), dtype=float)
# loop over the raw matches
for m in rawMatches:
if len(m) == 2 and m[0].distance < m[1].distance * 0.75:
matches.append((m[0].trainIdx, m[0].queryIdx))
# computing a homography (need at least 4 matches)
if len(matches) > 4:
# construct the two sets of points
ptsA = np.float32([kpA[i] for (_, i) in matches])
ptsB = np.float32([kpB[i] for (i, _) in matches])
# compute the homography using RANSAC method
(H, status) = cv.findHomography(ptsA, ptsB, cv.RANSAC, 4.0)
result = cv.warpPerspective(
image1, H, (image1.shape[1] + image2.shape[1], image1.shape[0]))
result[0:image2.shape[0], 0:image2.shape[1]] = image2
(hA, wA) = image1.shape[:2]
(hB, wB) = image2.shape[:2]
vis = np.zeros((max(hA, hB), wA + wB, 3), dtype="uint8")
vis[0:hA, 0:wA] = image1
vis[0:hB, wA:] = image2
# loop over the matches
for ((trainIdx, queryIdx), s) in zip(matches, status):
# only process the match if the keypoint was successfully
# matched
def find_homography_matrix(self, key1, key2):
if key1 in self.court_points.keys() and key2 in self.court_points.keys():
H, status = cv2.findHomography(self.court_points.get(key1), self.court_points.get(key2))
return H
else:
print("Enter valid tournament names")
# src_pts = Coordinates of the points in the original figure),
# (dst_pts = Coordinates of the points in the target figure by evaluating radius of the circles)
src_pts = []
dst_pts = []
for p1 in circlesA:
for p2 in circlesB:
# print(p1[2])
if p1[2] > 0 and p2[2] > 0 and abs(1 - p1[2] / p2[2]) < 0.05:
src_pts.append(p1[0:2])
dst_pts.append(p2[0:2])
src_pts = np.float32(src_pts)
dst_pts = np.float32(dst_pts)
# Find the transformation between points, standard RANSAC
transformation_matrix, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,
5.0)
#printing and saving the transformation_matrix
print(transformation_matrix)
np.savetxt('Problem3a.txt',
transformation_matrix,
header='transformation_matrix',
fmt='%.1e',
delimiter=',')
# Just for checking the code we add a "Gray Ring" to the figure A
cv2.circle(imgA, (20, 100), 10, 127, 4)
# apply transformation
imgAT = cv.warpPerspective(imgA, transformation_matrix,
(imgA.shape[1], imgA.shape[0]))
def RetrieveDataStep(self, imgpath):
"""
WordBoxes: List(ndarray 4x2)
CharBoxes: List(ndarray 4x2)
"""
# step 1: iterate through text instances, obtain coordinates of four corners, use TextInstance Attributes to calculate screen location
Texts = []
WordBoxes = []
CharBoxes = []
for TextID in range(self.MaxTextCount):
if self.AllTextObjects[TextID].IsValid:
BoxRelativePosition = self.AllTextObjects[TextID].adjusted_pos[:, 0, :] # 4x1x2 -> 4x2
source_pts = np.array([[0., 0.], [1., 0.], [1., 1.], [0., 1.]]).astype(np.float32) # 4x2
h, status = cv2.findHomography(source_pts, BoxRelativePosition) # h = ndarray(3x3)
h = h.transpose()
# text
Texts.extend(self.AllTextObjects[TextID].text)
# bbox
bbox = self.AllTextObjects[TextID].bbox # nx4x2
if True:
# homography transformation
bbox = bbox.reshape((-1, 2)) # 4n x 2
bbox = np.concatenate([bbox, np.ones((bbox.shape[0], 1), dtype=np.float32)],
axis=-1) # 4n x 3
bbox = np.matmul(bbox, h)
bbox = bbox / bbox[:, 2:3]
screenbox = bbox[:, :2].reshape((-1, 4, 2)) # 4n x 2 -> n x 4 x 2
else:
# affine transformation
UpperEdge = (1 - bbox[:, :, :1]) * BoxRelativePosition[:1, :, :] + bbox[:, :, :1] * BoxRelativePosition[
1:2, :, :] # nx4x2
BottomEdge = (1 - bbox[:, :, :1]) * BoxRelativePosition[3:, :, :] + bbox[:, :, :1] * BoxRelativePosition[
2:3, :, :] # nx4x2
screenbox = (1 - bbox[:, :, 1:]) * UpperEdge + bbox[:, :, 1:] * BottomEdge # nx4x2
# screenbox *= self.HighResFactor
WordBoxes.extend(
[
[Point[0][0], Point[0][1], Point[1][0], Point[1][1], Point[2][0], Point[2][1], Point[3][0], Point[3][1]] for Point in screenbox.astype(np.int32).tolist() # nx8
]
)
# cbox
cbox = self.AllTextObjects[TextID].cbox ## nx4x2
if True:
# homography transformation
cbox = cbox.reshape((-1, 2)) # 4n x 2
cbox = np.concatenate([cbox, np.ones((cbox.shape[0], 1), dtype=np.float32)],
axis=-1) # 4n x 3
cbox = np.matmul(cbox, h)
cbox = cbox / cbox[:, 2:3]
screencbox = cbox[:, :2].reshape((-1, 4, 2)) # 4n x 2 -> n x 4 x 2
else:
# affine transformation
UpperEdge = (1 - cbox[:, :, :1]) * BoxRelativePosition[:1, :, :] + cbox[:, :, :1] * BoxRelativePosition[
1:2, :, :] # nx4x2
BottomEdge = (1 - cbox[:, :, :1]) * BoxRelativePosition[3:, :, :] + cbox[:, :,
:1] * BoxRelativePosition[
2:3, :, :] # nx4x2
screencbox = (1 - cbox[:, :, 1:]) * UpperEdge + cbox[:, :, 1:] * BottomEdge # nx4x2
# screencbox *= self.HighResFactor
CharBoxes.extend(
[
[Point[0][0], Point[0][1], Point[1][0], Point[1][1], Point[2][0], Point[2][1], Point[3][0], Point[3][1]] for Point in screencbox.astype(np.int32).tolist() # nx8
]
)
self.AllTextObjects[TextID].IsValid = False
return imgpath, Texts, WordBoxes, CharBoxes, self.TextNum
def create_hive_coords(input_files_location, output_location):
# make sure input_files_location contains only directories with only csv-files and no subdirs
if not os.path.exists(output_location):
os.makedirs(output_location)
####################
# true world coordinate system corner points:
# t1 ------------------ t2
# | |
# | |
# | |
# t4 ------------------ t3
t1 = [0.0,0.0]
t2 = [35.2,0.0]
t3 = [35.2,20.0]
t4 = [0.0,20.0]
pts_dst = numpy.array([t1,t2,t3,t4])
# in cm
####################
dirnames = os.listdir(input_files_location)
print(dirnames)
dirnum = len(dirnames)
i = 0
for dirname in dirnames:
current_path = os.path.join(input_files_location,dirname)
cam0_text = ""
cam1_text = ""
files = os.listdir(current_path)
filenum = len(files)
print("starting directory " + str(i+1) + " of " + str(dirnum) + " which contains " + str(filenum) + " files")
i = i+1
for file in files:
f = open(os.path.join(current_path,file))
r = f.read()
f.close()
cam = file[14] # get from filename
####################
r2 = r.split("\n") # divide by line
position = r2[0]
time = r2[1]
corners = r2[2]
####################
posdata = position.split() # divide by whitespace
pos = numpy.array([float(posdata[0])*4,float(posdata[1])*4,1.0]) # position of dance in WDD (value was downscaled to 160x120 resolution; *4 gets it back to 640x480)
###
timedata = time.split()
time = timedata[0]
###
cornerdata = corners.split()
p1 = [float(cornerdata[0]),float(cornerdata[1])]
p2 = [float(cornerdata[2]),float(cornerdata[3])]
p3 = [float(cornerdata[4]),float(cornerdata[5])]
p4 = [float(cornerdata[6]),float(cornerdata[7])]
pts_src = numpy.array([p1,p2,p3,p4])
# p1 ------------------ p2
# | |
# | |
# | |
# p4 ------------------ p3
####################
h, status = cv2.findHomography(pts_src, pts_dst) # h is homography matrix
calc = numpy.dot(h,pos) # transforming the WDD dance position
world_coord_pos = [round(calc[0],4),round(calc[1],4)]
if cam == '0':
cam0_text = cam0_text + time + " " + str(world_coord_pos[0]) + " " + str(world_coord_pos[1]) + "\n"
elif cam == '1':
cam1_text = cam1_text + time + " " + str(world_coord_pos[0]) + " " + str(world_coord_pos[1]) + "\n"
else:
print(error)
f0 = open(os.path.join(output_location,dirname + " cam0.txt"),'w')
f0.write(cam0_text)
f0.close()
f1 = open(os.path.join(output_location,dirname + " cam1.txt"),'w')
f1.write(cam1_text)
f1.close()
print("completed!")
def feat_det(image):
MIN_MATCH_COUNT = 10
# Reads in source for cam feed
# if VideoCapture(0) doesn't work, try -1, 1, 2, 3 (if none of those work, the webcam's not supported!)
feed = 0
for src in range(-1, 4):
cam = cv2.VideoCapture(src)
print(src)
ret_val, testImg = cam.read()
if testImg is None:
continue
else:
feed = src
#cam = cv2.VideoCapture(feed)
cam = cv2.VideoCapture(0)
# Reads in the image
img1 = cv2.imread(image, 0)
# Labels the image as the name passed in
if args["label"] is not None:
label = args["label"]
else:
# Takes the name of the image as the name
if image[:2] == "./":
label = image[2:-4]
else:
label = image[:-4]
# Initiate ORB detector
orb = cv2.ORB_create()
# Initiate SIFT detector
sift = cv2.xfeatures2d.SIFT_create()
# Initiate SURF detector
surf = cv2.xfeatures2d.SURF_create(200)
# Find the keypoints and descriptors of the provided image with ORB
kp1, des1 = orb.detectAndCompute(img1, None)
# Find the keypoints and descriptors of the provided image with SIFT
kp2, des2 = sift.detectAndCompute(img1, None)
# Find the keypoints and descriptors of the provided image with SURF
kp3, des3 = surf.detectAndCompute(img1, None)
FLANN_INDEX_KDTREE = 0
# Option of changing 'trees' and 'checks' values for different levels of accuracy
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5) # 5
search_params = dict(checks = 50) # 50
# Fast Library for Approximate Nearest Neighbor Algorithm
# creates FLANN object for use below
flann = cv2.FlannBasedMatcher(index_params, search_params)
# Brute Force matcher
# creates BFMatcher object for use below
bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck = False)
# Setup for keypresses that allow the user to switch between the algorithms
# the default algorithm on startup is ORB
alg = 1
# handles keypresses
while True:
c = cv2.waitKey(1)
if c == ord('o'):
alg = 1
elif c == ord('i'):
alg = 2
elif c == ord('s'):
alg = 3
ret_val, img2 = cam.read()
if img2 is None : # did we get an image at all?
#print ("No image")
continue
# The 'o' key starts the Orb algorithm, trades accuracy for speed
if alg == 1:
kp1_s, des1_s = orb.detectAndCompute(img2, None)
# Uses the BFMatcher algorithm to search for similar elements of two images
# uses simpler calculations than FLANN
matches = bf.knnMatch(des1, des1_s, k = 2 )
kp = kp1
kp_L = kp1_s
alg_type = "Orb"
# The 'i' key starts the Sift algorithm, trades speed for accuracy
elif alg == 2:
kp2_s, des2_s = sift.detectAndCompute(img2, None)
# Uses the FLANN algorithm to search for nearest neighbors between elements of two images
# faster than the BFMatcher for larger datasets
matches = flann.knnMatch(des2, des2_s, k = 2)
kp = kp2
kp_L = kp2_s
alg_type = "Sift"
# The 's' key starts the Surf algorithm, trades speed for accuracy
# and is supposed to be faster than SIFT, but OpenCV's implementation is unoptimized
elif alg == 3:
kp3_s, des3_s = surf.detectAndCompute(img2, None)
# Uses the FLANN algorithm to search for nearest neighbors between elements of two images
# faster than the BFMatcher for larger datasets
matches = flann.knnMatch(des3, des3_s, k = 2)
kp = kp3
kp_L = kp3_s
alg_type = "Surf"
if len(matches) == 0:
continue
# Store all the good matches (based off Lowe's ratio test)
good = []
for m,n in matches:
if m.distance < 0.75 * n.distance:
good.append(m)
# When there are enouugh matches, we convert the keypoints to floats in order to draw them later
if len(good) >= MIN_MATCH_COUNT:
src_pts = np.float32([ kp[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp_L[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
# Homography adds a degree of rotation/translation invariance by mapping the transformation
# of points between two images
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()
h, w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
if M is not None:
dst = cv2.perspectiveTransform(pts, M)
intDst = np.int32(dst)
# Polylines is an alternative bounding shape around the largest area of recognition
#cv2.polylines(img2,[np.int32(dst)],True,(255, 0, 0), 2, cv2.LINE_AA)
# Draws a bounding box around the area of most matched points
cv2.rectangle(img2, (intDst[0][0][0], intDst[0][0][1]), (intDst[2][0][0], intDst[2][0][1]), (255, 0, 0), 1, cv2.LINE_AA, 0)
cv2.putText(img2, label, (dst[0][0][0], dst[0][0][1]) , cv2.FONT_HERSHEY_TRIPLEX, 1.0, (0, 0, 255), lineType = cv2.LINE_AA )
else:
matchesMask = None
else:
#print ("Not enough matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
matchesMask = None
draw_params = dict(matchColor = (0,255,0), # draw matches in green color
singlePointColor = None,
matchesMask = matchesMask, # draw only inliers
flags = 2)
# Option of slicing the 'good' list to display a certain number of matches (ex. good[:6])
img3 = cv2.drawMatches(img1, kp, img2, kp_L, good, None, **draw_params)
# Makes the window 1.5x bigger
img3 = cv2.resize(img3, (0,0), fx = 1.5, fy = 1.5)
# Displays the keys for switching algorithms and the current algorithm
height, width = img3.shape[:2]
cv2.putText(img3, "Hold 'o' for ORB, 'i' for SIFT, and 's' for SURF", (width - 950, 25),
cv2.FONT_HERSHEY_DUPLEX, 0.75, (0,0,255), 2, cv2.LINE_AA)
cv2.rectangle(img3, (width - 200, 0), (width, 40), (255,255,255), -1)
cv2.putText(img3, "Detection Alg: "+ alg_type, (width - 190, 25), cv2.FONT_HERSHEY_SIMPLEX,
0.6, (0,0,0), 2, 9)
if cv2.waitKey(1) == 27:
break # esc to quit
# Shows the current frame
cv2.imshow("My Webcam", img3)
cv2.destroyAllWindows()
p0 = []
p1 = []
for m in matches:
p0.append(trainingKPs[m.trainIdx].pt)
p1.append(targetKPs[m.queryIdx].pt)
p0, p1 = np.float32((p0, p1))
print(len(p1))
for (x, y) in np.int32(p0):
cv2.circle(trainingImage, (x, y), 10, (0, 0, 0))
for (x, y) in np.int32(p1):
cv2.circle(targetImage, (x, y), 10, (0, 255, 255))
H, status = cv2.findHomography(p0, p1, cv2.LMEDS, 5.0)
status = status.ravel() != 0
p0, p1 = p0[status], p1[status]
print(len(p1))
# for (x, y) in np.int32(p0):
# cv2.circle(trainingImage, (x, y), 8, (255, 255, 0))
# for (x, y) in np.int32(p1):
# cv2.circle(targetImage, (x, y), 8, (255, 255, 0))
h = trainingCopy.shape[0]
w = trainingCopy.shape[1]
trainBorder = np.float32(
[[[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]])
print(trainBorder)
def solution(left_img, right_img):
"""
:param left_img:
:param right_img:
:return: you need to return the result image which is stitched by left_img and right_img
"""
'gray scale image conversion'
left_img_gray = cv2.cvtColor(
left_img, cv2.COLOR_BGR2GRAY) # left-image in gray scale
right_img_gray = cv2.cvtColor(
right_img, cv2.COLOR_BGR2GRAY) # left-image in gray scale
'SIFT operation'
sift = cv2.xfeatures2d.SIFT_create() # applying sift
left_img_keypoint, left_img_descriptor = sift.detectAndCompute(
left_img_gray, None)
right_img_keypoint, right_img_descriptor = sift.detectAndCompute(
right_img_gray, None)
'FLANN matching technique to find K Nearest Neighbours'
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=100)
'Creating object for FLANN matching'
doFlann = cv2.FlannBasedMatcher(index_params, search_params)
matches = doFlann.knnMatch(left_img_descriptor, right_img_descriptor, k=2)
Keypoints = []
for m, n in matches:
if m.distance < 0.75 * n.distance:
Keypoints.append(m)
MIN_MATCHES = 10
if len(Keypoints) > MIN_MATCHES:
coordinate1 = np.array(
[left_img_keypoint[m.queryIdx].pt for m in Keypoints])
np.reshape(coordinate1, (-1, 1, 2))
coordinate1 = np.float32(coordinate1)
coordinate2 = np.array(
[right_img_keypoint[m.trainIdx].pt for m in Keypoints])
np.reshape(coordinate2, (-1, 1, 2))
coordinate2 = np.float32(coordinate2)
H, homographyStatus = cv2.findHomography(coordinate1, coordinate2,
cv2.RANSAC, 5.0)
else:
print(" Enough matches have not been found - %d/%d" %
(len(Keypoints), MIN_MATCHES))
'Shapes of the two images'
left_height, left_width = left_img_gray.shape
right_height, right_width = right_img_gray.shape
'Corners of each image'
points_m1 = np.float32([[0, 0], [0,
left_height], [left_width, left_height],
[left_width, 0]]).reshape(-1, 1, 2)
points_m2 = np.float32([[0, 0], [0, right_height],
[right_width, right_height],
[right_width, 0]]).reshape(-1, 1, 2)
'Translation and rotation of each corner of left image with respect to the right image using Homography matrix'
points_m1_modify = cv2.perspectiveTransform(points_m1, H)
'''Add the translated corners of the left image to the corners of the right image in the form of a
list of list for each corner location on the canvas. We have added along axis=0, i.e. along the rows'''
all_mPoints = np.concatenate((points_m2, points_m1_modify), axis=0)
'''Finding the minimum and maximum values from all the corners to create the size of the frame at a
distance of +-1 '''
[pano_xmin, pano_ymin] = np.int32(all_mPoints.min(axis=0).ravel() - 1.0)
[pano_xmax, pano_ymax] = np.int32(all_mPoints.max(axis=0).ravel() + 1.0)
'Calculation of the left upper most and lower most corners of the frame'
transformationM = [-pano_ymin, -pano_xmin]
'Compute the Translated matrix for the new frame, with respect to which the H will be translated'
translatedH = np.array([[1, 0, transformationM[1]],
[0, 1, transformationM[0]], [0, 0, 1]])
'Warp the left image with respect to the right image'
img_panorama = cv2.warpPerspective(
left_img, translatedH.dot(H),
(pano_xmax - pano_xmin, pano_ymax - pano_ymin))
img_panorama[transformationM[0]:right_height + transformationM[0],
transformationM[1]:right_width +
transformationM[1]] = right_img
return img_panorama
v = np.array([(n2 / 2, 0)])
P2 = np.array([(0, 4 * m2 / 5), (5 * n2 / 6, m2), (3 * n2 / 12, 7 * m2 / 12),
(n2, 8 * m2 / 12)])
cv2.line(I, (0, 0), (0, m - 1), (1, 1, 1), 4)
cv2.line(I, (0, 0), (n - 1, 0), (1, 1, 1), 4)
cv2.line(I, (n - 1, m - 1), (0, m - 1), (1, 1, 1), 4)
cv2.line(I, (n - 1, m - 1), (n - 1, 0), (1, 1, 1), 4)
for i in range(4):
cv2.line(J, (v[0, 0], v[0, 1]), (P2[i, 0], P2[i, 1]), (0, 0, 0), 2)
p21 = P2[1].copy()
for i in range(psize):
H, status = cv2.findHomography(P1, P2)
K = cv2.warpPerspective(I, H, (J.shape[1], J.shape[0]))
msk = K.max(axis=2) != 0
J[msk, :] = K[msk, :]
cv2.line(J, (v[0, 0], v[0, 1]), (p21[0], p21[1]), (0, 0, 0), 2)
cv2.imshow('', J)
cv2.waitKey()
P2 = (P2 + v) / 2
def correctImageByPerspectiveOrHough(img_to_correct,
img_result,
flag_of_path=0):
# img_to_correct = rodataImageToCorrect(img_to_correct)
# img_to_correct_gery = cv2.cvtColor(img_to_correct, cv2.COLOR_RGB2GRAY)
# rct, img_to_correct_binary = cv2.threshold(img_to_correct_gery, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
# to find all corner dots
img_to_correct_binary = thresholdImage(img_to_correct.copy())
shape_harr = img_to_correct.shape
corrdia_list = []
image_x = cv2.Sobel(img_to_correct, cv2.CV_32F, 1, 0, ksize=3) #X方向Sobel
absX = cv2.convertScaleAbs(image_x) # 转回uint8
absX = thresholdImage(absX)
showImg("absX", absX)
image_y = cv2.Sobel(img_to_correct, cv2.CV_32F, 0, 1, ksize=3) #Y方向Sobel
absY = cv2.convertScaleAbs(image_y)
absY = thresholdImage(absY)
showImg('absY', absY)
dst = np.ones((shape_harr[0], shape_harr[1], 1), dtype=np.uint8)
for i in range(shape_harr[0]):
for j in range(shape_harr[1]):
if absX[i][j] > 100 and absY[i][j] > 100:
corrdia_list.append([i, j])
dst[i][j] = 255
# dst=cv2.addWeighted(absX,0.5,absY,0.5,0)
# dst = thresholdImage(dst)
showImg('dst', dst)
# shape_harr = img_to_correct_binary.shape
# img_to_correct_gery_f32 = np.float32(img_to_correct_binary)#img_to_correct_gery)
# dst = cv2.cornerHarris(img_to_correct_gery_f32,40,3,0.04)
# dst = cv2.dilate(dst,None)
# showImg('img_to_correct_gery_f32',img_to_correct_gery_f32)
# corrdia_list = (np.array(np.argwhere((dst > 0.015*dst.max())==True))).tolist()#,dtype = dt)
# img_to_correct1 = img_to_correct.copy()
# img_to_correct1[dst>0.01*dst.max()]=[0,0,255]
# showImg('img_to_harr1',img_to_correct1)
# # img_to_correct[dst>0.01*dst.max()]=[0,0,255]
# # showImg('img_to_correct1',img_to_correct)
# 将角点按照x和y坐标位置分别排序，分别取前图像前四分之一和四分之一的角点，
#拿出来做环状扫描，得到开口方向和角度的，最终挑出4个角点做计算投影变换的变量
corrdia_list.sort(key=compator_x)
# corrdia_list.sort(key=compator_y)
threshold_to_harr_x = int(shape_harr[0] / 4)
threshold_to_harr_y = int(shape_harr[1] / 4)
# corrdia_list_result =[]
# lrub_list = [[]]
lu_list = []
lb_list = []
rb_list = []
ru_list = []
canvas_point = []
for position in corrdia_list:
x = position[0]
y = position[1]
# set x threshold to screen the dot
if x <= threshold_to_harr_x or x >= shape_harr[0] - threshold_to_harr_x:
begin, end, pixel_list, white_counter, black_counter = constructCircleToJudgeCornerDot(
img_to_correct_binary, x, y, 14, shape_harr)
print(
'x is {},y is {},begin is {},end is {},white_counter is {},black_counter is {}'
.format(x, y, begin, end, white_counter, black_counter))
if begin != -1:
foreground_percent = float(white_counter) / float(
black_counter + white_counter)
if foreground_percent < 0.6 and foreground_percent > 0.2:
index = judegeCircleDirection(begin, end, len(pixel_list),
0.08)
# if index == -1:
# cv2.circle(img_to_correct,(y,x),16,(0,255,255))
# ##print('*************hhhhh',x,y)
if index != -1:
# corrdia_list_result.append(position)
#img_to_correct[x][y] = [0,0,255]
if index == 1:
##cv2.circle(img_to_correct,(y,x),16,(0,255,0))
if lu_list.count(position) == 0:
lu_list.append(position)
if index == 2:
##cv2.circle(img_to_correct,(y,x),16,(0,0,255))
if lb_list.count(position) == 0:
lb_list.append(position)
if index == 3:
##cv2.circle(img_to_correct,(y,x),16,(255,0,0))
if rb_list.count(position) == 0:
rb_list.append(position)
if index == 4:
##cv2.circle(img_to_correct,(y,x),16,(255,255,0))
if ru_list.count(position) == 0:
ru_list.append(position)
# set y threshold to screen corner dots
if y <= threshold_to_harr_y or y >= shape_harr[1] - threshold_to_harr_y:
begin, end, pixel_list, white_counter, black_counter = constructCircleToJudgeCornerDot(
img_to_correct_binary, x, y, 14, shape_harr)
print(
'x is {},y is {},begin is {},end is {},white_counter is {},black_counter is {}'
.format(x, y, begin, end, white_counter, black_counter))
if begin != -1:
foreground_percent = float(white_counter) / float(
black_counter + white_counter)
if foreground_percent < 0.6 and foreground_percent > 0.2:
index = judegeCircleDirection(begin, end, len(pixel_list),
0.08)
# if index == -1:
# cv2.circle(img_to_correct,(y,x),16,(255,0,255))
# ##print('*************hhhhh',x,y)
if index != -1:
if index == 1:
##cv2.circle(img_to_correct,(y,x),16,(0,255,0))
if lu_list.count(position) == 0:
lu_list.append(position)
if index == 2:
##cv2.circle(img_to_correct,(y,x),16,(0,0,255))
if lb_list.count(position) == 0:
lb_list.append(position)
if index == 3:
##cv2.circle(img_to_correct,(y,x),16,(255,0,0))
if rb_list.count(position) == 0:
rb_list.append(position)
if index == 4:
##cv2.circle(img_to_correct,(y,x),16,(255,255,0))
if ru_list.count(position) == 0:
ru_list.append(position)
# lu_list.sort(key=compator_x)
# lu_list = list(set(lu_list))
shape = img_to_correct.shape
# canvas_point.append([1,2])
# canvas_point.append([100,30])
# canvas_point.append([75,26])
# canvas_point.append([94,94])
if len(lu_list) > 0:
# # #print('************aaaaaa',lu_list)
# result_list_x,need_y,p1 = choseFinalFornerDotList(lu_list,'x',0,0,10)
# if need_y :
# result_list_y,_,p1 = choseFinalFornerDotList(lu_list,'y',0,0,10)
# p1 = choseFinalFornerDot(result_list_x,result_list_y)
# if len(p1) == 2:
p1 = choseFinalFornerDot(lu_list, None)
cv2.circle(img_to_correct, (int(p1[1]), int(p1[0])), 8, (255, 255, 0))
canvas_point.append(p1)
if len(lb_list) > 0:
# result_list_x,need_y,p1 = choseFinalFornerDotList(lb_list,'x',shape[0],0,10)
print('************bbbb', lb_list)
# # ##print('************ddddddd',result_list_x)
# if need_y :
# result_list_y,_,p1 = choseFinalFornerDotList(lb_list,'y',shape[0],0,10)
# # ##print('************ddddddd',result_list_y)
# p1 = choseFinalFornerDot(result_list_x,result_list_y)
# # ##print('************ttttt',p1)
# if len(p1) == 2:
p1 = choseFinalFornerDot(lb_list, None)
cv2.circle(img_to_correct, (int(p1[1]), int(p1[0])), 8, (255, 0, 0))
canvas_point.append(p1)
if len(rb_list) > 0:
print('************cccccc', rb_list)
# result_list_x,need_y,p1 = choseFinalFornerDotList(rb_list,'x',shape[0],shape[1],10)
# if need_y :
# result_list_y,_,p1 = choseFinalFornerDotList(rb_list,'y',shape[0],shape[1],10)
# p1 = choseFinalFornerDot(result_list_x,result_list_y)
# if len(p1) == 2:
p1 = choseFinalFornerDot(rb_list, None)
cv2.circle(img_to_correct, (int(p1[1]), int(p1[0])), 8, (0, 255, 0))
canvas_point.append(p1)
if len(ru_list) > 0:
# # #print('************ddddddd',ru_list)
# result_list_x,need_y,p1 = choseFinalFornerDotList(ru_list,'x',0,shape[1],10)
# if need_y :
# result_list_y,_,p1 = choseFinalFornerDotList(ru_list,'y',0,shape[1],10)
# p1 = choseFinalFornerDot(result_list_x,result_list_y)
# if len(p1) == 2:
p1 = choseFinalFornerDot(ru_list, None)
cv2.circle(img_to_correct, (int(p1[1]), int(p1[0])), 8, (0, 0, 255))
canvas_point.append(p1)
showImg("canvas_point", img_to_correct)
# 如果 canvas_point 不等于4 不进行以下的操作
# #print("canvas_point",canvas_point)
if len(canvas_point) == 4:
#print('solution')
# showImg('img_to_correct',img_to_correct)
width_1 = math.sqrt((canvas_point[0][1] - canvas_point[3][1]) *
(canvas_point[0][1] - canvas_point[3][1]) +
(canvas_point[0][0] - canvas_point[3][0]) *
(canvas_point[0][0] - canvas_point[3][0]))
width_2 = math.sqrt((canvas_point[1][1] - canvas_point[2][1]) *
(canvas_point[1][1] - canvas_point[2][1]) +
(canvas_point[1][0] - canvas_point[2][0]) *
(canvas_point[1][0] - canvas_point[2][0]))
height_1 = math.sqrt((canvas_point[0][0] - canvas_point[1][0]) *
(canvas_point[0][0] - canvas_point[1][0]) +
(canvas_point[0][1] - canvas_point[1][1]) *
(canvas_point[0][1] - canvas_point[1][1]))
height_2 = math.sqrt((canvas_point[2][0] - canvas_point[3][0]) *
(canvas_point[2][0] - canvas_point[3][0]) +
(canvas_point[2][1] - canvas_point[3][1]) *
(canvas_point[2][1] - canvas_point[3][1]))
width = int(min(width_1, width_2))
height = int(
min(height_1, height_2)
) #math.sqrt((canvas_point[0][0]-canvas_point[1][0])*(canvas_point[0][0]-canvas_point[1][0]) +(canvas_point[0][1]-canvas_point[1][1])*(canvas_point[0][1]-canvas_point[1][1]))
#cv2.imencode('.jpg', img_to_correct)[1].tofile('Archive_to_split_2/harr'+str(flag_of_path)+'.jpg')#img
# dst = np.array([[0,0],[width ,0],[width,height],[0,height]])
dst = np.array([[0, 0], [height, 0], [height, width], [0, width]])
# dst = np.array([[0,0],[height ,0],[height,width],[0,width]])
AffineMatrix1 = cv2.getPerspectiveTransform(
np.array(np.float32(canvas_point)), np.array(np.float32(dst)))
# ##print(canvas_point)
# ##print(dst)
# ##print('ttttttttttAffineMatrix1',AffineMatrix1)
AffineMatrix, mask = cv2.findHomography(
np.float32(canvas_point).reshape(-1, 1, 2),
np.float32(dst).reshape(-1, 1, 2)) #, cv2.RANSAC,5.0)
# ##print('ttttttttttAffineMatrix',AffineMatrix)
# AffineImg = cv2.warpPerspective(img_to_correct, M, (width, height))
# Img = img_to_correct # cv2.imread('Archive_to_split_2/harr'+str(flag_of_path)+'.jpg')
a_begin = int(max(canvas_point[0][0], canvas_point[3][0]))
a_end = int(min(canvas_point[1][0], canvas_point[2][0]))
b_begin = int(max(canvas_point[0][1], canvas_point[1][1]))
b_end = int(min(canvas_point[2][1], canvas_point[3][1]))
# img_to_correct = img_to_correct[a_begin:a_end,b_begin:b_end]
AffineImg = cv2.warpPerspective(
img_to_correct, AffineMatrix,
(int(width), int(height)
)) # (img_to_correct.shape[1], img_to_correct.shape[0]))#
result_perspective = drawPictureUseCornerDotOri(
AffineImg, img_to_correct, AffineMatrix, width, height,
canvas_point)
# result_perspective = reShape(imageModel, result_perspective)
# ##print('contoursubcontoursubcontoursub',contoursub[0])
showImg('AffineImg1', result_perspective)
# ##print(min_rect)
#cv2.imencode('.jpg', result_perspective)[1].tofile('Archive_to_split_2/AffineImg'+str(flag_of_path)+'.jpg')#img
# to return
return result_perspective, 'perspective'
# else hough to correct image
else:
#print('use hough')
shape_of_hough = img_to_correct.shape
angle_result = Hough(img_to_correct)
shape = img_result.shape
matRotate = cv2.getRotationMatrix2D((shape[0] * 0.5, shape[1] * 0.5),
angle_result,
1) # 为方便旋转后图片能完整展示，所以我们将其缩小
img_result = cv2.warpAffine(img_result, matRotate,
(shape[1], shape[0]))
shape = img_result.shape
if abs(angle_result) >= 5.0:
img_result_1 = cv2.cvtColor(img_result, cv2.COLOR_RGB2GRAY)
rct, img_result_1 = cv2.threshold(
img_result_1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
contoursub, _ = cv2.findContours(img_result_1, cv2.RETR_CCOMP,
cv2.CHAIN_APPROX_SIMPLE)
contoursub.sort(key=takeAera, reverse=True)
if len(contoursub) > 0:
x, y, w, h = cv2.boundingRect(contoursub[0])
img_result = img_result[y:y + h, x:x + w]
return img_result, 'hough'
cv2.imshow("Detected corners", img)
max_p = corners.max(0, initial=0)
min_p = corners.min(0, initial=max(h, w))
new_corners = np.zeros((4, 2), dtype='uint16')
new_w = max_p[0][0] - min_p[0][0]
new_h = max_p[0][1] - min_p[0][0]
new_corners[0] = [0, 0]
new_corners[1] = [new_w, 0]
new_corners[2] = [0, new_h]
new_corners[3] = [new_w, new_h]
M, mask = cv2.findHomography(corners, new_corners)
n_img = cv2.warpPerspective(b_img, M, (new_w, new_h))
cv2.imshow('n IL', n_img)
lab = np.zeros((LAB_H, LAB_W), dtype=bool)
c_x = new_w / LAB_W
c_y = new_h / LAB_H
lab_v = np.zeros((V_LAB_H, V_LAB_W), dtype='uint8')
for y in range(LAB_H):
for x in range(LAB_W):
lab[y, x] = np.mean(n_img[int(y * c_y):int(y * c_y + c_y),
int(x * c_x):int(x * c_x + c_x)]) > THRESHOLD
def errorMarker(c):
H,_ = cv.findHomography(c, marker)
err = abs(htrans(H,c) - marker).sum()
return err, H
while (True):
ret, frame = cap.read()
if frame is not None:
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
res_corners, res_ids, _ = cv2.aruco.detectMarkers(
gray, aruco_dict, parameters=parameters)
# if we have markers
if res_ids is not None:
idx = which(refIds, res_ids)
# if our markers are int he reference image
if len(idx) > 0:
# take all markers and reshape the list of corners
these_res_corners = np.concatenate(res_corners, axis=1)
these_ref_corners = np.concatenate(
[refCorners[x] for x in idx], axis=1)
h, s = cv2.findHomography(these_ref_corners, these_res_corners)
if args.smooth:
h_array.append(h)
this_h = np.mean(h_array, axis=0)
else:
this_h = h
newRect = cv2.perspectiveTransform(
rect, this_h, (gray.shape[1], gray.shape[0]))
frame = cv2.polylines(frame, np.int32(newRect), True,
(0, 0, 0), 10)
if len(res_corners) > 0:
cv2.aruco.drawDetectedMarkers(frame, res_corners, res_ids)
# Display the resulting frame
if args.outputVideo is not None:
videoOut.write(frame)
work = mask.copy()
cv2.imshow(winName, mask)
cv2.waitKey(200)
# find the centroid of the dot and add it to c1
cnt = cv2.findContours(work, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
c2.append(np.array(cvk2.a2ti(getDot(cnt)[0]), dtype=np.float64))
##############
#PART B
#############
h**o = cv2.findHomography(np.array(c2), np.array(c1), method = cv.CV_RANSAC)
print h**o
print
# Construct a transformation matrix for the image that achieves
# the desired rotation and translation
M = np.eye(3,3, dtype='float32')
M2 = np.matrix(M) * h**o[0]
################ Gameplay ####################
########################################
def clearText(img, i=100, j=100):
# opencv use x,y and numpy uses y,x
img[j-50:j+120, 50:] = (0,0,0)
def updateGuesses(letters, guesses, img, i=100, j=100):
def match_frames(
f1,
f2,
nn_match_ratio=nn_match_ratio,
MIN_MATCH_COUNT = MIN_MATCH_COUNT,
inlier_threshold=inlier_threshold ) :
# brute force matcher
Matcher = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False) #cv2.BFMatcher(cv2.NORM_L1, crossCheck=False)
# matches
matches = Matcher.knnMatch(f1.des,f2.des,k=2)
# Lowe's ratio test
good = []
for m, n in matches:
if m.distance < nn_match_ratio * n.distance:
good.append(m)
if len(good) > MIN_MATCH_COUNT:
dst_pts = np.float32([f2.kp[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
src_pts = np.float32([f1.kp[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
homography, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5)
if np.any(homography)== None :
return None, None, None,None
else:
print(
"Not enouth match are found -%d/%d" % (len(good), MIN_MATCH_COUNT))
return None, None, None,None
# [homography check]
inliers1 = []
inliers2 = []
idx = {0:[], 1:[]}
good_matches = []
# Distance threshold to identify inliers with homography check
for i, m in enumerate(good):
col = np.ones((3,1), dtype=np.float64)
col[0:2,0] = f1.kp[m.queryIdx].pt
col = np.dot(homography, col)
col /= col[2,0]
dist = sqrt(
pow(col[0,0] - f2.kp[m.trainIdx].pt[0],2) + pow(col[1,0] - f2.kp[m.trainIdx].pt[1],2))
if dist < inlier_threshold:
good_matches.append(cv2.DMatch(len(inliers1), len(inliers2), 0))
inliers1.append(f1.kp[good[i].queryIdx])
idx[0].append(good[i].queryIdx)
inliers2.append(f2.kp[good[i].trainIdx])
idx[1].append(good[i].trainIdx)
return inliers1, idx[0], inliers2, idx[1]
# other (i.e. Lowe's ratio test)
if len(match) == 2 and match[0].distance < match[1].distance * ratio:
matches.append((match[0].trainIdx, match[0].queryIdx))
homography = None
status = None
print("Number of matches ", len(matches))
if len(matches) > 4:
print("More than 4 matches")
# construct the two sets of points
img1_points = np.float32([img1_kps[i] for (_, i) in matches])
img2_points = np.float32([img2_kps[i] for (i, _) in matches])
# compute the homography between the two sets of points
homography, status = cv2.findHomography(img1_points, img2_points,
cv2.RANSAC, reprojThresh)
print(homography)
#print(status)
#result = cv2.warpPerspective(img1, homography, (img1.shape[1] + img2.shape[1], img1.shape[0]))
#cv2.imwrite("result-p1.jpg", result)
#result[0:img2.shape[0], 0:img2.shape[1]] = img2
#cv2.imwrite("result-p2.jpg", result)
result = transform(homography, img1, img2).astype('uint8')
cv2.imshow("Base Image", result)
#output_image_array = transform(homography, img2, output_image_array)
#cv2.imshow("img2", output_image_array)
#vis = drawMatches(img1, img2, img1_points, img2_points, matches, status)
# mask2 = np.where((mask==2)|(mask==0),0,1).astype('uint8')
# frame = frame*mask2[:,:,np.newaxis]
kp2, des2 = orb.detectAndCompute(frame2, None)
#matches = bf.knnMatch(des1, des2, k=2)
matches = bf.match(des1, des2)
matches = sorted(matches, key=lambda x: x.distance)
if len(matches) > min_matches:
src_pts = np.float32([kp1[m.queryIdx].pt
for m in matches]).reshape(-1, 1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt
for m in matches]).reshape(-1, 1, 2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
h, w = template_gray.shape
pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1],
[w - 1, 0]]).reshape(-1, 1, 2)
# project corners into frame
dst = cv2.perspectiveTransform(pts, M)
# connect them with lines
img2 = cv2.polylines(frame, [np.int32(dst)], True, (0, 255, 0), 3,
cv2.LINE_AA)
# if M is not None:
# try:
# projection = projection_matrix(camera_parameters, M)
# # frame = render(frame, cont_ini["XYZ"], projection, template_gray, False)
diff = coor_ori - coor_ref
dis = np.sqrt(np.square(diff[:, 0]) + np.square(diff[:, 1]))
mask = dis < 0.005
matchesMask1 = mask * 1
# print(matchesMask1.sum())
n_matched = matchesMask1.sum()
print(f"{n_matched} matched after distance filter!")
# mask = np.ones(len(src_pts_list), 'uint8')
MIN_MATCH_COUNT = 50
if n_matched > MIN_MATCH_COUNT:
# RANSAC OPENCV官方
src_pts = np.float32(src_pts_list).reshape(-1, 1, 2)
dst_pts = np.float32(dst_pts_list).reshape(-1, 1, 2)
M, mask = cv.findHomography(src_pts, dst_pts, cv.RANSAC, 1.5)
# M, mask = cv.findHomography(src_pts, dst_pts, cv.LMEDS)
mask = (mask[:, 0]).astype('uint8')
# mask = matchesMask1 * mask
matchesMask = (mask * 1).ravel().tolist()
# matchesMask = list(mask*1)
mask = np.bool8(mask)
print("{} matched left after ransac.".format(mask.sum()))
gcps = np.hstack((coor_ori[mask], coor_ref[mask]))
# print("Writing gcp files of ArcMap format.")
out_gcp = f'd:/test/gcp_ori_{idx}.txt'
np.savetxt(out_gcp, gcps, fmt='%10.6f', delimiter='\t')
