def extract(image, model, device, multiscale=False, preprocessing='caffe'):
resized_image = image
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
input_image = preprocess_image(resized_image, preprocessing=preprocessing)
with torch.no_grad():
if multiscale:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=device), model)
else:
keypoints, scores, descriptors = process_multiscale(torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device),
model,
scales=[1])
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
keypoints = keypoints[:, [1, 0, 2]]
feat = {}
feat['keypoints'] = keypoints
feat['scores'] = scores
feat['descriptors'] = descriptors
return feat
def compute_kps_des(self, image):
with self.lock:
print('D2Net image shape:',image.shape)
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
# TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize.
resized_image = image
if max(resized_image.shape) > self.max_edge:
resized_image = scipy.misc.imresize(
resized_image,
self.max_edge / max(resized_image.shape)
).astype('float')
if sum(resized_image.shape[: 2]) > self.max_sum_edges:
resized_image = scipy.misc.imresize(
resized_image,
self.max_sum_edges / sum(resized_image.shape[: 2])
).astype('float')
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
print('scale factors: {}, {}'.format(fact_i,fact_j))
input_image = preprocess_image(
resized_image,
preprocessing=self.preprocessing
)
with torch.no_grad():
if self.multiscale:
self.pts, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=self.device
),
self.model
)
else:
self.pts, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=self.device
),
self.model,
scales=[1]
)
# Input image coordinates
self.pts[:, 0] *= fact_i
self.pts[:, 1] *= fact_j
# i, j -> u, v
self.pts = self.pts[:, [1, 0, 2]]
#print('pts.shape: ', self.pts.shape)
#print('pts:', self.pts)
self.kps = convert_pts_to_keypoints(self.pts, scores, self.keypoint_size)
self.des = descriptors
return self.kps, self.des
def extract(image, args, model, device):
# def extract(file, args, model, device):
# image = imageio.imread(file)
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
resized_image = image
if max(resized_image.shape) > args.max_edge:
resized_image = scipy.misc.imresize(
resized_image,
args.max_edge / max(resized_image.shape)
).astype('float')
if sum(resized_image.shape[: 2]) > args.max_sum_edges:
resized_image = scipy.misc.imresize(
resized_image,
args.max_sum_edges / sum(resized_image.shape[: 2])
).astype('float')
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
input_image = preprocess_image(
resized_image,
preprocessing=args.preprocessing
)
with torch.no_grad():
if args.multiscale:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device
),
model
)
else:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device
),
model,
scales=[1]
)
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
keypoints = keypoints[:, [1, 0, 2]]
feat = {}
feat['keypoints'] = keypoints
feat['scores'] = scores
feat['descriptors'] = descriptors
return feat
def evaluate_RGB(model_path, dataset):
# Creating CNN model
model = D2Net(model_file=model_path, use_relu=True, use_cuda=False)
model = model.to(device)
n_matches = []
n_feats = []
t_err = {thr: 0 for thr in rng}
for examp in dataset:
igp1, igp2, pts1, pts2, H = examp['image1'], examp['image2'], examp[
'pos1'], examp['pos2'], np.array(examp['H'])
# predicting keypoints and descriptors using the d2-net model
# print(igp1.shape, igp2.shape)
with torch.no_grad():
if args.multiscale:
keypoints1, scores1, descriptors1 = process_multiscale(
igp1.to(device).unsqueeze(0), model)
keypoints2, scores2, descriptors2 = process_multiscale(
igp2.to(device).unsqueeze(0), model)
else:
keypoints1, scores1, descriptors1 = process_multiscale(
igp1.to(device).unsqueeze(0), model, scales=[1])
keypoints2, scores2, descriptors2 = process_multiscale(
igp2.to(device).unsqueeze(0), model, scales=[1])
n_feats.append(keypoints1.shape[0])
n_feats.append(keypoints2.shape[0])
# applying nearest neighbor to find matches
matches = mnn_matcher(
torch.from_numpy(descriptors1).to(device=device),
torch.from_numpy(descriptors2).to(device=device))
pos_a = keypoints1[matches[:, 0], :2]
pos_a_h = np.concatenate([pos_a, np.ones([matches.shape[0], 1])],
axis=1)
pos_b_proj_h = np.transpose(np.dot(H, np.transpose(pos_a_h)))
pos_b_proj = pos_b_proj_h[:, :2] / pos_b_proj_h[:, 2:]
pos_b = keypoints2[matches[:, 1], :2]
dist = np.sqrt(np.sum((pos_b - pos_b_proj)**2, axis=1))
n_matches.append(matches.shape[0])
if dist.shape[0] == 0:
dist = np.array([float("inf")])
for thr in rng:
t_err[thr] += np.mean(dist <= thr)
return t_err, np.array(n_feats), np.array(n_matches)
def cnn_feature_extract(image, scales=[.25, 0.50, 1.0], nfeatures=1000):
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
# TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize.
resized_image = image
if max(resized_image.shape) > max_edge:
resized_image = scipy.misc.imresize(
resized_image, max_edge / max(resized_image.shape)).astype('float')
if sum(resized_image.shape[:2]) > max_sum_edges:
resized_image = scipy.misc.imresize(
resized_image,
max_sum_edges / sum(resized_image.shape[:2])).astype('float')
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
# lib - utils
input_image = preprocess_image(resized_image, preprocessing="torch")
with torch.no_grad():
if multiscale:
# lib - pyramid.py
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=device), model, scales)
else:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=device), model, scales)
# Input image coordinates
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
# i, j -> u, v
keypoints = keypoints[:, [1, 0, 2]]
if nfeatures != -1:
#根据scores排序, scores 정렬하기
scores2 = np.array([scores]).T
res = np.hstack((scores2, keypoints))
res = res[np.lexsort(-res[:, ::-1].T)]
res = np.hstack((res, descriptors))
#取前几个
scores = res[0:nfeatures, 0].copy()
keypoints = res[0:nfeatures, 1:4].copy()
descriptors = res[0:nfeatures, 4:].copy()
del res
return keypoints, scores, descriptors
def extract(image, args, model, device):
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
input_image = preprocess_image(
image,
preprocessing=args.preprocessing
)
with torch.no_grad():
keypoints, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device
),
model,
scales=[1]
)
keypoints = keypoints[:, [1, 0, 2]]
feat = {}
feat['keypoints'] = keypoints
feat['scores'] = scores
feat['descriptors'] = descriptors
return feat
def extract_features(image, args, device, model):
# image = cv2.resize(image,(720,576))
#cv2.imwrite('im2.jpg',image)
#t1=cv2.getTickCount()
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
# TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize.
resized_image = image
if max(resized_image.shape) > args.max_edge:
ratio = args.max_edge / max(resized_image.shape)
h, w, ch = resized_image.shape
resized_image = resize(resized_image, (int(h * ratio), int(w * ratio)))
# ).astype('float')
if sum(resized_image.shape[:2]) > args.max_sum_edges:
ratio = args.max_sum_edges / sum(resized_image.shape[:2])
h, w, ch = resized_image.shape
resized_image = resize(resized_image, (int(h * ratio), int(w * ratio)))
# ).astype('float')
()
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
input_image = preprocess_image(resized_image,
preprocessing=args.preprocessing)
with torch.no_grad():
if args.multiscale:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=device), model)
else:
keypoints, scores, descriptors = process_multiscale(torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device),
model,
scales=[1])
# Input image coordinates
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
# i, j -> u, v
keypoints = keypoints[:, [1, 0, 2]]
return keypoints, scores, descriptors
def _forward(self, data):
image = data['image']
image = image.flip(1) # RGB -> BGR
norm = image.new_tensor([103.939, 116.779, 123.68])
image = (image * 255 - norm.view(1, 3, 1, 1)) # caffe normalization
if self.conf['multiscale']:
keypoints, scores, descriptors = process_multiscale(
image, self.net)
else:
keypoints, scores, descriptors = process_multiscale(
image, self.net, scales=[1])
keypoints = keypoints[:, [1, 0]] # (x, y) and remove the scale
return {
'keypoints': torch.from_numpy(keypoints)[None],
'scores': torch.from_numpy(scores)[None],
'descriptors': torch.from_numpy(descriptors.T)[None],
}
def eval_d2net(model, parsed_batch, DES_THRSH, COO_THRSH):
im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parsed_batch
keypoints1, scores1, descriptors1 = process_multiscale(
im1_data,
model,
# scales=[1]
)
kp1, des1 = topk(keypoints1, scores1, descriptors1, 1024)
keypoints2, scores2, descriptors2 = process_multiscale(
im2_data,
model,
# scales=[1]
)
kp2, des2 = topk(keypoints2, scores2, descriptors2, 1024)
kp1 = torch.tensor(kp1, device=device)
kp2 = torch.tensor(kp2, device=device)
des1 = torch.tensor(des1, device=device)
des2 = torch.tensor(des2, device=device)
kp1w = ptCltoCr(kp1,
homo12,
clamp=False,
maxh=im1_data.shape[2],
maxw=im1_data.shape[3])
_, _, maxh, maxw = im2_data.size()
visible = kp1w[:, 2].lt(maxw) * kp1w[:, 1].lt(maxh)
TPNN, PNN = nearest_neighbor_match_score(des1, des2, kp1w, kp2, visible,
COO_THRSH)
TPNNT, PNNT = nearest_neighbor_threshold_match_score(
des1, des2, kp1w, kp2, visible, DES_THRSH, COO_THRSH)
TPNNDR, PNNDR = nearest_neighbor_distance_ratio_match_score(
des1, des2, kp1w, kp2, visible, COO_THRSH)
return TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR
def extractSingle(image, model, device):
with torch.no_grad():
keypoints, scores, descriptors = process_multiscale(
image.to(device).unsqueeze(0),
model,
scales=[1]
)
keypoints = keypoints[:, [1, 0, 2]]
feat = {}
feat['keypoints'] = keypoints
feat['scores'] = scores
feat['descriptors'] = descriptors
return feat
args.max_sum_edges / sum(resized_image.shape[: 2])
).astype('float')
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
input_image = preprocess_image(
resized_image,
preprocessing=args.preprocessing
)
with torch.no_grad():
if args.multiscale:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device
),
model
)
else:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(
input_image[np.newaxis, :, :, :].astype(np.float32),
device=device
),
model,
scales=[1]
)
# Input image coordinates
keypoints[:, 0] *= fact_i
ToTensor()
]),
),
batch_size=1,
shuffle=False,
num_workers=0)
useful_list = []
repeat_list = []
with torch.no_grad():
for i_batch, sample_batched in enumerate(data_loader, 1):
im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch(
sample_batched, device)
# (angle, class_id, octave, pt, response, size)
keypoints1, scores1, descriptors1 = process_multiscale(
im1_data.repeat(1, 3, 1, 1), model, scales=[1])
keypoints2, scores2, descriptors2 = process_multiscale(
im2_data.repeat(1, 3, 1, 1), model, scales=[1])
kp1c, _ = topk(keypoints1, scores1, args.k)
kp2c, _ = topk(keypoints2, scores2, args.k)
im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch_np(
sample_batched, mean, std)
repeatable, useful = caluseful(kp1c, kp2c, homo12, im2_data)
useful_list.append(useful), repeat_list.append(repeatable)
usefuls = np.array(useful_list)
repeats = np.array(repeat_list)
repeatability = repeats.sum() / usefuls.sum()
def infer(test_queue, model, criterion, result_folder, device, topk=512):
avg_loss = AvgrageMeter()
model.eval()
for step, valid_sample in enumerate(tqdm(test_queue)):
with torch.no_grad():
valid_sample = parse_batch_test(valid_sample, device)
kp1, score1, des1 = process_multiscale(valid_sample['image1'],
model,
scales=[1])
kp2, score2, des2 = process_multiscale(valid_sample['image2'],
model,
scales=[1])
if len(score1) > topk:
ids = arg_topK(score1, topk)
kp1 = kp1[ids, :]
des1 = des1[ids, :]
#
ids = arg_topK(score2, topk)
kp2 = kp2[ids, :]
des2 = des2[ids, :]
# output = {
# 'f1': des1.squeeze(0),
# 'f2': des2.squeeze(0),
# 's1': score1,
# 's2': score2
# }
#
# loss = criterion(valid_sample, output)
predict_label, nn_kp2 = nearest_neighbor_distance_ratio_match(
des1, des2, kp2, 0.8)
idx = predict_label.nonzero()[0]
# mkp1 = kp1.index_select(dim=0, index=idx.long()) # predict match keypoints in I1
mkp1 = np.take(kp1, indices=idx, axis=0)
# mkp2 = nn_kp2.index_select(dim=0, index=idx.long()) # predict match keypoints in I2
mkp2 = np.take(nn_kp2, indices=idx, axis=0)
keypoints1 = list(map(to_cv2_kp, mkp1))
keypoints2 = list(map(to_cv2_kp, mkp2))
DMatch = list(map(to_cv2_dmatch, np.arange(0, len(keypoints1))))
print('Matches num:', len(DMatch), valid_sample['name1'],
valid_sample['name2'])
# matches1to2 Matches from the first image to the second one, which means that
# keypoints1[i] has a corresponding point in keypoints2[matches[i]] .
img1 = valid_sample['image1_raw'].squeeze(
0).cpu().numpy().transpose(1, 2, 0)
img2 = valid_sample['image2_raw'].squeeze(
0).cpu().numpy().transpose(1, 2, 0)
img1 = (img1 * 255).astype(np.uint8)
img2 = (img2 * 255).astype(np.uint8)
draw_params = dict(
matchColor=(0, 255, 0), # draw matches in green color
singlePointColor=None,
flags=2)
matches_img = cv2.drawMatches(img1, keypoints1, img2, keypoints2,
DMatch, None, **draw_params)
matches_img_path = os.path.join(
result_folder, valid_sample['name1'][0] + '_' +
valid_sample['name2'][0] + '.png')
cv2.imwrite(matches_img_path, matches_img)
# cv2.imshow('matches', matches_img)
# cv2.waitKey()
# avg_loss.update(loss.item())
# print(gct(), f'[{step+1 :03d} / {len(test_queue):03d}]', f'Eval loss: {loss.item():.03f}')
# print(gct(), f'Eval size: {len(test_queue)}', f'Eval loss: {avg_loss.avg:.03f}')
# return avg_loss.avg
return 0
def getPerspKeypoints2(model1, model2, rgbFile1, rgbFile2, HFile1, HFile2, device):
if HFile1 is None:
igp1, img1 = read_and_process_image(rgbFile1, H=None)
else:
H1 = np.load(HFile1)
igp1, img1 = read_and_process_image(rgbFile1, H=H1)
c,h,w = igp1.shape
if HFile2 is None:
igp2, img2 = read_and_process_image(rgbFile2, H=None)
else:
H2 = np.load(HFile2)
igp2, img2 = read_and_process_image(rgbFile2, H=H2)
with torch.no_grad():
keypoints_a1, scores_a1, descriptors_a1 = process_multiscale(
igp1.to(device).unsqueeze(0),
model1,
scales=[1]
)
keypoints_a1 = keypoints_a1[:, [1, 0, 2]]
keypoints_a2, scores_a2, descriptors_a2 = process_multiscale(
igp1.to(device).unsqueeze(0),
model2,
scales=[1]
)
keypoints_a2 = keypoints_a2[:, [1, 0, 2]]
keypoints_b1, scores_b1, descriptors_b1 = process_multiscale(
igp2.to(device).unsqueeze(0),
model1,
scales=[1]
)
keypoints_b1 = keypoints_b1[:, [1, 0, 2]]
keypoints_b2, scores_b2, descriptors_b2 = process_multiscale(
igp2.to(device).unsqueeze(0),
model2,
scales=[1]
)
keypoints_b2 = keypoints_b2[:, [1, 0, 2]]
# calculating matches for both models
matches1, dist_1 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a1).to(device=device),
torch.from_numpy(descriptors_b1).to(device=device),
# len(matches1)
)
matches2, dist_2 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a2).to(device=device),
torch.from_numpy(descriptors_b2).to(device=device),
# len(matches1)
)
full_matches = torch.cat([matches1, matches2])
full_dist = torch.cat([dist_1, dist_2])
assert len(full_dist)==(len(dist_1)+len(dist_2)), "something wrong"
k_final = len(full_dist)//2
# k_final = len(full_dist)
# k_final = max(len(dist_1), len(dist_2))
top_k_mask = torch.topk(full_dist, k=k_final)[1]
first = []
second = []
for valid_id in top_k_mask:
if valid_id<len(dist_1):
first.append(valid_id)
else:
second.append(valid_id-len(dist_1))
# final_matches = full_matches[top_k_mask]
matches1 = matches1[torch.tensor(first, device=device).long()].data.cpu().numpy()
matches2 = matches2[torch.tensor(second, device=device).long()].data.cpu().numpy()
pos_a1 = keypoints_a1[matches1[:, 0], : 2]
pos_b1 = keypoints_b1[matches1[:, 1], : 2]
pos_a2 = keypoints_a2[matches2[:, 0], : 2]
pos_b2 = keypoints_b2[matches2[:, 1], : 2]
pos_a = np.concatenate([pos_a1, pos_a2], 0)
pos_b = np.concatenate([pos_b1, pos_b2], 0)
# pos_a, pos_b, inliers = apply_ransac(pos_a, pos_b)
H, inliers = pydegensac.findHomography(pos_a, pos_b, 8.0, 0.99, 10000)
pos_a = pos_a[inliers]
pos_b = pos_b[inliers]
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_a]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_b]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(len(pos_a))]
image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2, inlier_keypoints_right, placeholder_matches, None, matchColor=[0, 255, 0])
image3 = cv2.cvtColor(image3, cv2.COLOR_BGR2RGB)
# cv2.imshow('Matches', image3)
# cv2.waitKey()
orgSrc, orgDst = orgKeypoints(pos_a, pos_b, H1, H2)
drawOrg(cv2.imread(rgbFile1), cv2.imread(rgbFile2), orgSrc, orgDst)
return orgSrc, orgDst
def extract_features(
self,
image_list,
only_path=True, #only path means that the path to the images is given opposed to image files are given
preprocessing='caffe',
output_extension='.d2-net',
output_type='npz',
multiscale=False,
store_results=False):
#print(args)
if type(image_list) is not list:
image_list = [image_list]
# Process the file
#for image in tqdm(image_list, total=len(image_list)):
k, d, s = [], [], []
for image in image_list:
if only_path:
image = imageio.imread(image)
if len(image.shape) == 2:
image = image[:, :, np.newaxis]
image = np.repeat(image, 3, -1)
resized_image = self.__resize_image__(image)
fact_i = image.shape[0] / resized_image.shape[0]
fact_j = image.shape[1] / resized_image.shape[1]
input_image = preprocess_image(resized_image,
preprocessing=preprocessing)
with torch.no_grad():
if multiscale:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=self.device), self.model)
else:
keypoints, scores, descriptors = process_multiscale(
torch.tensor(input_image[np.newaxis, :, :, :].astype(
np.float32),
device=self.device),
self.model,
scales=[1])
# Input image coordinates
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
# i, j -> u, v
keypoints = keypoints[:, [1, 0, 2]]
if store_results:
if output_type == 'npz':
with open(path + output_extension, 'wb') as output_file:
np.savez(output_file,
keypoints=keypoints,
scores=scores,
descriptors=descriptors)
elif output_type == 'mat':
with open(path + output_extension, 'wb') as output_file:
scipy.io.savemat(
output_file, {
'keypoints': keypoints,
'scores': scores,
'descriptors': descriptors
})
else:
raise ValueError('Unknown output type.')
else:
k.append(keypoints)
d.append(descriptors)
s.append(scores)
return k, d, s
