def extract(image, model, device, multiscale=False, preprocessing='caffe'): resized_image = image fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] input_image = preprocess_image(resized_image, preprocessing=preprocessing) with torch.no_grad(): if multiscale: keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=device), model) else: keypoints, scores, descriptors = process_multiscale(torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device), model, scales=[1]) keypoints[:, 0] *= fact_i keypoints[:, 1] *= fact_j keypoints = keypoints[:, [1, 0, 2]] feat = {} feat['keypoints'] = keypoints feat['scores'] = scores feat['descriptors'] = descriptors return feat
def compute_kps_des(self, image): with self.lock: print('D2Net image shape:',image.shape) if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) # TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize. resized_image = image if max(resized_image.shape) > self.max_edge: resized_image = scipy.misc.imresize( resized_image, self.max_edge / max(resized_image.shape) ).astype('float') if sum(resized_image.shape[: 2]) > self.max_sum_edges: resized_image = scipy.misc.imresize( resized_image, self.max_sum_edges / sum(resized_image.shape[: 2]) ).astype('float') fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] print('scale factors: {}, {}'.format(fact_i,fact_j)) input_image = preprocess_image( resized_image, preprocessing=self.preprocessing ) with torch.no_grad(): if self.multiscale: self.pts, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=self.device ), self.model ) else: self.pts, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=self.device ), self.model, scales=[1] ) # Input image coordinates self.pts[:, 0] *= fact_i self.pts[:, 1] *= fact_j # i, j -> u, v self.pts = self.pts[:, [1, 0, 2]] #print('pts.shape: ', self.pts.shape) #print('pts:', self.pts) self.kps = convert_pts_to_keypoints(self.pts, scores, self.keypoint_size) self.des = descriptors return self.kps, self.des
def extract(image, args, model, device): # def extract(file, args, model, device): # image = imageio.imread(file) if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) resized_image = image if max(resized_image.shape) > args.max_edge: resized_image = scipy.misc.imresize( resized_image, args.max_edge / max(resized_image.shape) ).astype('float') if sum(resized_image.shape[: 2]) > args.max_sum_edges: resized_image = scipy.misc.imresize( resized_image, args.max_sum_edges / sum(resized_image.shape[: 2]) ).astype('float') fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] input_image = preprocess_image( resized_image, preprocessing=args.preprocessing ) with torch.no_grad(): if args.multiscale: keypoints, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device ), model ) else: keypoints, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device ), model, scales=[1] ) keypoints[:, 0] *= fact_i keypoints[:, 1] *= fact_j keypoints = keypoints[:, [1, 0, 2]] feat = {} feat['keypoints'] = keypoints feat['scores'] = scores feat['descriptors'] = descriptors return feat
def evaluate_RGB(model_path, dataset): # Creating CNN model model = D2Net(model_file=model_path, use_relu=True, use_cuda=False) model = model.to(device) n_matches = [] n_feats = [] t_err = {thr: 0 for thr in rng} for examp in dataset: igp1, igp2, pts1, pts2, H = examp['image1'], examp['image2'], examp[ 'pos1'], examp['pos2'], np.array(examp['H']) # predicting keypoints and descriptors using the d2-net model # print(igp1.shape, igp2.shape) with torch.no_grad(): if args.multiscale: keypoints1, scores1, descriptors1 = process_multiscale( igp1.to(device).unsqueeze(0), model) keypoints2, scores2, descriptors2 = process_multiscale( igp2.to(device).unsqueeze(0), model) else: keypoints1, scores1, descriptors1 = process_multiscale( igp1.to(device).unsqueeze(0), model, scales=[1]) keypoints2, scores2, descriptors2 = process_multiscale( igp2.to(device).unsqueeze(0), model, scales=[1]) n_feats.append(keypoints1.shape[0]) n_feats.append(keypoints2.shape[0]) # applying nearest neighbor to find matches matches = mnn_matcher( torch.from_numpy(descriptors1).to(device=device), torch.from_numpy(descriptors2).to(device=device)) pos_a = keypoints1[matches[:, 0], :2] pos_a_h = np.concatenate([pos_a, np.ones([matches.shape[0], 1])], axis=1) pos_b_proj_h = np.transpose(np.dot(H, np.transpose(pos_a_h))) pos_b_proj = pos_b_proj_h[:, :2] / pos_b_proj_h[:, 2:] pos_b = keypoints2[matches[:, 1], :2] dist = np.sqrt(np.sum((pos_b - pos_b_proj)**2, axis=1)) n_matches.append(matches.shape[0]) if dist.shape[0] == 0: dist = np.array([float("inf")]) for thr in rng: t_err[thr] += np.mean(dist <= thr) return t_err, np.array(n_feats), np.array(n_matches)
def cnn_feature_extract(image, scales=[.25, 0.50, 1.0], nfeatures=1000): if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) # TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize. resized_image = image if max(resized_image.shape) > max_edge: resized_image = scipy.misc.imresize( resized_image, max_edge / max(resized_image.shape)).astype('float') if sum(resized_image.shape[:2]) > max_sum_edges: resized_image = scipy.misc.imresize( resized_image, max_sum_edges / sum(resized_image.shape[:2])).astype('float') fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] # lib - utils input_image = preprocess_image(resized_image, preprocessing="torch") with torch.no_grad(): if multiscale: # lib - pyramid.py keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=device), model, scales) else: keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=device), model, scales) # Input image coordinates keypoints[:, 0] *= fact_i keypoints[:, 1] *= fact_j # i, j -> u, v keypoints = keypoints[:, [1, 0, 2]] if nfeatures != -1: #根据scores排序, scores 정렬하기 scores2 = np.array([scores]).T res = np.hstack((scores2, keypoints)) res = res[np.lexsort(-res[:, ::-1].T)] res = np.hstack((res, descriptors)) #取前几个 scores = res[0:nfeatures, 0].copy() keypoints = res[0:nfeatures, 1:4].copy() descriptors = res[0:nfeatures, 4:].copy() del res return keypoints, scores, descriptors
def extract(image, args, model, device): if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) input_image = preprocess_image( image, preprocessing=args.preprocessing ) with torch.no_grad(): keypoints, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device ), model, scales=[1] ) keypoints = keypoints[:, [1, 0, 2]] feat = {} feat['keypoints'] = keypoints feat['scores'] = scores feat['descriptors'] = descriptors return feat
def extract_features(image, args, device, model): # image = cv2.resize(image,(720,576)) #cv2.imwrite('im2.jpg',image) #t1=cv2.getTickCount() if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) # TODO: switch to PIL.Image due to deprecation of scipy.misc.imresize. resized_image = image if max(resized_image.shape) > args.max_edge: ratio = args.max_edge / max(resized_image.shape) h, w, ch = resized_image.shape resized_image = resize(resized_image, (int(h * ratio), int(w * ratio))) # ).astype('float') if sum(resized_image.shape[:2]) > args.max_sum_edges: ratio = args.max_sum_edges / sum(resized_image.shape[:2]) h, w, ch = resized_image.shape resized_image = resize(resized_image, (int(h * ratio), int(w * ratio))) # ).astype('float') () fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] input_image = preprocess_image(resized_image, preprocessing=args.preprocessing) with torch.no_grad(): if args.multiscale: keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=device), model) else: keypoints, scores, descriptors = process_multiscale(torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device), model, scales=[1]) # Input image coordinates keypoints[:, 0] *= fact_i keypoints[:, 1] *= fact_j # i, j -> u, v keypoints = keypoints[:, [1, 0, 2]] return keypoints, scores, descriptors
def _forward(self, data): image = data['image'] image = image.flip(1) # RGB -> BGR norm = image.new_tensor([103.939, 116.779, 123.68]) image = (image * 255 - norm.view(1, 3, 1, 1)) # caffe normalization if self.conf['multiscale']: keypoints, scores, descriptors = process_multiscale( image, self.net) else: keypoints, scores, descriptors = process_multiscale( image, self.net, scales=[1]) keypoints = keypoints[:, [1, 0]] # (x, y) and remove the scale return { 'keypoints': torch.from_numpy(keypoints)[None], 'scores': torch.from_numpy(scores)[None], 'descriptors': torch.from_numpy(descriptors.T)[None], }
def eval_d2net(model, parsed_batch, DES_THRSH, COO_THRSH): im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parsed_batch keypoints1, scores1, descriptors1 = process_multiscale( im1_data, model, # scales=[1] ) kp1, des1 = topk(keypoints1, scores1, descriptors1, 1024) keypoints2, scores2, descriptors2 = process_multiscale( im2_data, model, # scales=[1] ) kp2, des2 = topk(keypoints2, scores2, descriptors2, 1024) kp1 = torch.tensor(kp1, device=device) kp2 = torch.tensor(kp2, device=device) des1 = torch.tensor(des1, device=device) des2 = torch.tensor(des2, device=device) kp1w = ptCltoCr(kp1, homo12, clamp=False, maxh=im1_data.shape[2], maxw=im1_data.shape[3]) _, _, maxh, maxw = im2_data.size() visible = kp1w[:, 2].lt(maxw) * kp1w[:, 1].lt(maxh) TPNN, PNN = nearest_neighbor_match_score(des1, des2, kp1w, kp2, visible, COO_THRSH) TPNNT, PNNT = nearest_neighbor_threshold_match_score( des1, des2, kp1w, kp2, visible, DES_THRSH, COO_THRSH) TPNNDR, PNNDR = nearest_neighbor_distance_ratio_match_score( des1, des2, kp1w, kp2, visible, COO_THRSH) return TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR
def extractSingle(image, model, device): with torch.no_grad(): keypoints, scores, descriptors = process_multiscale( image.to(device).unsqueeze(0), model, scales=[1] ) keypoints = keypoints[:, [1, 0, 2]] feat = {} feat['keypoints'] = keypoints feat['scores'] = scores feat['descriptors'] = descriptors return feat
args.max_sum_edges / sum(resized_image.shape[: 2]) ).astype('float') fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] input_image = preprocess_image( resized_image, preprocessing=args.preprocessing ) with torch.no_grad(): if args.multiscale: keypoints, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device ), model ) else: keypoints, scores, descriptors = process_multiscale( torch.tensor( input_image[np.newaxis, :, :, :].astype(np.float32), device=device ), model, scales=[1] ) # Input image coordinates keypoints[:, 0] *= fact_i
ToTensor() ]), ), batch_size=1, shuffle=False, num_workers=0) useful_list = [] repeat_list = [] with torch.no_grad(): for i_batch, sample_batched in enumerate(data_loader, 1): im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch( sample_batched, device) # (angle, class_id, octave, pt, response, size) keypoints1, scores1, descriptors1 = process_multiscale( im1_data.repeat(1, 3, 1, 1), model, scales=[1]) keypoints2, scores2, descriptors2 = process_multiscale( im2_data.repeat(1, 3, 1, 1), model, scales=[1]) kp1c, _ = topk(keypoints1, scores1, args.k) kp2c, _ = topk(keypoints2, scores2, args.k) im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch_np( sample_batched, mean, std) repeatable, useful = caluseful(kp1c, kp2c, homo12, im2_data) useful_list.append(useful), repeat_list.append(repeatable) usefuls = np.array(useful_list) repeats = np.array(repeat_list) repeatability = repeats.sum() / usefuls.sum()
def infer(test_queue, model, criterion, result_folder, device, topk=512): avg_loss = AvgrageMeter() model.eval() for step, valid_sample in enumerate(tqdm(test_queue)): with torch.no_grad(): valid_sample = parse_batch_test(valid_sample, device) kp1, score1, des1 = process_multiscale(valid_sample['image1'], model, scales=[1]) kp2, score2, des2 = process_multiscale(valid_sample['image2'], model, scales=[1]) if len(score1) > topk: ids = arg_topK(score1, topk) kp1 = kp1[ids, :] des1 = des1[ids, :] # ids = arg_topK(score2, topk) kp2 = kp2[ids, :] des2 = des2[ids, :] # output = { # 'f1': des1.squeeze(0), # 'f2': des2.squeeze(0), # 's1': score1, # 's2': score2 # } # # loss = criterion(valid_sample, output) predict_label, nn_kp2 = nearest_neighbor_distance_ratio_match( des1, des2, kp2, 0.8) idx = predict_label.nonzero()[0] # mkp1 = kp1.index_select(dim=0, index=idx.long()) # predict match keypoints in I1 mkp1 = np.take(kp1, indices=idx, axis=0) # mkp2 = nn_kp2.index_select(dim=0, index=idx.long()) # predict match keypoints in I2 mkp2 = np.take(nn_kp2, indices=idx, axis=0) keypoints1 = list(map(to_cv2_kp, mkp1)) keypoints2 = list(map(to_cv2_kp, mkp2)) DMatch = list(map(to_cv2_dmatch, np.arange(0, len(keypoints1)))) print('Matches num:', len(DMatch), valid_sample['name1'], valid_sample['name2']) # matches1to2 Matches from the first image to the second one, which means that # keypoints1[i] has a corresponding point in keypoints2[matches[i]] . img1 = valid_sample['image1_raw'].squeeze( 0).cpu().numpy().transpose(1, 2, 0) img2 = valid_sample['image2_raw'].squeeze( 0).cpu().numpy().transpose(1, 2, 0) img1 = (img1 * 255).astype(np.uint8) img2 = (img2 * 255).astype(np.uint8) draw_params = dict( matchColor=(0, 255, 0), # draw matches in green color singlePointColor=None, flags=2) matches_img = cv2.drawMatches(img1, keypoints1, img2, keypoints2, DMatch, None, **draw_params) matches_img_path = os.path.join( result_folder, valid_sample['name1'][0] + '_' + valid_sample['name2'][0] + '.png') cv2.imwrite(matches_img_path, matches_img) # cv2.imshow('matches', matches_img) # cv2.waitKey() # avg_loss.update(loss.item()) # print(gct(), f'[{step+1 :03d} / {len(test_queue):03d}]', f'Eval loss: {loss.item():.03f}') # print(gct(), f'Eval size: {len(test_queue)}', f'Eval loss: {avg_loss.avg:.03f}') # return avg_loss.avg return 0
def getPerspKeypoints2(model1, model2, rgbFile1, rgbFile2, HFile1, HFile2, device): if HFile1 is None: igp1, img1 = read_and_process_image(rgbFile1, H=None) else: H1 = np.load(HFile1) igp1, img1 = read_and_process_image(rgbFile1, H=H1) c,h,w = igp1.shape if HFile2 is None: igp2, img2 = read_and_process_image(rgbFile2, H=None) else: H2 = np.load(HFile2) igp2, img2 = read_and_process_image(rgbFile2, H=H2) with torch.no_grad(): keypoints_a1, scores_a1, descriptors_a1 = process_multiscale( igp1.to(device).unsqueeze(0), model1, scales=[1] ) keypoints_a1 = keypoints_a1[:, [1, 0, 2]] keypoints_a2, scores_a2, descriptors_a2 = process_multiscale( igp1.to(device).unsqueeze(0), model2, scales=[1] ) keypoints_a2 = keypoints_a2[:, [1, 0, 2]] keypoints_b1, scores_b1, descriptors_b1 = process_multiscale( igp2.to(device).unsqueeze(0), model1, scales=[1] ) keypoints_b1 = keypoints_b1[:, [1, 0, 2]] keypoints_b2, scores_b2, descriptors_b2 = process_multiscale( igp2.to(device).unsqueeze(0), model2, scales=[1] ) keypoints_b2 = keypoints_b2[:, [1, 0, 2]] # calculating matches for both models matches1, dist_1 = mnn_matcher_scorer( torch.from_numpy(descriptors_a1).to(device=device), torch.from_numpy(descriptors_b1).to(device=device), # len(matches1) ) matches2, dist_2 = mnn_matcher_scorer( torch.from_numpy(descriptors_a2).to(device=device), torch.from_numpy(descriptors_b2).to(device=device), # len(matches1) ) full_matches = torch.cat([matches1, matches2]) full_dist = torch.cat([dist_1, dist_2]) assert len(full_dist)==(len(dist_1)+len(dist_2)), "something wrong" k_final = len(full_dist)//2 # k_final = len(full_dist) # k_final = max(len(dist_1), len(dist_2)) top_k_mask = torch.topk(full_dist, k=k_final)[1] first = [] second = [] for valid_id in top_k_mask: if valid_id<len(dist_1): first.append(valid_id) else: second.append(valid_id-len(dist_1)) # final_matches = full_matches[top_k_mask] matches1 = matches1[torch.tensor(first, device=device).long()].data.cpu().numpy() matches2 = matches2[torch.tensor(second, device=device).long()].data.cpu().numpy() pos_a1 = keypoints_a1[matches1[:, 0], : 2] pos_b1 = keypoints_b1[matches1[:, 1], : 2] pos_a2 = keypoints_a2[matches2[:, 0], : 2] pos_b2 = keypoints_b2[matches2[:, 1], : 2] pos_a = np.concatenate([pos_a1, pos_a2], 0) pos_b = np.concatenate([pos_b1, pos_b2], 0) # pos_a, pos_b, inliers = apply_ransac(pos_a, pos_b) H, inliers = pydegensac.findHomography(pos_a, pos_b, 8.0, 0.99, 10000) pos_a = pos_a[inliers] pos_b = pos_b[inliers] inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_a] inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_b] placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(len(pos_a))] image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2, inlier_keypoints_right, placeholder_matches, None, matchColor=[0, 255, 0]) image3 = cv2.cvtColor(image3, cv2.COLOR_BGR2RGB) # cv2.imshow('Matches', image3) # cv2.waitKey() orgSrc, orgDst = orgKeypoints(pos_a, pos_b, H1, H2) drawOrg(cv2.imread(rgbFile1), cv2.imread(rgbFile2), orgSrc, orgDst) return orgSrc, orgDst
def extract_features( self, image_list, only_path=True, #only path means that the path to the images is given opposed to image files are given preprocessing='caffe', output_extension='.d2-net', output_type='npz', multiscale=False, store_results=False): #print(args) if type(image_list) is not list: image_list = [image_list] # Process the file #for image in tqdm(image_list, total=len(image_list)): k, d, s = [], [], [] for image in image_list: if only_path: image = imageio.imread(image) if len(image.shape) == 2: image = image[:, :, np.newaxis] image = np.repeat(image, 3, -1) resized_image = self.__resize_image__(image) fact_i = image.shape[0] / resized_image.shape[0] fact_j = image.shape[1] / resized_image.shape[1] input_image = preprocess_image(resized_image, preprocessing=preprocessing) with torch.no_grad(): if multiscale: keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=self.device), self.model) else: keypoints, scores, descriptors = process_multiscale( torch.tensor(input_image[np.newaxis, :, :, :].astype( np.float32), device=self.device), self.model, scales=[1]) # Input image coordinates keypoints[:, 0] *= fact_i keypoints[:, 1] *= fact_j # i, j -> u, v keypoints = keypoints[:, [1, 0, 2]] if store_results: if output_type == 'npz': with open(path + output_extension, 'wb') as output_file: np.savez(output_file, keypoints=keypoints, scores=scores, descriptors=descriptors) elif output_type == 'mat': with open(path + output_extension, 'wb') as output_file: scipy.io.savemat( output_file, { 'keypoints': keypoints, 'scores': scores, 'descriptors': descriptors }) else: raise ValueError('Unknown output type.') else: k.append(keypoints) d.append(descriptors) s.append(scores) return k, d, s