def getPerspKeypoints(rgbFile1, rgbFile2, HFile1, HFile2, model_file='models/d2_kinal_ipr.pth'):
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
model = D2Net(
model_file=model_file,
use_relu=True,
use_cuda=use_cuda
)
image1 = np.array(Image.open(rgbFile1))
image2 = np.array(Image.open(rgbFile2))
H1 = np.load(HFile1)
H2 = np.load(HFile2)
imgTop1 = getTopImg(image1, H1)
imgTop2 = getTopImg(image2, H2)
feat1 = extract(imgTop1, model, device)
feat2 = extract(imgTop2, model, device)
print("Features extracted.")
src_pts, dst_pts = cv2D2netMatching(imgTop1, imgTop2, feat1, feat2, matcher="BF")
# src_pts, dst_pts = siftMatching(imgTop1, imgTop2)
orgSrc, orgDst = orgKeypoints(src_pts, dst_pts, H1, H2)
drawOrg(image1, image2, orgSrc, orgDst)
return orgSrc, orgDst
def evaluate_RGB(model_path, dataset):
# Creating CNN model
model = D2Net(model_file=model_path, use_relu=True, use_cuda=False)
model = model.to(device)
n_matches = []
n_feats = []
t_err = {thr: 0 for thr in rng}
for examp in dataset:
igp1, igp2, pts1, pts2, H = examp['image1'], examp['image2'], examp[
'pos1'], examp['pos2'], np.array(examp['H'])
# predicting keypoints and descriptors using the d2-net model
# print(igp1.shape, igp2.shape)
with torch.no_grad():
if args.multiscale:
keypoints1, scores1, descriptors1 = process_multiscale(
igp1.to(device).unsqueeze(0), model)
keypoints2, scores2, descriptors2 = process_multiscale(
igp2.to(device).unsqueeze(0), model)
else:
keypoints1, scores1, descriptors1 = process_multiscale(
igp1.to(device).unsqueeze(0), model, scales=[1])
keypoints2, scores2, descriptors2 = process_multiscale(
igp2.to(device).unsqueeze(0), model, scales=[1])
n_feats.append(keypoints1.shape[0])
n_feats.append(keypoints2.shape[0])
# applying nearest neighbor to find matches
matches = mnn_matcher(
torch.from_numpy(descriptors1).to(device=device),
torch.from_numpy(descriptors2).to(device=device))
pos_a = keypoints1[matches[:, 0], :2]
pos_a_h = np.concatenate([pos_a, np.ones([matches.shape[0], 1])],
axis=1)
pos_b_proj_h = np.transpose(np.dot(H, np.transpose(pos_a_h)))
pos_b_proj = pos_b_proj_h[:, :2] / pos_b_proj_h[:, 2:]
pos_b = keypoints2[matches[:, 1], :2]
dist = np.sqrt(np.sum((pos_b - pos_b_proj)**2, axis=1))
n_matches.append(matches.shape[0])
if dist.shape[0] == 0:
dist = np.array([float("inf")])
for thr in rng:
t_err[thr] += np.mean(dist <= thr)
return t_err, np.array(n_feats), np.array(n_matches)
def getPerspKeypoints(rgbFile1,
rgbFile2,
HFile1,
HFile2,
model_file='models/d2_kinal_ipr.pth'):
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
model = D2Net(model_file=model_file, use_relu=True, use_cuda=use_cuda)
image1 = np.array(Image.open(rgbFile1).convert('L'))
image1 = image1[:, :, np.newaxis]
image1 = np.repeat(image1, 3, -1)
image2 = np.array(Image.open(rgbFile2).convert('L'))
image2 = image2[:, :, np.newaxis]
image2 = np.repeat(image2, 3, -1)
# image1 = np.array(Image.open(rgbFile1))
# image2 = np.array(Image.open(rgbFile2))
H1 = np.load(HFile1)
H2 = np.load(HFile2)
# im1 = cv2.imread(rgbFile1)
# im1 = np.array(cv2.cvtColor(np.array(im1), cv2.COLOR_BGR2RGB))
# img1 = cv2.warpPerspective(im1, H1, (800, 800))
# cv2.imshow("Image", img1)
# cv2.waitKey(0)
imgTop1 = getTopImg(image1, H1)
imgTop2 = getTopImg(image2, H2)
# exit(1)
feat1 = extract(imgTop1, model, device)
feat2 = extract(imgTop2, model, device)
# print("Features extracted.")
src_pts, dst_pts, topMatchImg = cv2D2netMatching(imgTop1,
imgTop2,
feat1,
feat2,
matcher="BF")
# src_pts, dst_pts, topMatchImg = siftMatching(imgTop1, imgTop2)
orgSrc, orgDst = orgKeypoints(src_pts * 2, dst_pts * 2, H1, H2)
# drawOrg(image1, image2, orgSrc, orgDst)
perpMatchImg = drawOrg(np.array(Image.open(rgbFile1)),
np.array(Image.open(rgbFile2)), orgSrc, orgDst)
return topMatchImg, perpMatchImg
def __init__(self,
model_file='models/d2_tf.pth',
max_edge=1600,
max_sum_edges=2800,
use_relu=False,
use_cuda=torch.cuda.is_available()):
# CUDA
self.device = torch.device("cuda:0" if use_cuda else "cpu")
# Creating CNN model
self.model = D2Net(model_file=model_file,
use_relu=use_relu,
use_cuda=use_cuda)
self.max_edge = max_edge
self.max_sum_edges = max_sum_edges
def __init__(self, train_path, nfeatures):
self.files = []
self.files += [train_path + f for f in os.listdir(train_path)]
self.nfeatures = nfeatures
self.sift = cv2.xfeatures2d.SIFT_create(nfeatures=self.nfeatures)
self.matcher = cv2.BFMatcher_create(cv2.NORM_L1, crossCheck=False)
# Creating CNN model
self.model = D2Net(
model_file='/home/udit/d2-net/checkpoints/checkpoint_road_more/d2.15.pth',
use_relu=True,
use_cuda=use_cuda
)
self.device = torch.device("cuda:0" if use_cuda else "cpu")
def __init__(
self,
use_relu=True, # remove ReLU after the dense feature extraction module
multiscale=False, # extract multiscale features (read the note above)
max_edge=1600, # maximum image size at network input
max_sum_edges=2800, # maximum sum of image sizes at network input
preprocessing='torch', # image preprocessing (caffe or torch)
do_cuda=True):
print('Using D2NetFeature2D')
self.lock = RLock()
self.model_base_path = config.cfg.root_folder + '/thirdparty/d2net/'
self.models_path = self.model_base_path + 'models/d2_ots.pth' # best performances obtained with 'd2_ots.pth'
self.use_relu = use_relu
self.multiscale = multiscale
self.max_edge = max_edge
self.max_sum_edges = max_sum_edges
self.preprocessing = preprocessing
self.pts = []
self.kps = []
self.des = []
self.frame = None
self.keypoint_size = 20 # just a representative size for visualization and in order to convert extracted points to cv2.KeyPoint
self.do_cuda = do_cuda & torch.cuda.is_available()
print('cuda:', self.do_cuda)
self.device = torch.device("cuda:0" if self.do_cuda else "cpu")
torch.set_grad_enabled(False)
print('==> Loading pre-trained network.')
# Creating CNN model
self.model = D2Net(model_file=self.models_path,
use_relu=use_relu,
use_cuda=do_cuda)
if self.do_cuda:
print('Extracting on GPU')
else:
print('Extracting on CPU')
print('==> Successfully loaded pre-trained network.')
def benchmark_features():
i_err = {thr: 0 for thr in rng}
v_err = {thr: 0 for thr in rng}
# Creating CNN model
model = D2Net(model_file="/home/wang/d2-net/models/d2_tf.pth",
use_relu=True,
use_cuda=use_cuda)
with torch.no_grad():
data_loader = get_dataloader("view")
for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)):
batch = parse_batch(sample_batched, device)
for thr in rng:
TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_d2net(
model, batch, 1, thr)
PreNN = TPNN / PNN
PreNNT = TPNNT / PNNT
PreNNDR = TPNNDR / PNNDR
meanms = (PreNN + PreNNT + PreNNDR) / 3
v_err[thr] += meanms
n_v = i_batch
with torch.no_grad():
data_loader = get_dataloader("illu")
for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)):
batch = parse_batch(sample_batched, device)
for thr in rng:
TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_d2net(
model, batch, 1, thr)
PreNN = TPNN / PNN
PreNNT = TPNNT / PNNT
PreNNDR = TPNNDR / PNNDR
meanms = (PreNN + PreNNT + PreNNDR) / 3
i_err[thr] += meanms
n_i = i_batch
return i_err, v_err, n_i, n_v
for i in range(keypoints_left.shape[1]):
im4 = cv2.line(im4, (int(keypoints_left[0, i]), int(keypoints_left[1, i])), (int(keypoints_right[0, i]) + image1.shape[1], int(keypoints_right[1, i])), (0, 255, 0), 1)
cv2.imshow("Image_lines", im4)
cv2.waitKey(0)
if __name__ == '__main__':
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
args = parser.parse_args()
model = D2Net(
model_file=args.model_file,
use_relu=args.use_relu,
use_cuda=use_cuda
)
# image1 = np.array(Image.open(args.imgs[0]))
# image2 = np.array(Image.open(args.imgs[1]))
# image1 = np.array(Image.open(args.imgs[0]).convert('L'))
# image2 = np.array(Image.open(args.imgs[1]).convert('L'))
image1 = np.array(Image.open(args.imgs[0]).convert('L').resize((400, 400)))
image1 = image1[:, :, np.newaxis]
image1 = np.repeat(image1, 3, -1)
image2 = np.array(Image.open(args.imgs[1]).convert('L').resize((400, 400)))
image2 = image2[:, :, np.newaxis]
image2 = np.repeat(image2, 3, -1)
return repeatable_, max(useful_, 1)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--k', default=1024, type=int) # topk
parser.add_argument('--data', default='e', type=str) # dataset
args = parser.parse_args()
# CUDA
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
# Creating CNN model
model = D2Net(model_file='/home/wang/d2-net/models/d2_tf.pth',
use_relu=True,
use_cuda=use_cuda)
random.seed(cfg.PROJ.SEED)
torch.manual_seed(cfg.PROJ.SEED)
np.random.seed(cfg.PROJ.SEED)
root_dir = '/home/wang/workspace/RFSLAM_offline/RFNET/data/'
csv_file = None
seq = None
a = None
if args.data == 'v':
csv_file = 'hpatch_view.csv'
root_dir += 'hpatch_v_sequence'
seq = 'view'
a = False
def main():
if not torch.cuda.is_available():
print(gct(), 'no gpu device available')
sys.exit(1)
print(gct(), 'Args = %s', args)
cfg = configparser.ConfigParser()
cfg.read('settings.conf')
if args.cpu:
device = torch.device('cpu')
use_cuda = False
else:
device = torch.device('cuda')
use_cuda = True
criterion = D2Loss_hpatches(scaling_steps=3)
criterion = criterion.to(device)
model = D2Net(model_file=args.model_path, use_cuda=use_cuda).to(device)
print(gct(), 'Param size = %fMB', count_parameters_in_MB(model))
if args.dataset[:4] == 'view':
csv_file = cfg['hpatches']['view_csv']
root_dir = cfg['hpatches'][f'{args.dataset}_root']
dataset = HPatchesDataset(
csv_file=csv_file,
root_dir=root_dir,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=json.loads(cfg['hpatches']['view_mean']),
std=json.loads(cfg['hpatches']['view_std']))
]),
test_flag=True)
print(gct(), 'Load test dataset.')
print(gct(), f'Root dir: {root_dir}. #Image pair: {len(dataset)}')
dataset_len = len(dataset)
split_idx = list(range(dataset_len))
part_pos = [
int(dataset_len * float(cfg['hpatches']['train_part_pos'])),
int(dataset_len * float(cfg['hpatches']['eval_part_pos']))
]
test_queue = DataLoader(
dataset,
batch_size=int(cfg['params']['train_bs']),
sampler=sampler.SubsetRandomSampler(
split_idx[part_pos[1]:]), # split_idx[part_pos[1]:]
shuffle=False,
pin_memory=True,
num_workers=2)
result_folder = 'results/'
if not os.path.exists(result_folder):
os.makedirs(result_folder)
test_acc = infer(test_queue, model, criterion, result_folder, device)
print('Test loss', test_acc)
print(args)
# calculate scaling step
if args.model_type == 'vgg16':
scaling_steps = 5 - args.truncated_blocks
elif args.model_type == 'res50':
scaling_steps = 6 - args.dilation_blocks - args.truncated_blocks
elif args.model_type == 'res101':
scaling_steps = 6 - args.dilation_blocks - args.truncated_blocks
# Creating CNN model
model = D2Net(model_file=args.model_file,
use_relu=args.use_relu,
use_cuda=use_cuda,
truncated_blocks=args.truncated_blocks,
model_type=args.model_type,
edge_threshold=args.edge_threshold,
dilation_blocks=args.dilation_blocks,
pspnetTest=args.pspnetTest)
print(model)
if use_cuda:
model = model.cuda()
# Process the file
with open(args.image_list_file, 'r') as f:
lines = f.readlines()
for line in tqdm(lines, total=len(lines)):
path = line.strip()
image2 = cv2.circle(image2, (int(keypoints_right[0, i]), int(keypoints_right[1, i])), 2, (0, 0, 255), 4)
im4 = cv2.hconcat([image1, image2])
for i in range(keypoints_left.shape[1]):
im4 = cv2.line(im4, (int(keypoints_left[0, i]), int(keypoints_left[1, i])), (int(keypoints_right[0, i]) + image1.shape[1], int(keypoints_right[1, i])), (0, 255, 0), 1)
cv2.imshow("Image_lines", im4)
cv2.waitKey(0)
if __name__ == '__main__':
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
args = parser.parse_args()
model = D2Net(
#config = {},
model_file=args.model_file,
use_relu=args.use_relu,
use_cuda=use_cuda
)
feat1 = extract(args.imgs[0], args, model, device)
feat2 = extract(args.imgs[1], args, model, device)
print("Features extracted.")
drawMatches(args.imgs[0], args.imgs[1], feat1, feat2)
#drawMatches2(args.imgs[0], args.imgs[1], feat1, feat2)
if __name__ == "__main__":
focalX, focalY, centerX, centerY, scalingFactor = readCamera(
args.camera_file)
rgb_name_src = os.path.basename(args.rgb1)
H_name_src = os.path.splitext(rgb_name_src)[0] + '.npy'
srcH = os.path.join(os.path.dirname(args.rgb1), H_name_src)
rgb_name_trg = os.path.basename(args.rgb2)
H_name_trg = os.path.splitext(rgb_name_trg)[0] + '.npy'
trgH = os.path.join(os.path.dirname(args.rgb2), H_name_trg)
use_cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if use_cuda else 'cpu')
model1 = D2Net(model_file=args.model_d2)
model1 = model1.to(device)
model2 = D2Net(model_file=args.model_rord)
model2 = model2.to(device)
if args.model_rord:
srcPts, trgPts = getPerspKeypoints(args.rgb1, args.rgb2, srcH, trgH,
model2, device)
elif args.model_d2:
srcPts, trgPts = getPerspKeypoints(args.rgb1, args.rgb2, srcH, trgH,
model1, device)
elif args.model_ens:
model1 = D2Net(model_file=model1_ens)
model1 = model1.to(device)
model2 = D2Net(model_file=model2_ens)
model2 = model2.to(device)
if (sId != -1 and tId != -1):
corr.append((sId, tId))
corr = np.asarray(corr)
return corr
if __name__ == "__main__":
focalX, focalY, centerX, centerY, scalingFactor = readCamera(
args.camera_file)
df_rgb = pd.read_csv(args.rgb_csv)
df_dep = pd.read_csv(args.depth_csv)
model1 = D2Net(model_file=args.model_d2).to(device)
model2 = D2Net(model_file=args.model_rord).to(device)
i = 0
for im_q, dep_q in zip(df_rgb['query'], df_dep['query']):
filter_list = []
for im_d, dep_d in zip(df_rgb.iteritems(), df_dep.iteritems()):
if im_d[0] == 'query':
continue
rgb_name_src = os.path.basename(im_q)
H_name_src = os.path.splitext(rgb_name_src)[0] + '.npy'
srcH = os.path.join(os.path.dirname(im_q), H_name_src)
rgb_name_trg = os.path.basename(im_d[1][1])
H_name_trg = os.path.splitext(rgb_name_trg)[0] + '.npy'
trgH = os.path.join(os.path.dirname(im_d[1][1]), H_name_trg)
def init_d2net():
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
# Argument parsing
parser = argparse.ArgumentParser(description='Feature extraction script')
# parser.add_argument(
# '--image_list_file', type=str, required=True,
# help='path to a file containing a list of images to process'
# )
parser.add_argument('--preprocessing',
type=str,
default='caffe',
help='image preprocessing (caffe or torch)')
parser.add_argument('--model_file',
type=str,
default='models/d2_tf.pth',
help='path to the full model')
parser.add_argument('--max_edge',
type=int,
default=1600,
help='maximum image size at network input')
parser.add_argument('--max_sum_edges',
type=int,
default=2800,
help='maximum sum of image sizes at network input')
parser.add_argument('--output_extension',
type=str,
default='.d2-net',
help='extension for the output')
parser.add_argument('--output_type',
type=str,
default='npz',
help='output file type (npz or mat)')
# parser.add_argument(
# '--multiscale', dest='multiscale', action='store_true',
# help='extract multiscale features'
# )
# parser.set_defaults(multiscale=False)
parser.add_argument(
'--no-relu',
dest='use_relu',
action='store_false',
help='remove ReLU after the dense feature extraction module')
parser.set_defaults(use_relu=True)
# parser.add_argument(
# '--match_thresh', type=int, default=9,
# help='min match_thresh pair'
# )
#
parser.add_argument('--img_path_left', type=str, help='img_path_left')
parser.add_argument('--img_right_folder',
type=str,
help='img_right_folder')
args = parser.parse_args()
args.multiscale = True
model = D2Net(model_file=args.model_file,
use_relu=args.use_relu,
use_cuda=use_cuda)
return model, args, device