def sc_ringkeys_auc():
struct_filename = '/media/admini/LENOVO/dataset/kitti/lidar_odometry/birdview_dataset/tmp/struct_file_05_ds.txt'
images_info = make_images_info(struct_filename=struct_filename)
print(len(images_info))
positions = np.array(
[image_info['position'] for image_info in images_info])
print(positions.shape)
data_file = '/home/admini/yanhao/large-scale-pointcloud-matching/pnv-kitti05.npy'
descriptors = np.load(data_file)
print(descriptors.shape)
descriptors = torch.Tensor(descriptors)
positions = torch.Tensor(positions)
diff = descriptors[..., None] - descriptors.transpose(0, 1)[None, ...]
score_matrix = (1 - torch.einsum('mdn,mdn->mn', diff, diff)).numpy()
diff = positions[..., None] - positions.transpose(0, 1)[None, ...]
label_matrix = (torch.einsum('mdn,mdn->mn', diff, diff) <
(args.positive_search_radius**2)).numpy()
print('AUC:',
roc_auc_score(label_matrix.reshape(-1), score_matrix.reshape(-1)))
pass
def top_k_m2dp():
k = 25
positive_radius = 3
struct_filename = '/media/admini/LENOVO/dataset/kitti/lidar_odometry/birdview_dataset/tmp/struct_file_one_scan_02.txt'
images_info = make_images_info(struct_filename=struct_filename)
print(len(images_info))
positions = np.array(
[image_info['position'] for image_info in images_info])
print(positions.shape)
data_file = '/home/admini/yanhao/large-scale-pointcloud-matching/m2dp-kitti02.mat'
descriptors = scio.loadmat(data_file)['descriptors'][:-1]
print(descriptors.shape)
descriptors = np.asarray(descriptors, order='C').astype('float32')
database_descriptors, query_descriptors, database_positions, query_positions = \
train_test_split(descriptors, positions, test_size=0.4, random_state=10)
index = faiss.IndexFlatL2(database_descriptors.shape[-1])
index.add(database_descriptors)
topk_score_overall = np.zeros(k)
for descriptor, position in zip(query_descriptors, query_positions):
distances, indices = index.search(descriptor.reshape(1, -1), k)
candidate_positions = database_positions[indices[0]]
diff = candidate_positions - position
# print((diff * diff).sum(axis=1))
# print(diff)
is_true_result = (diff * diff).sum(axis=1) < positive_radius**2
topk_score = is_true_result.cumsum() > 0
# print(topk_score)
topk_score_overall += topk_score
topk_score_overall /= len(query_descriptors)
print(topk_score_overall)
print(database_descriptors.shape)
print(query_descriptors.shape)
print(database_positions.shape)
print(query_positions.shape)
pass
def main():
images_info_validate = make_images_info(struct_filename=os.path.join(
args.dataset_dir, 'struct_file_' + args.sequence_validate + '.txt'))
validate_images_dir = os.path.join(args.dataset_dir,
args.sequence_validate)
validate_dataset = SuperglueDataset(
images_info=images_info_validate,
images_dir=validate_images_dir,
positive_search_radius=args.positive_search_radius,
meters_per_pixel=args.meters_per_pixel)
validate_data_loader = DataLoader(validate_dataset,
batch_size=1,
shuffle=True)
sift = cv2.SIFT_create(nfeatures=191,
contrastThreshold=0.002,
edgeThreshold=15,
sigma=1.2)
orb = cv2.ORB_create(nfeatures=args.number_of_features)
validate_detector(orb, validate_data_loader)
pass
def verify():
"""
This function verify if the keypoints in from superpoint+superglue are correctly labelled by ground truth relative pose
"""
images_dir = os.path.join(args.dataset_dir, args.sequence)
images_info = make_images_info(
struct_filename=os.path.join(args.dataset_dir, 'struct_file_' +
args.sequence + '.txt'))
dataset = SuperglueDataset(
images_info=images_info,
images_dir=images_dir,
positive_search_radius=args.positive_search_radius,
meters_per_pixel=args.meters_per_pixel)
data_loader = DataLoader(dataset, batch_size=1, shuffle=True)
saved_model_file = os.path.join(args.saved_model_path,
'superglue-lidar-birdview.pth.tar')
config = {
'superpoint': {
'nms_radius': 4,
'keypoint_threshold': 0.005,
'max_keypoints': 200,
},
'Superglue': {
'weights': 'outdoor',
'sinkhorn_iterations': 100,
'match_threshold': 0.2,
}
}
model = Matching(config)
model_checkpoint = torch.load(saved_model_file,
map_location=lambda storage, loc: storage)
model.load_state_dict(model_checkpoint)
print("Loaded model checkpoints from \'{}\'.".format(saved_model_file))
device = torch.device(
'cuda' if torch.cuda.is_available() and args.use_gpu else 'cpu')
model.to(device)
torch.set_grad_enabled(False)
for target, source, T_target_source in data_loader:
# iteration += 1
assert (source.shape == target.shape)
B, C, W, H = source.shape
target = target.to(device)
source = source.to(device)
pred = model({'image0': target, 'image1': source})
target_kpts = pred['keypoints0'][0].cpu()
source_kpts = pred['keypoints1'][0].cpu()
if len(target_kpts) == 0 or len(source_kpts) == 0:
continue
# in superglue/numpy/tensor the coordinates are (i,j) which correspond to (v,u) in PIL Image/opencv
target_kpts_in_meters = target_kpts * args.meters_per_pixel - 50
source_kpts_in_meters = source_kpts * args.meters_per_pixel - 50
match_mask_ground_truth = make_ground_truth_matrix(
target_kpts_in_meters, source_kpts_in_meters, T_target_source[0],
args.tolerance_in_meters)
target_image_raw = target[0][0].cpu().numpy()
source_image_raw = source[0][0].cpu().numpy()
target_image_raw = np.stack([target_image_raw] * 3, -1) * 30
source_image_raw = np.stack([source_image_raw] * 3, -1) * 30
cv2.imshow('target_image_raw', target_image_raw)
# target_kpts = np.round(target_kpts.numpy()).astype(int)
T_target_source = T_target_source[0].numpy()
source_kpts = source_kpts.numpy()
source_kpts_in_meters = pts_from_pixel_to_meter(
source_kpts, args.meters_per_pixel)
print('T_target_source:\n', T_target_source)
source_kpts_in_meters_in_target_img = [
(T_target_source[:3, :3] @ np.array(
[source_kpt[0], source_kpt[1], 0]) +
T_target_source[:3, 3])[:2]
for source_kpt in source_kpts_in_meters
]
source_kpts_in_meters_in_target_img = np.array(
source_kpts_in_meters_in_target_img)
source_kpts_in_target_img = pts_from_meter_to_pixel(
source_kpts_in_meters_in_target_img, args.meters_per_pixel)
source_kpts = np.round(source_kpts).astype(int)
source_kpts_in_target_img = np.round(source_kpts_in_target_img).astype(
int)
target_image_poi = target_image_raw.copy()
source_image_poi = source_image_raw.copy()
for (x0, y0), (x1, y1) in zip(source_kpts, source_kpts_in_target_img):
# c = c.tolist()
# cv2.line(target_image, (x0, y0), (x0 + 50, y0 + 50),
# color=[255,0,0], thickness=1, lineType=cv2.LINE_AA)
# display line end-points as circles
cv2.circle(target_image_poi, (x1, y1),
2, (0, 255, 0),
1,
lineType=cv2.LINE_AA)
cv2.circle(source_image_poi, (x0, y0),
2, (255, 0, 0),
1,
lineType=cv2.LINE_AA)
# cv2.circle(out, (x1 + margin + W0, y1), 2, c, -1,
# lineType=cv2.LINE_AA)
cv2.imshow('target_image', target_image_poi)
cv2.imshow('source_image', source_image_poi)
cv2.waitKey(0)
torch.set_grad_enabled(True)
pass
def visualize_matching_all():
"""
This function visualize the feature point matching pipeline
"""
images_dir = os.path.join(args.dataset_dir, args.sequence)
images_info = make_images_info(
struct_filename=os.path.join(args.dataset_dir, 'struct_file_' +
args.sequence + '.txt'))
dataset = SuperglueDataset(
images_info=images_info,
images_dir=images_dir,
positive_search_radius=args.positive_search_radius,
meters_per_pixel=args.meters_per_pixel,
return_filename=True)
data_loader = DataLoader(dataset, batch_size=1, shuffle=False)
saved_model_file = os.path.join(args.saved_model_path,
'spsg-lidar-birdview.pth.tar')
config = {
'superpoint': {
'nms_radius': 4,
'keypoint_threshold': 0.005,
'max_keypoints': 200,
},
'Superglue': {
'weights': 'outdoor',
'sinkhorn_iterations': 100,
'match_threshold': 0.1,
}
}
model = Matching(config)
model_checkpoint = torch.load(saved_model_file,
map_location=lambda storage, loc: storage)
model.load_state_dict(model_checkpoint)
print("Loaded model checkpoints from \'{}\'.".format(saved_model_file))
device = torch.device(
'cuda' if torch.cuda.is_available() and args.use_gpu else 'cpu')
model.to(device)
torch.set_grad_enabled(False)
for target, source, T_target_source, target_filename, source_filename in data_loader:
# iteration += 1
assert (source.shape == target.shape)
print(target_filename[0])
print(source_filename[0])
B, C, W, H = source.shape
target = target.to(device)
source = source.to(device)
pred = model({'image0': target, 'image1': source})
target_kpts = pred['keypoints0'][0].cpu()
source_kpts = pred['keypoints1'][0].cpu()
if len(target_kpts) == 0 or len(source_kpts) == 0:
continue
# in superglue/numpy/tensor the coordinates are (i,j) which correspond to (v,u) in PIL Image/opencv
target_kpts_in_meters = target_kpts * args.meters_per_pixel - 50
source_kpts_in_meters = source_kpts * args.meters_per_pixel - 50
match_mask_ground_truth = make_ground_truth_matrix(
target_kpts_in_meters, source_kpts_in_meters, T_target_source[0],
args.tolerance_in_meters)
target_image_raw = target[0][0].cpu().numpy()
source_image_raw = source[0][0].cpu().numpy()
target_image_raw = np.stack([target_image_raw] * 3, -1) * 10
source_image_raw = np.stack([source_image_raw] * 3, -1) * 10
cv2.imshow('target_image_raw', target_image_raw)
cv2.imshow('source_image_raw', source_image_raw)
# target_kpts = np.round(target_kpts.numpy()).astype(int)
T_target_source = T_target_source[0].numpy()
source_kpts = source_kpts.numpy()
target_kpts = target_kpts.numpy()
source_kpts_in_meters = pts_from_pixel_to_meter(
source_kpts, args.meters_per_pixel)
print('T_target_source:\n', T_target_source)
source_kpts_in_meters_in_target_img = [
(T_target_source[:3, :3] @ np.array(
[source_kpt[0], source_kpt[1], 0]) +
T_target_source[:3, 3])[:2]
for source_kpt in source_kpts_in_meters
]
source_kpts_in_meters_in_target_img = np.array(
source_kpts_in_meters_in_target_img)
source_kpts_in_target_img = pts_from_meter_to_pixel(
source_kpts_in_meters_in_target_img, args.meters_per_pixel)
source_kpts = np.round(source_kpts).astype(int)
source_kpts_in_target_img = np.round(source_kpts_in_target_img).astype(
int)
target_image_poi = visualize_poi(target_image_raw.copy(), target_kpts,
(0, 1, 0))
source_image_poi = visualize_poi(source_image_raw.copy(), source_kpts,
(1, 0, 0))
# target_image_poi = target_image_raw.copy()
# source_image_poi = source_image_raw.copy()
# for (x0, y0), (x1, y1) in zip(source_kpts, target_kpts):
# # c = c.tolist()
# # cv2.line(target_image, (x0, y0), (x0 + 50, y0 + 50),
# # color=[255,0,0], thickness=1, lineType=cv2.LINE_AA)
# # display line end-points as circles
# cv2.circle(target_image_poi, (x1, y1), 4, (0, 255, 0), 1, lineType=cv2.LINE_AA)
# cv2.circle(source_image_poi, (x0, y0), 4, (255, 0, 0), 1, lineType=cv2.LINE_AA)
# # cv2.circle(out, (x1 + margin + W0, y1), 2, c, -1,
# # lineType=cv2.LINE_AA)
cv2.imshow('target_image_poi', target_image_poi)
cv2.imshow('source_image_poi', source_image_poi)
matches = pred['matches0'][0].cpu().numpy()
valid = matches > -1
target_kpts_matched = target_kpts[valid]
source_kpts_matched = source_kpts[matches[valid]]
# Matching visualize
match_image = visualize_matching(target_image_poi, source_image_poi,
target_kpts_matched,
source_kpts_matched)
W, H = 480, 460
h_margin = 10
v_margin = 10
window_image = np.ones((2 * H + 2 * v_margin, 2 * W + h_margin, 3))
window_image[:H, :(W)] = cv2.resize(target_image_raw, (W, H),
cv2.INTER_NEAREST)
window_image[:H, -W:] = cv2.resize(source_image_raw, (W, H),
cv2.INTER_NEAREST)
window_image[H + v_margin:, :] = cv2.resize(
match_image, (2 * W + h_margin, H + v_margin), cv2.INTER_NEAREST)
cv2.imshow('match_image', match_image)
cv2.imshow("window_image", window_image)
cv2.waitKey(0)
# margin = 10
# match_image = np.ones((H, 2 * W + margin))
# match_image = np.stack([match_image] * 3, -1)
#
# match_image[:, :W] = target_image_poi
# match_image[:, W + margin:] = source_image_poi
#
#
# for (x0, y0), (x1, y1) in zip(target_kpts_matched, source_kpts_matched):
# cv2.line(match_image, (x0, y0), (x1 + margin + W, y1),
# color=[0.9, 0.9, 0], thickness=1, lineType=cv2.LINE_AA)
# # display line end-points as circles
# # cv2.circle(match_image, (x0, y0), 2, (0, 255, 0), -1, lineType=cv2.LINE_AA)
# # cv2.circle(match_image, (x1 + margin + W, y1), 2, (255, 0, 0), -1,
# # lineType=cv2.LINE_AA)
torch.set_grad_enabled(True)
pass
def spi_vlad_roc_auc():
images_info_validate = make_images_info(struct_filename=os.path.join(
args.dataset_dir, 'struct_file_' + args.sequence_validate + '.txt'))
validate_images_dir = os.path.join(args.dataset_dir,
args.sequence_validate)
# validate_database_images_info, validate_query_images_info = train_test_split(images_info_validate,
# test_size=0.4, random_state=20)
base_model = BaseModel(300, 300)
dim = 256
# Define model for embedding
net_vlad = NetVLAD(num_clusters=args.num_clusters,
dim=dim,
alpha=1.0,
outdim=args.final_dim)
model = EmbedNet(base_model, net_vlad)
saved_model_file = os.path.join(args.saved_model_path,
'model-to-check-top1.pth.tar')
model_checkpoint = torch.load(saved_model_file,
map_location=lambda storage, loc: storage)
model.load_state_dict(model_checkpoint)
dev = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(dev)
print("Loaded model checkpoints from \'{}\'.".format(saved_model_file))
validate_dataset = PureDataset(images_info=images_info_validate,
images_dir=validate_images_dir)
validate_data_loader = DataLoader(validate_dataset, batch_size=1)
descriptors = []
positions = []
torch.set_grad_enabled(False)
i = 0
for query, query_info in tqdm(validate_data_loader):
# print(query.shape)
netvlad_encoding = model(query.to(dev)).cpu().view(-1)
# print(netvlad_encoding.shape)
descriptors.append(netvlad_encoding)
position = query_info['position'].view(3)
positions.append(position)
i = i + 1
# if i > 100:
# break
# print(netvlad_encoding)
descriptors = torch.cat(descriptors).view(-1, args.final_dim)
# print(descriptors.shape)
N = len(descriptors)
diff = descriptors[..., None] - descriptors.transpose(0, 1)[None, ...]
score_matrix = (1 - torch.einsum('mdn,mdn->mn', diff, diff)).numpy()
# print(score_matrix)
positions = torch.cat(positions).view(-1, 3)
diff = positions[..., None] - positions.transpose(0, 1)[None, ...]
label_matrix = (torch.einsum('mdn,mdn->mn', diff, diff) <
(args.positive_search_radius**2)).numpy()
# print(label_matrix.reshape(-1))
# print(score_matrix.reshape(-1))
print('AUC:',
roc_auc_score(label_matrix.reshape(-1), score_matrix.reshape(-1)))
# print('F1-score:', f1_score(label_matrix.reshape(-1), score_matrix.reshape(-1)))
precision, recall, thresholds = precision_recall_curve(
label_matrix.reshape(-1), score_matrix.reshape(-1))
print(recall, precision)
plt.plot(recall, precision, lw=1)
# plt.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label="Luck")
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel("Recall Rate")
plt.ylabel("Precision Rate")
plt.show()
descriptors = descriptors.numpy()
np.save('spi-vlad-kitti' + args.sequence_validate + '.npy', descriptors)
torch.set_grad_enabled(True)
# label_matrix =
pass
def main():
images_info_train = make_images_info(struct_filename=os.path.join(
args.dataset_dir, 'struct_file_' + args.sequence_train + '.txt'))
images_info_validate = make_images_info(struct_filename=os.path.join(
args.dataset_dir, 'struct_file_' + args.sequence_validate + '.txt'))
train_images_dir = os.path.join(args.dataset_dir, args.sequence_train)
validate_images_dir = os.path.join(args.dataset_dir,
args.sequence_validate)
# train_database_images_info, train_query_images_info = train_test_split(images_info_train, test_size=0.1,
# random_state=23)
# validate_database_images_info, validate_query_images_info = train_test_split(images_info_validate, test_size=0.2,
# random_state=23)
train_dataset = SuperglueDataset(
images_info=images_info_train,
images_dir=train_images_dir,
positive_search_radius=args.positive_search_radius,
meters_per_pixel=args.meters_per_pixel)
validate_dataset = SuperglueDataset(
images_info=images_info_validate,
images_dir=validate_images_dir,
positive_search_radius=args.positive_search_radius,
meters_per_pixel=args.meters_per_pixel)
train_data_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
validate_data_loader = DataLoader(validate_dataset,
batch_size=1,
shuffle=True)
saved_model_file = os.path.join(args.saved_model_path,
'spsg-rotation-invariant.pth.tar')
# saved_model_file = os.path.join(args.saved_model_path, 'spsg-juxin.pth.tar')
config = {
'superpoint': {
'nms_radius': 4,
'keypoint_threshold': 0.005,
'max_keypoints': 400,
},
'Superglue': {
'weights': 'outdoor',
'sinkhorn_iterations': 100,
'match_threshold': 0.2,
}
}
device = torch.device(
'cuda' if torch.cuda.is_available() and args.use_gpu else 'cpu')
model = Matching(config).to(device)
if args.load_checkpoints:
model_checkpoint = torch.load(
saved_model_file, map_location=lambda storage, loc: storage)
model.load_state_dict(model_checkpoint)
print("Loaded model checkpoints from \'{}\'.".format(saved_model_file))
optimizer = optim.Adam(list(model.superglue.parameters()) +
list(model.superpoint.convDa.parameters()) +
list(model.superpoint.convDb.parameters()),
lr=args.learning_rate)
viz = visdom.Visdom()
train_loss = viz.scatter(X=np.asarray([[0, 0]]))
train_precision = viz.scatter(X=np.asarray([[0, 0]]))
train_recall = viz.scatter(X=np.asarray([[0, 0]]))
train_true_pairs = viz.scatter(X=np.asarray([[0, 0]]))
viz_train = {
'viz': viz,
'train_loss': train_loss,
'train_precision': train_precision,
'train_recall': train_recall,
'train_true_pairs': train_true_pairs,
}
viz_validate = {
'viz': viz,
'validate_precision': train_precision,
'validate_recall': train_recall,
'validate_true_pairs': train_true_pairs,
}
for epoch in range(args.epochs):
epoch = epoch + 1
if epoch % 1 == 0:
validate(epoch,
model,
validate_data_loader,
viz_validate=viz_validate)
# validate_sift(sift_matcher, validate_data_loader)
train(epoch, model, optimizer, train_data_loader, viz_train=viz_train)
torch.save(model.state_dict(), saved_model_file)
print("Saved models in \'{}\'.".format(saved_model_file))
pass
self.images_info[pos_index]['orientation'])
T_query_positive = np.linalg.inv(
T_w_query) @ T_w_positive @ T_rot_wrt_z(theta_in_deg / 180 * np.pi)
# convert translation in pixels
# T_query_positive[:3,3] = T_query_positive[:3,3] / self.meters_per_pixel
if self.return_filename:
return query, positive, T_query_positive, \
os.path.join(self.images_dir, self.images_info[index]['image_file']), \
os.path.join(self.images_dir, self.images_info[pos_index]['image_file'])
else:
return query, positive, T_query_positive
if __name__ == '__main__':
# dataset_dir = '/media/admini/My_data/0904/dataset'
dataset_dir = '/media/admini/lavie/dataset/birdview_dataset'
sequence = '00'
positive_search_radius = 8
images_info = make_images_info(
struct_filename=os.path.join(dataset_dir, 'struct_file_' + sequence +
'.txt'))
images_dir = os.path.join(dataset_dir, sequence)
dataset = SuperglueDataset(images_info=images_info,
images_dir=images_dir,
positive_search_radius=positive_search_radius)
data_loader = DataLoader(dataset, batch_size=2, shuffle=True)
# with tqdm(data_loader) as tq:
for item in data_loader:
print(item)
print(len(dataset))