def dump_val_res(img1, img2, x1, x2, mask_before, mask_after, cx1, cy1, f1,
cx2, cy2, f2, R, t, dump):
if not os.path.exists(dump):
os.makedirs(dump)
# Images
img1 = img1.transpose(1, 2, 0)
img2 = img2.transpose(1, 2, 0)
cv2.imwrite(os.path.join(dump, "img1.png"), img1)
cv2.imwrite(os.path.join(dump, "img2.png"), img2)
dump_dict = {}
dump_dict["x1"] = x1
dump_dict["cx1"] = cx1
dump_dict["cy1"] = cy1
dump_dict["f1"] = f1
dump_dict["x2"] = x2
dump_dict["cx2"] = cx2
dump_dict["cy2"] = cy2
dump_dict["f2"] = f2
dump_dict["R"] = R
dump_dict["t"] = t
if mask_before is not None:
dump_dict["mask_before"] = mask_before
if mask_after is not None:
dump_dict["mask_after"] = mask_after
saveh5(dump_dict, os.path.join(dump, "dump.h5"))
def dump_data_pair(args):
dump_dir, idx, ii, jj, queue = args
# queue for monitoring
if queue is not None:
queue.put(idx)
dump_file = os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj))
# txt_dict = os.path.join(dump_dir, "idx_sort-{}-{}.txt".format(ii, jj))
if not os.path.exists(dump_file):
# Load descriptors for ii
desc_ii = loadh5(os.path.join(dump_dir,
"kp-aff-desc-{}.h5".format(ii)))["desc"]
desc_jj = loadh5(os.path.join(dump_dir,
"kp-aff-desc-{}.h5".format(jj)))["desc"]
# compute decriptor distance matrix
distmat = np.sqrt(
np.sum(
(np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2,
axis=2))
# Choose K best from N
idx_sort = np.argsort(distmat, axis=1)[:, :config.obj_num_nn]
idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],
idx_sort.shape[1],
axis=1), idx_sort)
distmat = distmat[idx_sort]
# Dump to disk
dump_dict = {}
dump_dict["idx_sort"] = idx_sort
saveh5(dump_dict, dump_file)
def dump_data_pair(args):
dump_dir, idx, ii, jj, queue = args
# queue for monitoring
if queue is not None:
queue.put(idx)
dump_file = os.path.join(
dump_dir, "idx_sort-{}-{}.h5".format(ii, jj))
dump_file_mutual_ratio = os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj))
if not os.path.exists(dump_file) or not os.path.exists(dump_file_mutual_ratio):
# if 1==1:
# Load descriptors for ii
desc_ii = loadh5(
os.path.join(dump_dir, "kp-z-desc-{}.h5".format(ii)))["desc"]
desc_jj = loadh5(
os.path.join(dump_dir, "kp-z-desc-{}.h5".format(jj)))["desc"]
# compute decriptor distance matrix
# distmat = np.sqrt(
# np.sum(
# (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2,
# axis=2))
##### replace with faster distmat computation
distmat = np.sqrt(np.sum(desc_ii**2, axis=1, keepdims=True) +
np.sum(desc_jj**2, axis=1) -
2 * np.dot(desc_ii, desc_jj.T))
# Choose K best from N
idx_sort0 = np.argsort(distmat, axis=1)[:, :1]
idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort0.shape[1], axis=1),idx_sort0)
# saving the ratio
idx_sort_2nd = np.argsort(distmat, axis=1)[:, :2]
idx_sort_2nd = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort_2nd.shape[1], axis=1),idx_sort_2nd)
dist2nd = distmat[idx_sort_2nd]
ratio = dist2nd[:, 0] / dist2nd[:, 1]
# saving mutual neighbor information
# Please note that crossCheck in opencv doesn't work properly and thus is deprecated in this code.
idx_sort1 = np.argsort(distmat, axis=0)[0, :]
mutual_neigh = idx_sort1[idx_sort0.squeeze()] == np.arange(idx_sort0.shape[0])
# Dump to disk
dump_dict = {}
dump_dict_mutual_ratio = {}
dump_dict["idx_sort"] = idx_sort
dump_dict_mutual_ratio["ratio"] = ratio
dump_dict_mutual_ratio["mutual"] = mutual_neigh
# Looks ugly because we just add ratio and mutual neighboring information to CNe dataset
if not os.path.exists(dump_file):
saveh5(dump_dict, dump_file)
if not os.path.exists(dump_file_mutual_ratio):
saveh5(dump_dict_mutual_ratio, dump_file_mutual_ratio)
def dump_nn(self, ii, jj):
dump_file = os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj))
if not os.path.exists(dump_file):
image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj]
desc_ii = loadh5(image_i+'.'+self.desc_name+'.hdf5')["descriptors"]
desc_jj = loadh5(image_j+'.'+self.desc_name+'.hdf5')["descriptors"]
idx_sort, ratio_test, mutual_nearest = computeNN(desc_ii, desc_jj)
# Dump to disk
dump_dict = {}
dump_dict["idx_sort"] = idx_sort
dump_dict["ratio_test"] = ratio_test
dump_dict["mutual_nearest"] = mutual_nearest
saveh5(dump_dict, dump_file)
def _compute_desc(self):
"""Compute Descriptors """
total_time = 0.0
# Read image
start_time = time.clock()
cur_data = self.dataset.load_data()
end_time = time.clock()
load_time = (end_time - start_time) * 1000.0
if self.config.verbose:
print("Time taken to load patches is {} ms".format(load_time))
total_time += load_time
# Test using the test function
start_time = time.clock()
descs = self._test_multibatch(cur_data)
end_time = time.clock()
compute_time = (end_time - start_time) * 1000.0
if self.config.verbose:
print("Time taken to compute is {} ms".format(compute_time))
total_time += compute_time
if self.config.verbose:
print("Total time for descriptor is {} ms".format(total_time))
# Overwrite angle
kps = cur_data["kps"].copy()
kps[:, 3] = cur_data["angle"][:, 0]
# Save as h5 file
save_dict = {}
# save_dict['keypoints'] = cur_data["kps"]
save_dict['keypoints'] = kps
save_dict['descriptors'] = descs
saveh5(save_dict, self.config.test_out_file)
def make_xy(pair_index, img, kp, desc, aff, K, R, t, cur_folder):
kp_initial = copy.deepcopy(kp)
xs = []
xs_initial = []
ys = []
Rs = []
ts = []
img1s = []
img2s = []
cx1s = []
cy1s = []
f1s = []
cx2s = []
cy2s = []
f2s = []
affine = []
# Create a random folder in scratch
dump_dir = os.path.join(cur_folder, "dump")
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
pool_arg = []
for idx in range(pair_index.shape[0]):
ii = pair_index[idx, 0]
jj = pair_index[idx, 1]
print("\rExtracting keypoints {} / {}".format(idx,
pair_index.shape[0]),
end="")
sys.stdout.flush()
# Check and extract keypoints if necessary
for i in [ii, jj]:
i = int(i)
dump_file = os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(i))
if not os.path.exists(dump_file):
# Correct coordinates using K
cx = K[i, 2]
cy = K[i, 5]
# Correct focals
fx = K[i, 0]
fy = K[i, 4]
kp[i] = (kp[i] - np.array([[cx, cy]])) / np.asarray([[fx, fy]])
# Write descs to harddisk to parallize
dump_dict = {}
dump_dict["kp_normal"] = kp[i]
dump_dict["kp"] = kp_initial[i]
dump_dict["desc"] = desc[i]
dump_dict["aff"] = aff[i]
saveh5(dump_dict, dump_file)
else:
dump_dict = loadh5(dump_file)
kp[i] = dump_dict["kp_normal"]
kp_initial[i] = dump_dict["kp"]
desc[i] = dump_dict["desc"]
aff[i] = dump_dict["aff"]
pool_arg += [(dump_dir, idx, int(ii), int(jj))]
print("")
# Run mp job
ratio_CPU = 0.9
number_of_process = int(ratio_CPU * mp.cpu_count())
pool = mp.Pool(processes=number_of_process)
manager = mp.Manager()
queue = manager.Queue()
for idx_arg in xrange(len(pool_arg)):
pool_arg[idx_arg] = pool_arg[idx_arg] + (queue, )
# map async
pool_res = pool.map_async(dump_data_pair, pool_arg)
# monitor loop
while True:
if pool_res.ready():
break
else:
size = queue.qsize()
print("\rDistMat {} / {}".format(size, len(pool_arg)), end="")
sys.stdout.flush()
time.sleep(1)
pool.close()
pool.join()
print("")
# Pack data
idx = 0
total_num = 0
good_num = 0
bad_num = 0
ratio1 = []
ratio2 = []
for idx in range(pair_index.shape[0]):
ii = int(pair_index[idx, 0])
jj = int(pair_index[idx, 1])
print("\rWorking on {} / {}".format(idx, pair_index.shape[0]), end="")
sys.stdout.flush()
K1 = K[ii].reshape(3, 3)
K2 = K[jj].reshape(3, 3)
# ------------------------------
# Get dR
R_i = R[ii].reshape(3, 3)
R_j = R[jj].reshape(3, 3)
dR = np.dot(R_j, R_i.T)
# Get dt
t_i = t[ii].reshape(3, 1)
t_j = t[jj].reshape(3, 1)
dt = t_j - np.dot(dR, t_i)
# ------------------------------
# Get keypoints for the first image
x1 = kp[ii]
y1 = np.concatenate((kp[ii], np.ones((kp[ii].shape[0], 1))), axis=1)
# Project the first points into the second image
y1p = np.matmul(dR[None], y1[..., None]) + dt[None]
# move back to the canonical plane
x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None]
# ------------------------------
# Get keypoints for the second image
x2 = kp[jj]
# # DEBUG ------------------------------
# # Check if the image projections make sense
# draw_val_res(
# img[ii],
# img[jj],
# x1, x1p, np.random.rand(x1.shape[0]) < 0.1,
# (img[ii][0].shape[1] - 1.0) * 0.5,
# (img[ii][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][ii, 0, 0],
# (img[jj][0].shape[1] - 1.0) * 0.5,
# (img[jj][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][jj, 0, 0],
# "./debug_imgs/",
# "debug_img{:04d}.png".format(idx)
# )
# ------------------------------
# create x1, y1, x2, y2 as a matrix combo
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1)
y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
# Load precomputed nearest neighbors
idx_sort = loadh5(
os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii,
jj)))["idx_sort"]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x1pmat = x1pmat[idx_sort]
y1pmat = y1pmat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x1p = np.concatenate([x1pmat.reshape(-1, 1),
y1pmat.reshape(-1, 1)],
axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
# make xs in NHWC
xs += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
# ------------------------------
# Get the geodesic distance using with x1, x2, dR, dt
if config.obj_geod_type == "sampson":
geod_d = get_sampsons(x1, x2, dR, dt)
elif config.obj_geod_type == "episqr":
geod_d = get_episqr(x1, x2, dR, dt)
elif config.obj_geod_type == "episym":
geod_d = get_episym(x1, x2, dR, dt)
# geod_d = get_episym(x1, x2, dR, dt, K1_inv, K2_inv, K1, K2)
# Get *rough* reprojection errors. Note that the depth may be noisy. We
# ended up not using this...
reproj_d = np.sum((x2 - x1p)**2, axis=1)
# count inliers and outliers
total_num += len(geod_d)
good_num += np.sum((geod_d < config.obj_geod_th))
bad_num += np.sum((geod_d >= config.obj_geod_th))
'''
mask = np.zeros(len(geod_d))
for i in range(len(geod_d)):
if geod_d[i] < config.obj_geod_th:
mask[i] = 1
np.savetxt(os.path.join(dump_dir, "mask-{}-{}.txt".format(ii, jj)), mask)
'''
ys += [np.stack([geod_d, reproj_d], axis=1)]
# Save R, t for evaluation
Rs += [np.array(dR).reshape(3, 3)]
# normalize t before saving
dtnorm = np.sqrt(np.sum(dt**2))
assert (dtnorm > 1e-5)
dt /= dtnorm
ts += [np.array(dt).flatten()]
# Save img1 and img2 for display
img1s += [img[ii]]
img2s += [img[jj]]
cx = K[ii, 2]
cy = K[ii, 5]
cx1s += [cx]
cy1s += [cy]
cx = K[jj, 2]
cy = K[jj, 5]
cx2s += [cx]
cy2s += [cy]
fx = K[ii, 0]
fy = K[ii, 4]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
f1s += [f]
fx = K[jj, 0]
fy = K[jj, 4]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
f2s += [f]
# Generate xs_initial and T-transform
x1 = kp_initial[ii]
x2 = kp_initial[jj]
aff1 = aff[ii]
aff2 = aff[jj]
aff1 = np.repeat(aff1[:, :][..., None], len(aff2),
axis=-1).transpose(0, 2, 1)
aff2 = np.repeat(aff2[:, :][None], len(aff1), axis=0)
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
idx_sort = loadh5(
os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii,
jj)))["idx_sort"]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
aff1 = aff1[idx_sort]
aff2 = aff2[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
affine += [
np.concatenate(
[aff1.reshape(-1, 9), aff2.reshape(-1, 9)], axis=1)
]
xs_initial += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
print("")
# Do *not* convert to numpy arrays, as the number of keypoints may differ
# now. Simply return it
print(".... done")
if total_num > 0:
print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format(
good_num, total_num,
float(good_num) / float(total_num)))
print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format(
bad_num, total_num,
float(bad_num) / float(total_num)))
res_dict = {}
res_dict["xs"] = xs
res_dict["xs_initial"] = xs_initial
res_dict["affine"] = affine
res_dict["ys"] = ys
res_dict["Rs"] = Rs
res_dict["ts"] = ts
res_dict["img1s"] = img1s
res_dict["cx1s"] = cx1s
res_dict["cy1s"] = cy1s
res_dict["f1s"] = f1s
res_dict["img2s"] = img2s
res_dict["cx2s"] = cx2s
res_dict["cy2s"] = cy2s
res_dict["f2s"] = f2s
return res_dict
def _compute_desc(self):
"""Compute Descriptors """
total_time = 0.0
global_start_time = time.time()
for scene_name in cst.SCENE_LIST:
duration = time.time() - global_start_time
print('****** %s ****** %d:%02d' %
(scene_name, duration / 60, duration % 60))
if not os.path.exists(os.path.join(cst.DES_DIR, scene_name)):
os.makedirs(os.path.join(cst.DES_DIR, scene_name))
for img_key in range(1, cst.MAX_IMG_NUM + 1):
if ((scene_name == 'v_feast' and img_key == 6)
or (scene_name == 'v_wall' and img_key == 6)):
if cst.DATA == 'hpatches':
continue
img_fn = os.path.join(cst.DATA_DIR, scene_name,
'%d.ppm' % (img_key))
new_kp_fn = os.path.join(cst.ORI_DIR, scene_name,
'%d_kp.txt' % (img_key))
des_fn = os.path.join(cst.DES_DIR, scene_name,
'%d.h5' % (img_key))
# Read image
start_time = time.clock()
cur_data = self.dataset.my_load_data(img_fn, new_kp_fn,
cst.NEW_SIZE)
end_time = time.clock()
load_time = (end_time - start_time) * 1000.0
#print("Time taken to load patches is {} ms".format(
# load_time
#))
total_time += load_time
# import IPython
# IPython.embed()
# -------------------------------------------------------------------------
# Test using the test function
start_time = time.clock()
descs = self._test_multibatch(cur_data)
end_time = time.clock()
compute_time = (end_time - start_time) * 1000.0
#print("Time taken to compute is {} ms".format(
# compute_time
#))
total_time += compute_time
#print("Total time for descriptor is {} ms".format(total_time))
# Overwrite angle
kps = cur_data["kps"].copy()
kps[:, 3] = cur_data["angle"][:, 0]
# Save as h5 file
save_dict = {}
# save_dict['keypoints'] = cur_data["kps"]
save_dict['keypoints'] = kps
save_dict['descriptors'] = descs
saveh5(save_dict, des_fn)
def make_xy(num_sample, pairs, kp, z, desc, img, geom, vis, depth, geom_type,
cur_folder):
xs = []
ys = []
Rs = []
ts = []
mutuals = []
ratios = []
img1s = []
img2s = []
cx1s = []
cy1s = []
f1s = []
cx2s = []
cy2s = []
f2s = []
# Create a random folder in scratch
dump_dir = os.path.join(cur_folder, "dump")
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
# randomly suffle the pairs and select num_sample amount
np.random.seed(1234)
cur_pairs = [
pairs[_i] for _i in np.random.permutation(len(pairs))[:num_sample]
]
idx = 0
for ii, jj in cur_pairs:
idx += 1
print(
"\rExtracting keypoints {} / {}".format(idx, len(cur_pairs)),
end="")
sys.stdout.flush()
# Check and extract keypoints if necessary
for i in [ii, jj]:
dump_file = os.path.join(dump_dir, "kp-z-desc-{}.h5".format(i))
if not os.path.exists(dump_file):
if kp[i] is None:
cv_kp, cv_desc = sift.detectAndCompute(img[i].transpose(
1, 2, 0), None)
cx = (img[i][0].shape[1] - 1.0) * 0.5
cy = (img[i][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][i, 0, 2]
cy += parse_geom(geom, geom_type)["K"][i, 1, 2]
xy = np.array([_kp.pt for _kp in cv_kp])
# Correct focals
fx = parse_geom(geom, geom_type)["K"][i, 0, 0]
fy = parse_geom(geom, geom_type)["K"][i, 1, 1]
kp[i] = (
xy - np.array([[cx, cy]])
) / np.asarray([[fx, fy]])
desc[i] = cv_desc
if z[i] is None:
cx = (img[i][0].shape[1] - 1.0) * 0.5
cy = (img[i][0].shape[0] - 1.0) * 0.5
fx = parse_geom(geom, geom_type)["K"][i, 0, 0]
fy = parse_geom(geom, geom_type)["K"][i, 1, 1]
xy = kp[i] * np.asarray([[fx, fy]]) + np.array([[cx, cy]])
if len(depth) > 0:
z[i] = depth[i][
0,
np.round(xy[:, 1]).astype(int),
np.round(xy[:, 0]).astype(int)][..., None]
else:
z[i] = np.ones((xy.shape[0], 1))
# Write descs to harddisk to parallize
dump_dict = {}
dump_dict["kp"] = kp[i]
dump_dict["z"] = z[i]
dump_dict["desc"] = desc[i]
saveh5(dump_dict, dump_file)
else:
dump_dict = loadh5(dump_file)
kp[i] = dump_dict["kp"]
z[i] = dump_dict["z"]
desc[i] = dump_dict["desc"]
print("")
# Create arguments
pool_arg = []
idx = 0
for ii, jj in cur_pairs:
idx += 1
pool_arg += [(dump_dir, idx, ii, jj)]
# Run mp job
ratio_CPU = 0.8
number_of_process = int(ratio_CPU * mp.cpu_count())
pool = mp.Pool(processes=number_of_process)
manager = mp.Manager()
queue = manager.Queue()
# for debugging in dump_data_pair
# dump_data_pair(pool_arg[1] + (queue,))
for idx_arg in xrange(len(pool_arg)):
pool_arg[idx_arg] = pool_arg[idx_arg] + (queue,)
# map async
pool_res = pool.map_async(dump_data_pair, pool_arg)
# monitor loop
while True:
if pool_res.ready():
break
else:
size = queue.qsize()
print("\rDistMat {} / {}".format(size, len(pool_arg)), end="")
sys.stdout.flush()
time.sleep(1)
pool.close()
pool.join()
print("")
# Pack data
idx = 0
total_num = 0
good_num = 0
bad_num = 0
for ii, jj in cur_pairs:
idx += 1
print("\rWorking on {} / {}".format(idx, len(cur_pairs)), end="")
sys.stdout.flush()
# ------------------------------
# Get dR
R_i = parse_geom(geom, geom_type)["R"][ii]
R_j = parse_geom(geom, geom_type)["R"][jj]
dR = np.dot(R_j, R_i.T)
# Get dt
t_i = parse_geom(geom, geom_type)["t"][ii].reshape([3, 1])
t_j = parse_geom(geom, geom_type)["t"][jj].reshape([3, 1])
dt = t_j - np.dot(dR, t_i)
# ------------------------------
# Get sift points for the first image
x1 = kp[ii]
y1 = np.concatenate([kp[ii] * z[ii], z[ii]], axis=1)
# Project the first points into the second image
y1p = np.matmul(dR[None], y1[..., None]) + dt[None]
# move back to the canonical plane
x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None]
# ------------------------------
# Get sift points for the second image
x2 = kp[jj]
# # DEBUG ------------------------------
# # Check if the image projections make sense
# draw_val_res(
# img[ii],
# img[jj],
# x1, x1p, np.random.rand(x1.shape[0]) < 0.1,
# (img[ii][0].shape[1] - 1.0) * 0.5,
# (img[ii][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][ii, 0, 0],
# (img[jj][0].shape[1] - 1.0) * 0.5,
# (img[jj][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][jj, 0, 0],
# "./debug_imgs/",
# "debug_img{:04d}.png".format(idx)
# )
# ------------------------------
# create x1, y1, x2, y2 as a matrix combo
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1)
y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
# Load precomputed nearest neighbors, ratios and mutual
idx_sort = loadh5(os.path.join(
dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"]
mutual = loadh5(os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["mutual"]
ratio = loadh5(os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["ratio"]
mutuals += [mutual]
ratios += [ratio]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x1pmat = x1pmat[idx_sort]
y1pmat = y1pmat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x1p = np.concatenate(
[x1pmat.reshape(-1, 1),
y1pmat.reshape(-1, 1)], axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
# make xs in NHWC
xs += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
# ------------------------------
# Get the geodesic distance using with x1, x2, dR, dt
if config.obj_geod_type == "sampson":
geod_d = get_sampsons(x1, x2, dR, dt)
elif config.obj_geod_type == "episqr":
geod_d = get_episqr(x1, x2, dR, dt)
elif config.obj_geod_type == "episym":
geod_d = get_episym(x1, x2, dR, dt)
# Get *rough* reprojection errors. Note that the depth may be noisy. We
# ended up not using this...
reproj_d = np.sum((x2 - x1p)**2, axis=1)
# count inliers and outliers
total_num += len(geod_d)
good_num += np.sum((geod_d < config.obj_geod_th))
bad_num += np.sum((geod_d >= config.obj_geod_th))
ys += [np.stack([geod_d, reproj_d], axis=1)]
# Save R, t for evaluation
Rs += [np.array(dR).reshape(3, 3)]
# normalize t before saving
dtnorm = np.sqrt(np.sum(dt**2))
assert (dtnorm > 1e-5)
dt /= dtnorm
ts += [np.array(dt).flatten()]
# Save img1 and img2 for display
img1s += [img[ii]]
img2s += [img[jj]]
cx = (img[ii][0].shape[1] - 1.0) * 0.5
cy = (img[ii][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][ii, 0, 2]
cy += parse_geom(geom, geom_type)["K"][ii, 1, 2]
fx = parse_geom(geom, geom_type)["K"][ii, 0, 0]
fy = parse_geom(geom, geom_type)["K"][ii, 1, 1]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
cx1s += [cx]
cy1s += [cy]
f1s += [f]
cx = (img[jj][0].shape[1] - 1.0) * 0.5
cy = (img[jj][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][jj, 0, 2]
cy += parse_geom(geom, geom_type)["K"][jj, 1, 2]
fx = parse_geom(geom, geom_type)["K"][jj, 0, 0]
fy = parse_geom(geom, geom_type)["K"][jj, 1, 1]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
cx2s += [cx]
cy2s += [cy]
f2s += [f]
# Do *not* convert to numpy arrays, as the number of keypoints may differ
# now. Simply return it
print(".... done")
if total_num > 0:
print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format(
good_num, total_num, float(good_num) / float(total_num)))
print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format(
bad_num, total_num, float(bad_num) / float(total_num)))
res_dict = {}
res_dict["xs"] = xs
res_dict["ys"] = ys
res_dict["Rs"] = Rs
res_dict["ts"] = ts
res_dict["img1s"] = img1s
res_dict["cx1s"] = cx1s
res_dict["cy1s"] = cy1s
res_dict["f1s"] = f1s
res_dict["img2s"] = img2s
res_dict["cx2s"] = cx2s
res_dict["cy2s"] = cy2s
res_dict["f2s"] = f2s
res_dict["mutuals"] = mutuals
res_dict["ratios"] = ratios
res_dict["pairs"] = cur_pairs
return res_dict
def createDump(args):
# print("createDump called")
idx_jpg, jpg_file, train_data_dir, dump_data_dir, tmp_patch_dir, scale_hist, scale_hist_c, out_dim, param, queue = args
# print(idx_jpg)
# print(jpg_file)
# print(args)
# queue for monitoring
if queue is not None:
queue.put(idx_jpg)
final_dump_file_name = os.path.join(
tmp_patch_dir,
os.path.basename(jpg_file).split(".")[0] + ".h5")
# print(final_dump_file_name)
if not os.path.exists(final_dump_file_name):
# load image
bUseColorImage = getattr(param.patch, "bUseColorImage", False)
if not bUseColorImage:
bUseDebugImage = getattr(param.patch, "bUseDebugImage", False)
if not bUseDebugImage:
img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY)
else:
# debug image contains keypoints survive the SfM
# pipeline (blue) and the rest (in red)
img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY)
# debug_jpg = train_data_dir + jpg_file.replace(".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) +".jpg")
debug_jpg = os.path.splitext(jpg_file)[0] + "-kp-minsc-" + str(
param.dataset.fMinKpSize) + ".jpg"
imgd = cv2.cvtColor(cv2.imread(debug_jpg), cv2.COLOR_BGR2GRAY)
img = cv2.resize(imgd, img.shape[:2][::-1])
in_dim = 1
else:
img = cv2.imread(train_data_dir + jpg_file)
in_dim = 3
assert (img.shape[-1] == in_dim)
# ----------------------------------------
# load kp data
# Note that data is in order of [x,y,scale,angle,pointid,setid]]
# guessing here: pointid is the point id number for the keypoint in the two 2d image
# or maybe it's a unique idx for a point in a list of all points?
# of maybe its the index of a point for all points in a set?(I think it's this last one, maybe)
# setid is the id num for the pair which is created from the image keypoints (this would keep a uniform )
# For positive: select randomly from "valid_keypoints"
pos_kp_file_name = jpg_file.replace(
".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) + ".h5")
# print(pos_kp_file_name)
with h5py.File(pos_kp_file_name, "r") as h5file:
sfm_kp = h5file["valid_keypoints"][()]
non_sfm_kp = h5file["other_keypoints"][()]
# add two dummy fields to non_sfm since they don"t have id
# and group. THis means pointid,setid are set to -1 for non_sfm_kp
# non_sfm_kp = np.concatenate([non_sfm_kp, -np.ones((non_sfm_kp.shape[0], 2))],axis=1)
# if there are no keypoints which were leveraged in SFM
if (len(sfm_kp) == 0):
sfm_kp = np.empty([0, 6])
if (len(non_sfm_kp) == 0):
non_sfm_kp = np.empty([0, 6])
# return None
# print(sfm_kp)
# print(type(sfm_kp))
# print(sfm_kp.shape)
# print(non_sfm_kp)
# print(type(non_sfm_kp))
# print(non_sfm_kp.shape)
pos_kp = np.concatenate(
(sfm_kp, np.ones((sfm_kp.shape[0], 1), dtype="float")), axis=1)
if (len(sfm_kp) == 0):
pos_kp = pos_kp[1:, :]
if (len(non_sfm_kp) == 0):
non_sfm_kp = non_sfm_kp[1:, :]
# Select subset for positives (assuming we have same coordinates for all image)
# dump_file_name = dump_data_dir + jpg_file.replace(".jpg","_idxPosSel.h5")
dump_file_name = os.path.join(
dump_data_dir,
os.path.basename(jpg_file).split(".")[0] + "_idxPosSel.h5")
# if dump file not exist, create it; otherwise load it
if not os.path.exists(dump_file_name):
# idxPosShuffle = np.argsort(pos_kp[3,:])[::-1] # sort backwards
idxPosShuffle = np.random.permutation(
len(pos_kp)) # random shuffle
pos_2_keep = len(idxPosShuffle)
if param.dataset.nPosPerImg > 0:
pos_2_keep = min(pos_2_keep, param.dataset.nPosPerImg)
idxPosSel = idxPosShuffle[:pos_2_keep] # shuffle the points
to_save = {"saveval": idxPosSel}
# print(to_save)
saveh5(to_save, dump_file_name)
else:
to_load = loadh5(dump_file_name)
idxPosSel = to_load["saveval"]
pos_kp = pos_kp[idxPosSel]
# print(pos_kp)
# negative sampling:
#
# 1) only use Sfm points, too close keypoints will be rejected as
# negative pair (because of high potential overlapping)
#
# 2) Use all SIFT points, when the overlapping of two feature context
# windows is larger than a threshold, it will be rejected as a negative
# pair (because it shares too much common regions..). In this step, we
# concatenate sfm_kp and non_sfm_kp to form keypoint class.
neg_mine_method = getattr(param.patch, "sNegMineMethod",
"use_only_SfM_points")
if neg_mine_method == "use_only_SfM_points":
# print("mining sfm points...")
# exit()
# where is
neg_kp = random_mine_non_kp_with_2d_distance(
img, sfm_kp, scale_hist, scale_hist_c, param)
elif neg_mine_method == "use_all_SIFT_points":
# print("mining SIFT points...")
# exit()
max_iter = getattr(param.patch, "nMaxRandomNegMineIter", 100)
# print(sfm_kp)
# print(type(sfm_kp))
# print(sfm_kp.shape)
# print(non_sfm_kp)
# print(type(non_sfm_kp))
# print(non_sfm_kp.shape)
# print(pos_kp)
# print(type(pos_kp))
# print(pos_kp.shape)
try:
sift_kp = np.concatenate([sfm_kp, non_sfm_kp], axis=0)
neg_kp = random_mine_non_kp_with_3d_blocking(img,
sift_kp,
scale_hist,
scale_hist_c,
param,
max_iter=max_iter)
# neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, neg_per_iter = 10, max_iter=max_iter)
except ValueError as err:
# print(err)
# print("Possibly no non_sfm_keypoints?")
neg_kp = np.empty([0, 6])
else:
raise ValueError(
"Mining method {} is not supported!".format(neg_mine_method))
# add another dim indicating good or bad
neg_kp = np.concatenate(
(neg_kp, np.zeros((len(neg_kp), 1), dtype="float")), axis=1)
# concatenate negative and positives
# print(pos_kp)
# print(type(pos_kp))
# print(pos_kp.shape)
# print(neg_kp)
# print(type(neg_kp))
# print(neg_kp.shape)
kp = np.concatenate((pos_kp, neg_kp), axis=0)
# print(kp)
# Retrive target values, 1 for pos and 0 for neg
y = kp[:, 6]
# Retrieve angles
angle = kp[:, 3]
# Assign ID to keypoints (for sfm points, ID = 3d point ind; for non
# sfm kp, ID = -1)
ID = kp[:, 4]
# load patches with id (drop out of boundary)
bPerturb = getattr(param.patch, "bPerturb", False)
fPerturbInfo = getattr(param.patch, "fPerturbInfo", np.zeros((3, )))
nAugmentedRotations = getattr(param.patch, "nAugmentedRotations", 1)
fAugmentRange = getattr(param.patch, "fAugmentRange", 0)
fAugmentCenterRandStrength = getattr(param.patch,
"fAugmentCenterRandStrength", 0)
sAugmentCenterRandMethod = getattr(param.patch,
"sAugmentCenterRandMethod",
"uniform")
cur_data_set = load_patches(
img,
kp,
y,
ID,
angle,
param.patch.fRatioScale,
param.patch.fMaxScale,
param.patch.nPatchSize,
param.model.nDescInputSize,
in_dim,
bPerturb,
fPerturbInfo,
bReturnCoords=True,
nAugmentedRotations=nAugmentedRotations,
fAugmentRange=fAugmentRange,
fAugmentCenterRandStrength=fAugmentCenterRandStrength,
sAugmentCenterRandMethod=sAugmentCenterRandMethod,
nPatchSizeAug=param.patch.nPatchSizeAug,
)
# save dump as dictionary using saveh5
# from here Kernel died, because of not finding "MKL_intel_thread.dll"
# pdb.set_trace()
# here the data are saved in tmp_dump file, keys are numbers [0,1..]
tmpdict = dict(
(str(_idx), np.asarray(_data))
for _idx, _data in zip(np.arange(len(cur_data_set)), cur_data_set))
saveh5(tmpdict, final_dump_file_name)
return idx_jpg
def comp_process(mode, data, res_dir, config):
# Unpack some references
xs = data["xs"]
ys = data["ys"]
Rs = data["Rs"]
ts = data["ts"]
img1s = data["img1s"]
cx1s = data["cx1s"]
cy1s = data["cy1s"]
f1s = data["f1s"]
img2s = data["img2s"]
cx2s = data["cx2s"]
cy2s = data["cy2s"]
f2s = data["f2s"]
# Make fs numpy array
f1s = np.array(f1s)
f2s = np.array(f2s)
# Prepare directory
if not os.path.exists(res_dir):
os.makedirs(res_dir)
print("[{}] {}: Start testing".format(config.data_tr, time.asctime()))
# Validation
num_sample = len(xs)
if config.use_lift:
comp_list = [
"lmeds",
"ransac",
"mlesac",
# "usac5point", "usac8point",
# "usacnolo5point", "usacnolo8point"
]
else:
# comp_list = [
# "lmeds", "ransac", "top8_8point", "top50_lmeds", "top50_ransac",
# "gms", "gms_orb", "gms_default",
# "gms_orb_resize", "gms_orb_resize_ransac",
# "gms_orb_resize_tt", "gms_orb_resize_ransac_tt",
# "mlesac",
# ]
comp_list = [
"lmeds",
"ransac",
"mlesac",
# "usac5point", "usac8point",
# "usacnolo5point", "usacnolo8point"
]
if config.obj_num_kp == 2000:
comp_list += [
"gms_orb_resize_ransac_tt",
"gms_orb_resize_tt",
]
# Initialize arrays that will store measurements
err_q = {}
err_t = {}
num = {}
for _comp in comp_list:
err_q[_comp] = np.zeros(num_sample)
err_t[_comp] = np.zeros(num_sample)
num[_comp] = np.zeros(num_sample)
NUM_KP = config.obj_num_kp
# batch_size = config.val_batch_size
# num_batch = int(len(xs) / batch_size)
from gms_matcher import GmsMatcher
# SIFT
sift = cv2.xfeatures2d.SIFT_create(nfeatures=NUM_KP,
contrastThreshold=1e-5)
if cv2.__version__.startswith('3'):
sift_matcher = cv2.BFMatcher(cv2.NORM_L2)
else:
sift_matcher = cv2.BFMatcher_create(cv2.NORM_L2)
sift_gms = GmsMatcher(sift, sift_matcher)
# ORB
orb = cv2.ORB_create(10000)
orb.setFastThreshold(0)
if cv2.__version__.startswith('3'):
orb_matcher = cv2.BFMatcher(cv2.NORM_HAMMING)
else:
orb_matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING)
orb_gms = GmsMatcher(orb, orb_matcher)
for method_name in comp_list:
# Check res_dir if we have the dump ready
dump_dir = os.path.join(res_dir, mode, method_name)
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
# Check for full dump
full_dump_file = os.path.join(dump_dir, "qtn_all.h5")
if not os.path.exists(full_dump_file) or config.vis_dump:
for _idx in xrange(num_sample):
print("\rWorking on {} / {}".format(_idx + 1, num_sample),
end="")
sys.stdout.flush()
dump_file = os.path.join(dump_dir, "qtn_{}.txt".format(_idx))
# If dump exists, just load it
if os.path.exists(dump_file) and not config.vis_dump:
with open(dump_file, "r") as ifp:
dump_res = ifp.read()
dump_res = parse("{err_q:e}, {err_t:e}, {num_inlier:d}\n",
dump_res)
_err_q = dump_res["err_q"]
_err_t = dump_res["err_t"]
_num_inlier = dump_res["num_inlier"]
else:
# Otherwise compute
_xs = xs[_idx][:, :, :].reshape(1, 1, -1, 4)
_ys = ys[_idx][:, :].reshape(1, -1, 2)
_dR = Rs[_idx]
_dt = ts[_idx]
# Eval decompose for all pairs
_xs = _xs.reshape(-1, 4)
# x coordinates
_x1 = _xs[:, :2]
_x2 = _xs[:, 2:]
# Prepare input
if method_name == "lmeds":
eval_func = eval_decompose
_method = cv2.LMEDS
_mask = None
elif method_name == "ransac":
eval_func = eval_decompose
_method = cv2.RANSAC
_mask = None
elif method_name == "mlesac":
eval_func = eval_decompose
_method = "MLESAC"
_mask = None
elif method_name == "gms":
eval_func = eval_decompose_8points
_method = None
sift_gms.empty_matches()
_x1, _x2, _mask = sift_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=True,
with_rotation=True)
elif method_name == "gms_default":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_x1, _x2, _mask = orb_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb_resize_ransac_tt":
eval_func = eval_decompose
_method = cv2.RANSAC
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=True,
with_rotation=True)
elif method_name == "gms_orb_resize_tt":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=True,
with_rotation=True)
elif method_name == "gms_orb_resize_ransac":
eval_func = eval_decompose
_method = cv2.RANSAC
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb_resize":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_x1, _x2, _mask = orb_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=True,
with_rotation=True)
# Compute errors
_err_q, _err_t, _, _, _num_inlier, _mask_after = eval_func(
_x1, _x2, _dR, _dt, mask=_mask, method=_method)
if config.vis_dump:
dump_val_res(
img1s[_idx],
img2s[_idx],
_x1,
_x2,
_mask,
_mask_after,
cx1s[_idx],
cy1s[_idx],
f1s[_idx],
cx2s[_idx],
cy2s[_idx],
f2s[_idx],
Rs[_idx],
ts[_idx],
os.path.join(res_dir, mode, "match", method_name,
"pair{:08d}".format(_idx)),
)
else:
# Write dump
with open(dump_file, "w") as ofp:
ofp.write("{:e}, {:e}, {:d}\n".format(
_err_q, _err_t, _num_inlier))
# Load them in list
err_q[method_name][_idx] = _err_q
err_t[method_name][_idx] = _err_t
num[method_name][_idx] = _num_inlier
if not config.vis_dump:
# Save to full dump
dump_dict = {}
dump_dict["err_q"] = err_q[method_name]
dump_dict["err_t"] = err_t[method_name]
dump_dict["num"] = num[method_name]
saveh5(dump_dict, full_dump_file)
# Remove all intermediate cache
for _f in os.listdir(dump_dir):
if _f.startswith("qtn_") and _f.endswith(".txt"):
os.remove(os.path.join(dump_dir, _f))
# Load the full dump file
else:
dump_dict = loadh5(full_dump_file)
err_q[method_name] = dump_dict["err_q"]
err_t[method_name] = dump_dict["err_t"]
num[method_name] = dump_dict["num"]
# Remove all intermediate cache
for _f in os.listdir(dump_dir):
if _f.startswith("qtn_") and _f.endswith(".txt"):
os.remove(os.path.join(dump_dir, _f))
print("")
if config.vis_dump:
return
# Report results
for _tag in comp_list:
# For median error
ofn = os.path.join(res_dir, mode, "median_err_q_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(err_q[_tag])))
ofn = os.path.join(res_dir, mode, "median_err_t_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(err_t[_tag])))
ofn = os.path.join(res_dir, mode, "median_num_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(num[_tag])))
# For accuracy AUC
ths = np.arange(7) * 5
cur_err_q = np.array(err_q[_tag]) * 180.0 / np.pi
cur_err_t = np.array(err_t[_tag]) * 180.0 / np.pi
# Get histogram
q_acc_hist, _ = np.histogram(cur_err_q, ths)
t_acc_hist, _ = np.histogram(cur_err_t, ths)
qt_acc_hist, _ = np.histogram(np.maximum(cur_err_q, cur_err_t), ths)
num_pair = float(len(cur_err_q))
q_acc_hist = q_acc_hist.astype(float) / num_pair
t_acc_hist = t_acc_hist.astype(float) / num_pair
qt_acc_hist = qt_acc_hist.astype(float) / num_pair
q_acc = np.cumsum(q_acc_hist)
t_acc = np.cumsum(t_acc_hist)
qt_acc = np.cumsum(qt_acc_hist)
for _idx_th in xrange(1, len(ths)):
# for q_auc
ofn = os.path.join(res_dir, mode,
"acc_q_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(q_acc[:_idx_th])))
# for t_auc
ofn = os.path.join(res_dir, mode,
"acc_t_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(t_acc[:_idx_th])))
# for qt_auc
ofn = os.path.join(
res_dir, mode,
"acc_qt_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(qt_acc[:_idx_th])))
print("[{}] {}: End testing".format(config.data_tr, time.asctime()))
def _compute_desc(self):
"""Compute Descriptors """
total_time = 0.0
global_start_time = time.time()
for scene_name in cst.SCENE_LIST:
print('scene_name: %s'%scene_name)
img_dir = '%s/%s/'%(cst.DATA_DIR, scene_name)
img_list = [ l for l in sorted(os.listdir(img_dir)) if l[-3:]=='png']
img_num = len(img_list)
duration = time.time() - global_start_time
print('****** %s ****** %d:%02d'%(scene_name, duration/60,
duration%60))
if not os.path.exists(os.path.join(cst.DES_DIR, scene_name)):
os.makedirs(os.path.join(cst.DES_DIR, scene_name))
for img_id in range(img_num):
root_name = '%04d'%img_id
img_fn = os.path.join(img_dir, '%s.png'%root_name)
new_kp_fn = os.path.join(cst.ORI_DIR, scene_name,'%s.txt'%(root_name))
des_fn = os.path.join(cst.DES_DIR, scene_name, '%s.h5'%(root_name))
# Read image
start_time = time.clock()
cur_data = self.dataset.my_load_data(img_fn, new_kp_fn,
cst.NEW_SIZE)
end_time = time.clock()
load_time = (end_time - start_time) * 1000.0
#print("Time taken to load patches is {} ms".format(
# load_time
#))
total_time += load_time
# import IPython
# IPython.embed()
# -------------------------------------------------------------------------
# Test using the test function
start_time = time.clock()
descs = self._test_multibatch(cur_data)
end_time = time.clock()
compute_time = (end_time - start_time) * 1000.0
#print("Time taken to compute is {} ms".format(
# compute_time
#))
total_time += compute_time
#print("Total time for descriptor is {} ms".format(total_time))
# Overwrite angle
kps = cur_data["kps"].copy()
kps[:, 3] = cur_data["angle"][:, 0]
# Save as h5 file
save_dict = {}
# save_dict['keypoints'] = cur_data["kps"]
save_dict['keypoints'] = kps
save_dict['descriptors'] = descs
saveh5(save_dict, des_fn)
def get_list_of_img(train_data_dir, dump_data_dir, param, mode):
# Check if split file exists
split_prefix = train_data_dir + 'split-'
split_prefix += str(param.dataset.nTrainPercent) + '-'
split_prefix += str(param.dataset.nValidPercent) + '-'
split_prefix += str(param.dataset.nTestPercent) + '-'
# If it does not exist, create one
if not os.path.exists(split_prefix + mode + '.txt'):
# Read list of images
list_png_file = []
for files in os.listdir(train_data_dir):
if files.endswith(".png"):
list_png_file = list_png_file + [files]
# Shuffle the image list
if not os.path.exists(dump_data_dir + 'permute_png_idx.h5'):
print(' -- ' + mode + ': '
'Creating new shuffle for reading images')
permute_png_idx = np.random.permutation(len(list_png_file))
to_save = {"saveval": permute_png_idx}
saveh5(to_save, dump_data_dir + 'permute_png_idx.h5')
# dt.save(permute_png_idx,
# dump_data_dir + 'permute_png_idx.h5')
else:
print(' -- ' + mode + ': '
'Loading shuffle for reading images from '
'{}'.format(dump_data_dir))
to_load = loadh5(dump_data_dir + 'permute_png_idx.h5')
permute_png_idx = to_load["saveval"]
# permute_png_idx = dt.load(dump_data_dir +
# 'permute_png_idx.h5')
list_png_file = [
list_png_file[permute_png_idx[idx]]
for idx in range(len(list_png_file))
]
# Write to file (all three)
f_train = open(split_prefix + 'train.txt', 'w')
f_valid = open(split_prefix + 'valid.txt', 'w')
f_test = open(split_prefix + 'test.txt', 'w')
train_end = int(
float(param.dataset.nTrainPercent) / 100.0 * len(list_png_file))
valid_end = int(
float(param.dataset.nTrainPercent + param.dataset.nValidPercent) /
100.0 * len(list_png_file))
for idx_png in six.moves.xrange(len(list_png_file)):
if idx_png > valid_end:
print(list_png_file[idx_png], file=f_test)
elif idx_png > train_end:
print(list_png_file[idx_png], file=f_valid)
else:
print(list_png_file[idx_png], file=f_train)
f_train.close()
f_valid.close()
f_test.close()
# Read the list
list_png_file = list(np.loadtxt(split_prefix + mode + '.txt', dtype='str'))
return list_png_file
def _create_pairs(self, task, use_cache=True, overwrite=False):
"""Generate the pairs from ID
This function should return the pair indices given the task. The return
type is expected to be a dictionary, where you can access by doing
res["P1"], for example.
"""
# Use the cache file if asked
if use_cache:
pairs_file = os.path.join(
self.pair_dir, "{}.h5".format(task))
if not os.path.exists(self.pair_dir):
os.makedirs(self.pair_dir)
if os.path.exists(pairs_file) and not overwrite:
if self.config.use_augmented_set:
# Add rotation augmentation if it does not exist (compat)
_f = h5py.File(pairs_file, "r+")
for name in ["P1", "P2", "P3", "P4"]:
if (name + "_rot_aug") not in _f:
_f.create_dataset(
name + "_rot_aug", data=2 *
np.pi * self.rng.rand(_f[name].size))
_f.close()
return loadh5(pairs_file)
# Make a lookup table for getting the indices of each sample. This can
# be easily done by sorting by ID, and storing the indices to a
# dictionary
if self.LUT_kp[task] is None:
# Create empty dictionary
self.LUT_kp[task] = {}
self.LUT_nonkp[task] = {}
if hasattr(self, "th_dist"):
self.LUT_below_th[task] = {}
# Build list of indices
dist_idx_reverse = np.zeros(
self.data[task]["ID"].max() + 1,
dtype=np.int64)
dist_idx_reverse[self.data[task]["dist_idx"]] = range(
1, self.data[task]["dist_idx"].size + 1)
dist_idx_reverse -= 1
# Argsort the ID
print("[{}] sorting IDs...".format(task))
ID = self.data[task]["ID"]
ind = np.argsort(ID)
# Go through them one by one -- the easy way
print("[{}] building LUT...".format(task))
start = 0
start_ID = ID[ind[start]]
for end in xrange(1, len(ind)):
if end % 100 == 0:
print("\r --- working on {}/{}".format(
end + 1, len(ind)), end="")
sys.stdout.flush()
# Check if indices have changed or we reached the end
if start_ID != ID[ind[end]] or end == len(ind) - 1:
# Store them in proper LUT
if start_ID < 0:
self.LUT_nonkp[task][-1] = ind[start:end]
else:
self.LUT_kp[task][start_ID] = ind[start:end]
if hasattr(self, "th_dist"):
v = dist_idx_reverse[start_ID]
assert v >= 0, "Invalid index"
d = self.data[task]["dist_mat"][v]
# 3D points are 1-indexed
self.LUT_below_th[task][start_ID] = 1 + \
np.where((d > 0) & (d < self.th_dist))[0]
# Update start position
start = end
start_ID = ID[ind[start]]
print("\r --- done. ")
# The dictionary to return
cur_pairs = {}
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name] = []
# For each kp item, create a random pair
#
# Note that this is different from what we used to do. This way seems
# better as we will look at each keypoint once.
print("[{}] creating pairs...".format(task))
num_kp = len(self.LUT_kp[task])
for i in xrange(num_kp):
if i % 100 == 0:
print("\r --- working on {}/{}".format(
i + 1, num_kp), end="")
sys.stdout.flush()
_kp = list(self.LUT_kp[task].keys())[i]
# Check if we have enough views of this point and skip
if len(self.LUT_kp[task][_kp]) < 2:
continue
# For P1 and P2 -- Select two random patches
P1, P2 = self.rng.choice(
self.LUT_kp[task][_kp], 2, replace=False)
# For P3 -- Select a different keypoint randomly
# Generate a list of keys
valid_keys = set(self.LUT_kp[task])
# Remove self
valid_keys -= set([_kp])
# Remove points below the distance threshold
if hasattr(self, "th_dist"):
valid_keys -= set(valid_keys.intersection(
self.LUT_below_th[task][_kp]))
_kp2 = self.rng.choice(list(valid_keys))
P3 = self.rng.choice(self.LUT_kp[task][_kp2], 1)[0]
# For P4 -- Select a random nonkp
P4 = self.rng.choice(self.LUT_nonkp[task][-1], 1)[0]
# Append to the list
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name].append(eval(name))
print("\r --- done. ")
# Convert them to np arrays
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name] = np.asarray(cur_pairs[name])
# Augment pairs with rotation data
if self.config.use_augmented_set:
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name + "_rot_aug"] = 2 * np.pi * \
self.rng.rand(cur_pairs[name].size)
# Save to cache if asked
if use_cache:
if not os.path.exists(self.pair_dir):
os.makedirs(self.pair_dir)
saveh5(cur_pairs, pairs_file)
return cur_pairs
