def __init__(self, config, subtask, dataset, tfconfig):
logdir = os.path.join(config.logdir, subtask)
self.config = copy.deepcopy(config)
self.config.subtask = subtask
self.graph = tf.Graph()
self.sess = tf.Session(graph=self.graph, config=tfconfig)
with self.graph.as_default():
if os.path.exists(os.path.join(logdir, "mean.h5")):
training_mean = loadh5(os.path.join(logdir, "mean.h5"))
training_std = loadh5(os.path.join(logdir, "std.h5"))
print("[{}] Loaded input normalizers for testing".format(
subtask))
# Create the model instance
self.network = Network(self.sess, self.config, dataset, {
'mean': training_mean, 'std': training_std})
else:
self.network = Network(self.sess, self.config, dataset)
self.saver = {}
self.best_val_loss = {}
self.best_step = {}
# Create the saver instance for both joint and the current subtask
for _key in ["joint", subtask]:
self.saver[_key] = tf.train.Saver(self.network.allparams[_key])
# We have everything ready. We finalize and initialie the network here.
self.sess.run(tf.global_variables_initializer())
restore_res = self.restore_network()
if not restore_res:
raise RuntimeError("Could not load network weights!")
def make_xy(self, ii, jj):
geom_i, geom_j = parse_geom(self.geom[ii]), parse_geom(self.geom[jj])
# should check the image size here
#load img and check img_size
image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj]
kp_i = loadh5(image_i+'.'+self.desc_name+'.hdf5')["keypoints"][:, :2]
kp_j = loadh5(image_j+'.'+self.desc_name+'.hdf5')["keypoints"][:, :2]
cx1, cy1, f1 = self.unpack_K(geom_i)
cx2, cy2, f2 = self.unpack_K(geom_j)
x1 = self.norm_kp(cx1, cy1, f1[0], f1[1], kp_i)
x2 = self.norm_kp(cx2, cy2, f2[0], f2[1], kp_j)
R_i, R_j = geom_i["R"], geom_j["R"]
dR = np.dot(R_j, R_i.T)
t_i, t_j = geom_i["t"].reshape([3, 1]), geom_j["t"].reshape([3, 1])
dt = t_j - np.dot(dR, t_i)
if np.sqrt(np.sum(dt**2)) <= 1e-5:
return []
dtnorm = np.sqrt(np.sum(dt**2))
dt /= dtnorm
nn_info = loadh5(os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj)))
idx_sort, ratio_test, mutual_nearest = nn_info["idx_sort"], nn_info["ratio_test"], nn_info["mutual_nearest"]
x2 = x2[idx_sort[1],:]
xs = np.concatenate([x1, x2], axis=1).reshape(1,-1,4)
geod_d = get_episym(x1, x2, dR, dt)
ys = geod_d.reshape(-1,1)
return xs, ys, dR, dt, ratio_test, mutual_nearest, cx1, cy1, f1, cx2, cy2, f2
def load_legacy_network(self, load_dir):
"""Load function for our old framework"""
print("[{}] Checking if old pre-trained weights exists in {}"
"".format(self.config.subtask, load_dir))
model_file = os.path.join(load_dir, "model.h5")
norm_file = os.path.join(load_dir, "norm.h5")
base_file = os.path.join(load_dir, "base.h5")
if os.path.exists(model_file) and os.path.exists(norm_file) and \
os.path.exists(base_file):
model = loadh5(model_file)
norm = loadh5(norm_file)
base = loadh5(base_file)
# Load the input normalization parameters.
with self.graph.as_default():
self.network.mean["kp"] = float(norm["mean_x"])
self.network.mean["ori"] = float(norm["mean_x"])
self.network.mean["desc"] = float(base["patch-mean"])
self.network.std["kp"] = float(norm["std_x"])
self.network.std["ori"] = float(norm["std_x"])
self.network.std["desc"] = float(base["patch-std"])
# Load weights for the component
self.network.legacy_load_func[self.config.subtask](self.sess, model)
print("[{}] Loaded previously trained weights".format(self.config.subtask))
return True
else:
print("[{}] No pretrained weights from the old framework"
"".format(self.config.subtask))
return False
def load_legacy_network(supervisor, subtask, load_dir):
"""Load function for our old framework"""
# Lazy loading to prevent import issues
from utils import loadh5
print("[{}] Checking if old pre-trained weights exists in {}"
"".format(subtask, load_dir))
model_file = os.path.join(load_dir, "model.h5")
norm_file = os.path.join(load_dir, "norm.h5")
base_file = os.path.join(load_dir, "base.h5")
if os.path.exists(model_file) and os.path.exists(norm_file) and \
os.path.exists(base_file):
model = loadh5(model_file)
norm = loadh5(norm_file)
base = loadh5(base_file)
# Load the input normalization parameters
supervisor.network.mean["kp"] = float(norm["mean_x"])
supervisor.network.mean["ori"] = float(norm["mean_x"])
supervisor.network.mean["desc"] = float(base["patch-mean"])
supervisor.network.std["kp"] = float(norm["std_x"])
supervisor.network.std["ori"] = float(norm["std_x"])
supervisor.network.std["desc"] = float(base["patch-std"])
# Load weights for the component
supervisor.network.legacy_load_func[subtask](supervisor.sess, model)
print("[{}] Loaded previously trained weights".format(subtask))
return True
else:
print("[{}] No pretrained weights from the old framework"
"".format(subtask))
return False
def dump_data_pair(args):
dump_dir, idx, ii, jj, queue = args
# queue for monitoring
if queue is not None:
queue.put(idx)
dump_file = os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj))
# txt_dict = os.path.join(dump_dir, "idx_sort-{}-{}.txt".format(ii, jj))
if not os.path.exists(dump_file):
# Load descriptors for ii
desc_ii = loadh5(os.path.join(dump_dir,
"kp-aff-desc-{}.h5".format(ii)))["desc"]
desc_jj = loadh5(os.path.join(dump_dir,
"kp-aff-desc-{}.h5".format(jj)))["desc"]
# compute decriptor distance matrix
distmat = np.sqrt(
np.sum(
(np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2,
axis=2))
# Choose K best from N
idx_sort = np.argsort(distmat, axis=1)[:, :config.obj_num_nn]
idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],
idx_sort.shape[1],
axis=1), idx_sort)
distmat = distmat[idx_sort]
# Dump to disk
dump_dict = {}
dump_dict["idx_sort"] = idx_sort
saveh5(dump_dict, dump_file)
def dump_data_pair(args):
dump_dir, idx, ii, jj, queue = args
# queue for monitoring
if queue is not None:
queue.put(idx)
dump_file = os.path.join(
dump_dir, "idx_sort-{}-{}.h5".format(ii, jj))
dump_file_mutual_ratio = os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj))
if not os.path.exists(dump_file) or not os.path.exists(dump_file_mutual_ratio):
# if 1==1:
# Load descriptors for ii
desc_ii = loadh5(
os.path.join(dump_dir, "kp-z-desc-{}.h5".format(ii)))["desc"]
desc_jj = loadh5(
os.path.join(dump_dir, "kp-z-desc-{}.h5".format(jj)))["desc"]
# compute decriptor distance matrix
# distmat = np.sqrt(
# np.sum(
# (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2,
# axis=2))
##### replace with faster distmat computation
distmat = np.sqrt(np.sum(desc_ii**2, axis=1, keepdims=True) +
np.sum(desc_jj**2, axis=1) -
2 * np.dot(desc_ii, desc_jj.T))
# Choose K best from N
idx_sort0 = np.argsort(distmat, axis=1)[:, :1]
idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort0.shape[1], axis=1),idx_sort0)
# saving the ratio
idx_sort_2nd = np.argsort(distmat, axis=1)[:, :2]
idx_sort_2nd = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort_2nd.shape[1], axis=1),idx_sort_2nd)
dist2nd = distmat[idx_sort_2nd]
ratio = dist2nd[:, 0] / dist2nd[:, 1]
# saving mutual neighbor information
# Please note that crossCheck in opencv doesn't work properly and thus is deprecated in this code.
idx_sort1 = np.argsort(distmat, axis=0)[0, :]
mutual_neigh = idx_sort1[idx_sort0.squeeze()] == np.arange(idx_sort0.shape[0])
# Dump to disk
dump_dict = {}
dump_dict_mutual_ratio = {}
dump_dict["idx_sort"] = idx_sort
dump_dict_mutual_ratio["ratio"] = ratio
dump_dict_mutual_ratio["mutual"] = mutual_neigh
# Looks ugly because we just add ratio and mutual neighboring information to CNe dataset
if not os.path.exists(dump_file):
saveh5(dump_dict, dump_file)
if not os.path.exists(dump_file_mutual_ratio):
saveh5(dump_dict_mutual_ratio, dump_file_mutual_ratio)
def dump_nn(self, ii, jj):
dump_file = os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj))
if not os.path.exists(dump_file):
image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj]
desc_ii = loadh5(image_i+'.'+self.desc_name+'.hdf5')["descriptors"]
desc_jj = loadh5(image_j+'.'+self.desc_name+'.hdf5')["descriptors"]
idx_sort, ratio_test, mutual_nearest = computeNN(desc_ii, desc_jj)
# Dump to disk
dump_dict = {}
dump_dict["idx_sort"] = idx_sort
dump_dict["ratio_test"] = ratio_test
dump_dict["mutual_nearest"] = mutual_nearest
saveh5(dump_dict, dump_file)
def load_geom(geom_file, scale_factor=1.0, flip_R=False):
# load geometry file
geom_dict = loadh5(geom_file)
# Check if principal point is at the center
K = geom_dict["K"]
# assert(abs(K[0, 2]) < 1e-3 and abs(K[1, 2]) < 1e-3)
# Rescale calbration according to previous resizing
S = np.asarray([[scale_factor, 0, 0],
[0, scale_factor, 0],
[0, 0, 1]])
K = np.dot(S, K)
geom_dict["K"] = K
# Transpose Rotation Matrix if needed
if flip_R:
R = geom_dict["R"].T.copy()
geom_dict["R"] = R
# append things to list
geom_list = []
geom_info_name_list = ["K", "R", "T", "imsize"]
for geom_info_name in geom_info_name_list:
geom_list += [geom_dict[geom_info_name].flatten()]
# Finally do K_inv since inverting K is tricky with theano
geom_list += [np.linalg.inv(geom_dict["K"]).flatten()]
# Get the quaternion from Rotation matrices as well
q = quaternion_from_matrix(geom_dict["R"])
geom_list += [q.flatten()]
# Also add the inverse of the quaternion
q_inv = q.copy()
np.negative(q_inv[1:], q_inv[1:])
geom_list += [q_inv.flatten()]
# Add to list
geom = np.concatenate(geom_list)
return geom
def __init__(self, config, rng):
self.config = config
self.rng = rng
# Open a tensorflow session. I like keeping things simple, so I don't
# use a supervisor. I'm just going to do everything manually. I also
# will just allow the gpu memory to grow
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
self.sess = tf.Session(config=tfconfig)
# Create the dataset instance
self.dataset = Dataset(self.config, rng)
# Retrieve mean/std (yes it is hacky)
logdir = os.path.join(self.config.logdir, self.config.subtask)
if os.path.exists(os.path.join(logdir, "mean.h5")):
training_mean = loadh5(os.path.join(logdir, "mean.h5"))
training_std = loadh5(os.path.join(logdir, "std.h5"))
print("[{}] Loaded input normalizers for testing".format(
self.config.subtask))
# Create the model instance
self.network = Network(self.sess, self.config, self.dataset, {
'mean': training_mean,
'std': training_std
})
else:
self.network = Network(self.sess, self.config, self.dataset)
# Make individual saver instances for each module.
self.saver = {}
self.best_val_loss = {}
self.best_step = {}
# Create the saver instance for both joint and the current subtask
for _key in ["joint", self.config.subtask]:
self.saver[_key] = tf.train.Saver(self.network.allparams[_key])
#print('\nNETWORK PARAMETERS')
#for item in self.network.allparams[_key]:
# print(item)
# We have everything ready. We finalize and initialie the network here.
self.sess.run(tf.global_variables_initializer())
def load_network(self, load_dir):
"""Load function for our new framework"""
print("[{}] Checking if previous Tensorflow run exists in {}"
"".format(self.config.subtask, load_dir))
latest_checkpoint = tf.train.latest_checkpoint(load_dir)
if latest_checkpoint is not None:
# Load parameters
with self.graph.as_default():
self.saver[self.config.subtask].restore(
self.sess,
latest_checkpoint
)
print("[{}] Loaded previously trained weights".format(self.config.subtask))
# Save mean std (falls back to default if non-existent)
if os.path.exists(os.path.join(load_dir, "mean.h5")):
self.network.mean = loadh5(os.path.join(load_dir, "mean.h5"))
self.network.std = loadh5(os.path.join(load_dir, "std.h5"))
print("[{}] Loaded input normalizers".format(self.config.subtask))
# Load best validation result
self.best_val_loss[self.config.subtask] = loadh5(
os.path.join(load_dir, best_val_loss_filename)
)[self.config.subtask]
print("[{}] Loaded best validation result = {}".format(
self.config.subtask,self.best_val_loss[self.config.subtask]))
# Load best validation result
self.best_step[self.config.subtask] = loadh5(
os.path.join(load_dir, best_step_filename)
)[self.config.subtask]
print("[{}] Loaded best step = {}".format(
self.config.subtask, self.best_step[self.config.subtask]))
return True
else:
print("[{}] No previous Tensorflow result".format(self.config.subtask))
return False
def load_data_for_set(self, pathconf, param, mode):
# ----------------------------------------------------------------------
# Train, Validation, and Test
# mlab = matlab.engine.start_matlab()
# Read from pathconf
# Original implementation
train_data_dir = os.path.normpath(pathconf.dataset)
dump_data_dir = os.path.normpath(pathconf.train_dump)
dump_patch_dir = os.path.normpath(pathconf.patch_dump)
# local (or volatile) copy of the dump data
tmp_patch_dir = os.path.normpath(pathconf.volatile_patch_dump)
# print("train_data_dir = {}".format(train_data_dir))
# print("dump_data_dir = {}".format(dump_data_dir))
# print("dump_patch_dir = {}".format(dump_patch_dir))
# print("tmp_patch_dir = {}".format(tmp_patch_dir))
if not os.path.exists(dump_data_dir):
os.makedirs(dump_data_dir)
if not os.path.exists(dump_patch_dir):
os.makedirs(dump_patch_dir)
if not os.path.exists(tmp_patch_dir):
os.makedirs(tmp_patch_dir)
# Check if we have the big h5 file ready
big_file_name = dump_patch_dir + mode + "-data-chunked.h5"
# if os.getenv("MLTEST_DEBUG", default=""):
# import pdb
# pdb.set_trace()
# Mutex lock
#
# We will create an nfs-safe lock file in a temporary directory to
# prevent our script from using corrupted, or data that is still being
# generated. This allows us to launch multiple instances at the same
# time, and allow only a single instance to generate the big_file.
if not os.path.exists(".locks"):
os.makedirs(".locks")
check_lock_file = ".locks/" + \
hashlib.md5(big_file_name.encode()).hexdigest()
if os.name == "posix":
check_lock = Lock(check_lock_file)
check_lock.lifetime = timedelta(days=2)
frameinfo = getframeinfo(currentframe())
print("-- {}/{}: waiting to obtain lock --".format(
frameinfo.filename, frameinfo.lineno))
check_lock.lock()
print(">> obtained lock for posix system<<")
elif os.name == "nt":
import filelock
check_lock = filelock.FileLock(check_lock_file)
check_lock.timeout = 2000
check_lock.acquire()
if check_lock.is_locked:
print(">> obtained lock for windows system <<")
else:
print("Unknown operating system, lock unavailable")
# if the large training data file does not exist
if not os.path.exists(big_file_name):
print("big data file does not exist...")
# if the patch-mode-data file does not exist
if not os.path.exists(
os.path.join(dump_patch_dir, mode + "-data.h5")):
print("{0} does not exist...".format(
os.path.join(dump_patch_dir, mode + "-data.h5")))
# Read scale histogram
hist_file_path = train_data_dir + "scales-histogram-minsc-" + str(
param.dataset.fMinKpSize) + ".h5"
if not os.path.exists(hist_file_path):
print("Hist file does not exist, creating...")
get_scale_hist(train_data_dir, param)
# print("Loading hist file...")
hist_file = h5py.File(hist_file_path, "r")
scale_hist = np.asarray(hist_file["histogram_bins"],
dtype=float).flatten()
# print(scale_hist)
scale_hist /= np.sum(scale_hist)
scale_hist_c = np.asarray(
hist_file["histogram_centers"]).flatten()
# Read list of images from split files
split_name = ""
split_name += str(param.dataset.nTrainPercent) + "-"
split_name += str(param.dataset.nValidPercent) + "-"
split_name += str(param.dataset.nTestPercent) + "-"
if mode == "train":
# split_name += "train-"
split_name += "train"
elif mode == "valid":
# split_name += "val-"
split_name += "val"
elif mode == "test":
# split_name += "test-"
split_name += "test"
print("split_name: {}".format(split_name))
# split_file_name = train_data_dir + "split-" \
# + split_name + "minsc-" \
# + str(param.dataset.fMinKpSize) + ".h.txt"
# split_file_name = "split-" + split_name + "minsc-" + str(param.dataset.fMinKpSize) + ".h.txt"
split_file_name = "split-" + split_name + ".txt"
split_file_name = train_data_dir + split_file_name
# split_file_name = os.path.join(train_data_dir, split_file_name)
print("split_file_name: {}".format(split_file_name))
if not os.path.exists(split_file_name):
print("split_file_name does not exist...")
list_jpg_file = get_list_of_img(train_data_dir,
dump_data_dir, param, mode)
else:
print("split_file_name exists...")
list_jpg_file = []
for file_name in list(
np.loadtxt(split_file_name, dtype=bytes)):
list_jpg_file += [
file_name.decode("utf-8").replace(
"-kp-minsc-" + str(param.dataset.fMinKpSize),
".jpg")
]
# -------------------------------------------------
# Create dumps in parallel
# I am lazy so create arguments in loop lol
pool_arg = [None] * len(list_jpg_file)
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
pool_arg[idx_jpg] = (idx_jpg, list_jpg_file[idx_jpg],
train_data_dir, dump_data_dir,
tmp_patch_dir, scale_hist,
scale_hist_c, self.out_dim, param)
# # if true, use multi thread, otherwise use only single thread
prod = True
if prod:
number_of_process = int(ratio_CPU * mp.cpu_count())
pool = mp.Pool(processes=number_of_process)
manager = mp.Manager()
queue = manager.Queue()
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
pool_arg[idx_jpg] = pool_arg[idx_jpg] + (queue, )
# map async
pool_res = pool.map_async(createDump, pool_arg)
# pool_res = pool.map_async(createDump, pool_arg, chunksize = int(len(list_jpg_file)/(number_of_process* mp.cpu_count())))
# monitor loop
while True:
if pool_res.ready():
print("Pool_res ready?")
break
else:
size = queue.qsize()
print("\r -- " + mode +
": Processing image {}/{}".format(
size, len(list_jpg_file)),
end="")
# print(list_jpg_file[size])
sys.stdout.flush()
time.sleep(1)
pool.close()
pool.join()
print("\r -- " + mode + ": Finished Processing Images!")
# for debugging, if multi thread is used, then it is difficult
# to debug
else:
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
pool_arg[idx_jpg] = pool_arg[idx_jpg] + (None, )
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
createDump(pool_arg[idx_jpg])
print("\r -- " + mode + ": Processing image "
"{}/{}".format(idx_jpg + 1, len(list_jpg_file)),
end="")
sys.stdout.flush()
print("\r -- " + mode + ": Finished Processing Images!")
# -------------------------------------------------
# # --------------------
# use single thread for simplify debugging
# for idx_jpg in six.moves.xrange(len(list_jpg_file)):
# pool_arg[idx_jpg] = pool_arg[idx_jpg] + (None,)
# for idx_jpg in six.moves.xrange(len(list_jpg_file)):
# createDump(pool_arg[idx_jpg])
# print("\r -- " + mode + ": Processing image "
# "{}/{}".format(idx_jpg + 1, len(list_jpg_file)),
# end="")
# sys.stdout.flush()
# print("\r -- " + mode + ": Finished Processing Images!")
# ------------------------------------------------------------------
# Use only valid indices to ascertain mutual exclusiveness
id_file_name = train_data_dir + "split-"
id_file_name += str(param.dataset.nTrainPercent) + "-"
id_file_name += str(param.dataset.nValidPercent) + "-"
id_file_name += str(param.dataset.nTestPercent) + "-"
id_file_name += ("minsc-" + str(param.dataset.fMinKpSize) +
".h5")
if mode == "train":
id_key = "indices_train"
elif mode == "valid":
id_key = "indices_val"
elif mode == "test":
id_key = "indices_test"
# print(id_file_name)
try:
with h5py.File(id_file_name, "r") as id_file:
id_2_keep = np.asarray(id_file[id_key])
except OSError as err:
print(err)
print("Creating idx file...")
# if "unable to open file" in err:
createsplitindexh5file(id_file_name, train_data_dir, param)
with h5py.File(id_file_name, "r") as id_file:
id_2_keep = np.asarray(id_file[id_key])
# print(id_2_keep)
print("{0} has {1} sfmid points to keep...".format(
id_key, len(id_2_keep)))
# exit()
# ind_2_keep = np.in1d(dataset[2], id_2_keep)
# ind_2_keep += dataset[2] < 0
# loop through files to figure out how many valid items we have
# pdb.set_trace() # for tracking of the dataset
num_valid = 0
# print(len(list_jpg_file))
# exit()
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
jpg_file = list_jpg_file[idx_jpg]
print("\r -- " + mode + ": "
"Reading dumps to figure out number of valid "
"{}/{}".format(idx_jpg + 1, len(list_jpg_file)),
end="")
sys.stdout.flush()
# Load created dump
# final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5")
# print(tmp_patch_dir)
# print(jpg_file)
final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename(
jpg_file)[:-4] + ".h5"
# print(final_dump_file_name)
# Use loadh5 and turn it back to original cur_data_set
try:
with h5py.File(final_dump_file_name, "r") as dump_file:
# print(list(dump_file.keys()))
cur_ids = dump_file["2"].value
# kps = dump_file["valid_keypoints"][()]
# cur_ids = np.asarray(kps[:, 4])
# print(cur_ids)
except OSError as err:
# print(err)
continue
# Find cur valid by looking at id_2_keep
cur_valid = np.in1d(cur_ids, id_2_keep)
# print(cur_valid)
# Add all negative labels as valid (neg data)
cur_valid += cur_ids < 0
# Sum it up
num_valid += np.sum(cur_valid)
# print(num_valid)
print("\n -- " + mode + ": "
"Found {} valid data points from {} files"
"".format(num_valid, len(list_jpg_file)))
# Get the first data to simply check the shape
tmp_dump_file_name = tmp_patch_dir + "\\" + os.path.basename(
list_jpg_file[-1])[:-4] + ".h5"
with h5py.File(tmp_dump_file_name, "r") as dump_file:
dataset_shape = []
dataset_type = []
for _idx in six.moves.xrange(len(dump_file.keys())):
dataset_shape += [dump_file[str(_idx)].shape]
dataset_type += [dump_file[str(_idx)].dtype]
# create and save the large dataset chunk
with h5py.File(big_file_name, "w-") as big_file:
big_file["time_stamp"] = np.asarray(time.localtime())
name_list = ["x", "y", "ID", "pos", "angle", "coords"]
# create the dataset storage chunk
for __i in six.moves.xrange(len(dataset_shape)):
big_file.create_dataset(
name_list[__i],
(num_valid, ) + dataset_shape[__i][1:],
chunks=(1, ) + dataset_shape[__i][1:],
maxshape=((num_valid, ) + dataset_shape[__i][1:]),
dtype=dataset_type[__i])
# loop through the file to save to a big chunk
save_base = 0
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
jpg_file = list_jpg_file[idx_jpg]
print("\r -- " + mode + ": "
"Saving the data to the big dump "
"{}/{}".format(idx_jpg + 1, len(list_jpg_file)),
end="")
sys.stdout.flush()
# Load created dump
# final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5")
final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename(
jpg_file)[:-4] + ".h5"
# print(final_dump_file_name)
# Use loadh5 and turn it back to original cur_data_set
try:
tmpdict = loadh5(final_dump_file_name)
cur_data_set = tuple([
tmpdict[str(_idx)]
for _idx in range(len(tmpdict.keys()))
])
# Find cur valid by looking at id_2_keep
cur_valid = np.in1d(cur_data_set[2], id_2_keep)
# Add all negative labels as valid (neg data)
cur_valid += cur_data_set[2] < 0
for __i in six.moves.xrange(len(dataset_shape)):
big_file[name_list[__i]][
save_base:save_base +
np.sum(cur_valid
)] = cur_data_set[__i][cur_valid]
# Move base to the next chunk
save_base += np.sum(cur_valid)
except OSError as err:
# print(err)
# print("{0} skipped due to invalidity...".format(final_dump_file_name))
# sys.stdout.flush()
continue
# Assert that we saved all
assert save_base == num_valid
print("\n -- " + mode + ": "
"Done saving {} valid data points from {} files"
"".format(num_valid, len(list_jpg_file)))
# --------------------------------------------------
# Cleanup dump
for idx_jpg in six.moves.xrange(len(list_jpg_file)):
jpg_file = list_jpg_file[idx_jpg]
print("\r -- " + mode + ": "
"Removing dump "
"{}/{}".format(idx_jpg + 1, len(list_jpg_file)),
end="")
sys.stdout.flush()
# Delete dump
# final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5")
final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename(
jpg_file)[:-4] + ".h5"
try:
os.remove(final_dump_file_name)
except FileNotFoundError as err:
pass
print("\r -- " + mode + ": "
"Cleaned up dumps! "
"Local dump is now clean!")
else:
print(" -- Found old file without chunks. "
"Copying to new file with chunks...")
old_big_file_name = dump_patch_dir + mode + "-data.h5"
with h5py.File(old_big_file_name, "r") as old_big_file, \
h5py.File(big_file_name, "w-") as big_file:
dataset = []
# load old train into array
name_list = ["x", "y", "ID", "pos", "angle", "coords"]
for __i in six.moves.xrange(len(name_list)):
dataset += [np.asarray(old_big_file[name_list[__i]])]
# save train
big_file["time_stamp"] = np.asarray(time.localtime())
# allocate and write
for __i in six.moves.xrange(len(name_list)):
if name_list[__i] == "x":
chunk_shape = (1, ) + dataset[__i].shape[1:]
else:
chunk_shape = None
big_file.create_dataset(
name_list[__i],
dataset[__i].shape,
data=dataset[__i],
chunks=chunk_shape,
maxshape=dataset[__i].shape,
)
print(" -- Finished creating chunked file, removing old...")
os.remove(old_big_file_name)
# ----------------------------------------------------------------------
# Copy to local tmp if necessary
if not os.path.exists(tmp_patch_dir + mode + "-data-chunked.h5"):
print(" -- " + mode + ": "
"Local dump does not exist! "
"Copying big dump to local drive... ")
shutil.copy(dump_patch_dir + mode + "-data-chunked.h5",
tmp_patch_dir + mode + "-data-chunked.h5")
else:
print(" -- " + mode + ": "
"Local dump exists. Checking timestamp...")
# get timestamp from nfs
with h5py.File(dump_patch_dir + mode + "-data-chunked.h5", "r") \
as nfs_file:
nfs_time = np.asarray(nfs_file["time_stamp"])
# get timestamp from local
with h5py.File(tmp_patch_dir + mode + "-data-chunked.h5", "r") \
as local_file:
local_time = np.asarray(local_file["time_stamp"])
# if the two files have different time stamps
if any(nfs_time != local_time):
print(" -- " + mode + ": "
"Time stamps are different! "
"Copying big dump to local drive... ")
shutil.copy(dump_patch_dir + mode + "-data-chunked.h5",
tmp_patch_dir + mode + "-data-chunked.h5")
else:
print(" -- " + mode + ": "
"Time stamps are identical! Re-using local dump")
# Free lock
if os.name == "posix":
check_lock.unlock()
print("-- free lock --")
elif os.name == "nt":
check_lock.release()
print("-- free lock --")
else:
pass
# ----------------------------------------------------------------------
# Use local copy for faster speed
print(" -- " + mode + ": Loading from local drive... ")
big_file_name = tmp_patch_dir + mode + "-data-chunked.h5"
# open big_file and don"t close
big_file = h5py.File(big_file_name, "r")
x = big_file["x"]
# work arround for h5py loading all things to memory
read_batch_size = 10000
read_batch_num = int(
np.ceil(float(big_file["x"].shape[0]) / float(read_batch_size)))
# Manual, since I don't want to bother debugging the below
# fields = ["y", "ID", "pos", "angle", "coords"]
# for var_name in fields:
# # allocate data
# exec("{0} = np.zeros(big_file['{0}'].shape, "
# "dtype=big_file['{0}'].dtype)".format(var_name))
# # copy data in batches
# for idx_batch in six.moves.xrange(read_batch_num):
# idx_s = idx_batch * read_batch_size
# idx_e = (idx_batch + 1) * read_batch_size
# idx_e = np.minimum(idx_e, big_file["x"].shape[0])
# exec("{0}[idx_s:idx_e] = np.asarray(big_file['{0}'][idx_s:idx_e])"
# "". format(var_name))
# Allocate
y = np.zeros(big_file["y"].shape, dtype=big_file["y"].dtype)
ID = np.zeros(big_file["ID"].shape, dtype=big_file["ID"].dtype)
pos = np.zeros(big_file["pos"].shape, dtype=big_file["pos"].dtype)
angle = np.zeros(big_file["angle"].shape,
dtype=big_file["angle"].dtype)
coords = np.zeros(big_file["coords"].shape,
dtype=big_file["coords"].dtype)
# Copy data in batches
for idx_batch in six.moves.xrange(read_batch_num):
idx_s = idx_batch * read_batch_size
idx_e = (idx_batch + 1) * read_batch_size
idx_e = np.minimum(idx_e, big_file["x"].shape[0])
y[idx_s:idx_e] = np.asarray(big_file['y'][idx_s:idx_e])
ID[idx_s:idx_e] = np.asarray(big_file['ID'][idx_s:idx_e])
pos[idx_s:idx_e] = np.asarray(big_file['pos'][idx_s:idx_e])
angle[idx_s:idx_e] = np.asarray(big_file['angle'][idx_s:idx_e])
coords[idx_s:idx_e] = np.asarray(big_file['coords'][idx_s:idx_e])
# import pdb
# pdb.set_trace()
# # Make sure data is contiguos
# y = np.ascontiguousarray(y)
# ID = np.ascontiguousarray(ID)
# pos = np.ascontiguousarray(pos)
# angle = np.ascontiguousarray(angle)
# coords = np.ascontiguousarray(coords)
print(" -- " + mode + ": Done... ")
return x, y, ID, pos, angle, coords
def make_xy(pair_index, img, kp, desc, aff, K, R, t, cur_folder):
kp_initial = copy.deepcopy(kp)
xs = []
xs_initial = []
ys = []
Rs = []
ts = []
img1s = []
img2s = []
cx1s = []
cy1s = []
f1s = []
cx2s = []
cy2s = []
f2s = []
affine = []
# Create a random folder in scratch
dump_dir = os.path.join(cur_folder, "dump")
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
pool_arg = []
for idx in range(pair_index.shape[0]):
ii = pair_index[idx, 0]
jj = pair_index[idx, 1]
print("\rExtracting keypoints {} / {}".format(idx,
pair_index.shape[0]),
end="")
sys.stdout.flush()
# Check and extract keypoints if necessary
for i in [ii, jj]:
i = int(i)
dump_file = os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(i))
if not os.path.exists(dump_file):
# Correct coordinates using K
cx = K[i, 2]
cy = K[i, 5]
# Correct focals
fx = K[i, 0]
fy = K[i, 4]
kp[i] = (kp[i] - np.array([[cx, cy]])) / np.asarray([[fx, fy]])
# Write descs to harddisk to parallize
dump_dict = {}
dump_dict["kp_normal"] = kp[i]
dump_dict["kp"] = kp_initial[i]
dump_dict["desc"] = desc[i]
dump_dict["aff"] = aff[i]
saveh5(dump_dict, dump_file)
else:
dump_dict = loadh5(dump_file)
kp[i] = dump_dict["kp_normal"]
kp_initial[i] = dump_dict["kp"]
desc[i] = dump_dict["desc"]
aff[i] = dump_dict["aff"]
pool_arg += [(dump_dir, idx, int(ii), int(jj))]
print("")
# Run mp job
ratio_CPU = 0.9
number_of_process = int(ratio_CPU * mp.cpu_count())
pool = mp.Pool(processes=number_of_process)
manager = mp.Manager()
queue = manager.Queue()
for idx_arg in xrange(len(pool_arg)):
pool_arg[idx_arg] = pool_arg[idx_arg] + (queue, )
# map async
pool_res = pool.map_async(dump_data_pair, pool_arg)
# monitor loop
while True:
if pool_res.ready():
break
else:
size = queue.qsize()
print("\rDistMat {} / {}".format(size, len(pool_arg)), end="")
sys.stdout.flush()
time.sleep(1)
pool.close()
pool.join()
print("")
# Pack data
idx = 0
total_num = 0
good_num = 0
bad_num = 0
ratio1 = []
ratio2 = []
for idx in range(pair_index.shape[0]):
ii = int(pair_index[idx, 0])
jj = int(pair_index[idx, 1])
print("\rWorking on {} / {}".format(idx, pair_index.shape[0]), end="")
sys.stdout.flush()
K1 = K[ii].reshape(3, 3)
K2 = K[jj].reshape(3, 3)
# ------------------------------
# Get dR
R_i = R[ii].reshape(3, 3)
R_j = R[jj].reshape(3, 3)
dR = np.dot(R_j, R_i.T)
# Get dt
t_i = t[ii].reshape(3, 1)
t_j = t[jj].reshape(3, 1)
dt = t_j - np.dot(dR, t_i)
# ------------------------------
# Get keypoints for the first image
x1 = kp[ii]
y1 = np.concatenate((kp[ii], np.ones((kp[ii].shape[0], 1))), axis=1)
# Project the first points into the second image
y1p = np.matmul(dR[None], y1[..., None]) + dt[None]
# move back to the canonical plane
x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None]
# ------------------------------
# Get keypoints for the second image
x2 = kp[jj]
# # DEBUG ------------------------------
# # Check if the image projections make sense
# draw_val_res(
# img[ii],
# img[jj],
# x1, x1p, np.random.rand(x1.shape[0]) < 0.1,
# (img[ii][0].shape[1] - 1.0) * 0.5,
# (img[ii][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][ii, 0, 0],
# (img[jj][0].shape[1] - 1.0) * 0.5,
# (img[jj][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][jj, 0, 0],
# "./debug_imgs/",
# "debug_img{:04d}.png".format(idx)
# )
# ------------------------------
# create x1, y1, x2, y2 as a matrix combo
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1)
y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
# Load precomputed nearest neighbors
idx_sort = loadh5(
os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii,
jj)))["idx_sort"]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x1pmat = x1pmat[idx_sort]
y1pmat = y1pmat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x1p = np.concatenate([x1pmat.reshape(-1, 1),
y1pmat.reshape(-1, 1)],
axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
# make xs in NHWC
xs += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
# ------------------------------
# Get the geodesic distance using with x1, x2, dR, dt
if config.obj_geod_type == "sampson":
geod_d = get_sampsons(x1, x2, dR, dt)
elif config.obj_geod_type == "episqr":
geod_d = get_episqr(x1, x2, dR, dt)
elif config.obj_geod_type == "episym":
geod_d = get_episym(x1, x2, dR, dt)
# geod_d = get_episym(x1, x2, dR, dt, K1_inv, K2_inv, K1, K2)
# Get *rough* reprojection errors. Note that the depth may be noisy. We
# ended up not using this...
reproj_d = np.sum((x2 - x1p)**2, axis=1)
# count inliers and outliers
total_num += len(geod_d)
good_num += np.sum((geod_d < config.obj_geod_th))
bad_num += np.sum((geod_d >= config.obj_geod_th))
'''
mask = np.zeros(len(geod_d))
for i in range(len(geod_d)):
if geod_d[i] < config.obj_geod_th:
mask[i] = 1
np.savetxt(os.path.join(dump_dir, "mask-{}-{}.txt".format(ii, jj)), mask)
'''
ys += [np.stack([geod_d, reproj_d], axis=1)]
# Save R, t for evaluation
Rs += [np.array(dR).reshape(3, 3)]
# normalize t before saving
dtnorm = np.sqrt(np.sum(dt**2))
assert (dtnorm > 1e-5)
dt /= dtnorm
ts += [np.array(dt).flatten()]
# Save img1 and img2 for display
img1s += [img[ii]]
img2s += [img[jj]]
cx = K[ii, 2]
cy = K[ii, 5]
cx1s += [cx]
cy1s += [cy]
cx = K[jj, 2]
cy = K[jj, 5]
cx2s += [cx]
cy2s += [cy]
fx = K[ii, 0]
fy = K[ii, 4]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
f1s += [f]
fx = K[jj, 0]
fy = K[jj, 4]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
f2s += [f]
# Generate xs_initial and T-transform
x1 = kp_initial[ii]
x2 = kp_initial[jj]
aff1 = aff[ii]
aff2 = aff[jj]
aff1 = np.repeat(aff1[:, :][..., None], len(aff2),
axis=-1).transpose(0, 2, 1)
aff2 = np.repeat(aff2[:, :][None], len(aff1), axis=0)
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
idx_sort = loadh5(
os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii,
jj)))["idx_sort"]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
aff1 = aff1[idx_sort]
aff2 = aff2[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
affine += [
np.concatenate(
[aff1.reshape(-1, 9), aff2.reshape(-1, 9)], axis=1)
]
xs_initial += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
print("")
# Do *not* convert to numpy arrays, as the number of keypoints may differ
# now. Simply return it
print(".... done")
if total_num > 0:
print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format(
good_num, total_num,
float(good_num) / float(total_num)))
print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format(
bad_num, total_num,
float(bad_num) / float(total_num)))
res_dict = {}
res_dict["xs"] = xs
res_dict["xs_initial"] = xs_initial
res_dict["affine"] = affine
res_dict["ys"] = ys
res_dict["Rs"] = Rs
res_dict["ts"] = ts
res_dict["img1s"] = img1s
res_dict["cx1s"] = cx1s
res_dict["cy1s"] = cy1s
res_dict["f1s"] = f1s
res_dict["img2s"] = img2s
res_dict["cx2s"] = cx2s
res_dict["cy2s"] = cy2s
res_dict["f2s"] = f2s
return res_dict
def loadFromDir(train_data_dir,
cur_folder,
gt_div_str="",
bUseColorImage=True,
input_width=512,
crop_center=False,
load_hessian=True):
"""Loads data from directory.
train_data_dir : Directory containing data
gt_div_str : suffix for depth (e.g. -8x8)
bUseColorImage : whether to use color or gray (default false)
input_width : input image rescaling size
"""
# read the list of imgs and the homography
train_data_dir = train_data_dir.rstrip("/") + "/"
img_list_file = train_data_dir + "list.txt"
K_list_file = train_data_dir + "models/K/K.txt"
R_list_file = train_data_dir + "models/R/R.txt"
t_list_file = train_data_dir + "models/t/t.txt"
# parse the file
image_fullpath_list = []
with open(img_list_file, "r") as img_list:
while True:
# read a single line
tmp = img_list.readline()
if type(tmp) != str:
line2parse = tmp.decode("utf-8")
else:
line2parse = tmp
if not line2parse:
break
# strip the newline at the end and add to list with full path
image_fullpath_list += [train_data_dir + line2parse.rstrip("\n")]
K = np.loadtxt(K_list_file)
R = np.loadtxt(R_list_file)
t = np.loadtxt(t_list_file)
# For each image and geom file in the list, read the image onto
# memory. We may later on want to simply save it to a hdf5 file
x = []
kp = []
desc = []
img_name = []
aff = []
idxImg = 0
for img_file in zip(image_fullpath_list):
idxImg += 1
print('\r -- Loading Image {} / {}'.format(idxImg,
len(image_fullpath_list)),
end="")
img_file = img_file[0]
img_file = img_file.split('\\')[0] + '/' + img_file.split('\\')[1]
# ---------------------------------------------------------------------
# Read the color image
if not bUseColorImage:
# If there is not gray image, load the color one and convert to
# gray
if os.path.exists(img_file.replace("image_color", "image_gray")):
img = cv2.imread(img_file.replace("image_color", "image_gray"),
0)
assert len(img.shape) == 2
else:
# read the image
img = cv2.cvtColor(cv2.imread(img_file), cv2.COLOR_BGR2GRAY)
if len(img.shape) == 2:
img = img[..., None]
in_dim = 1
else:
img = cv2.imread(img_file)
in_dim = 3
assert (img.shape[-1] == in_dim)
# Crop center and resize image into something reasonable
if crop_center:
rows, cols = img.shape[:2]
if rows > cols:
cut = (rows - cols) // 2
img_cropped = img[cut:cut + cols, :]
else:
cut = (cols - rows) // 2
img_cropped = img[:, cut:cut + rows]
scale_factor = float(input_width) / float(img_cropped.shape[0])
img = cv2.resize(img_cropped, (input_width, input_width))
else:
scale_factor = 1.0
if load_hessian == True:
if os.path.exists(
os.path.join(cur_folder,
"dump/kp-aff-desc-{}.h5".format(idxImg - 1))):
dump_dict = loadh5(
os.path.join(cur_folder,
"dump/kp-aff-desc-{}.h5".format(idxImg - 1)))
x += [img.transpose(2, 0, 1)]
kp += [dump_dict["kp"]]
aff += [dump_dict["aff"]]
desc += [dump_dict["desc"]]
else:
eng = matlab.engine.start_matlab()
frames = eng.hessian(img_file, nargout=2)
keypoints = np.array(frames[0]).transpose()
patches = np.array(frames[1]).transpose()
x += [img.transpose(2, 0, 1)]
keypoint = keypoints[:, :2]
kp += [keypoint]
affine = keypoints[:, 2:6].reshape(-1, 2, 2)
affine = np.concatenate(
[affine, np.expand_dims(keypoint, axis=-1)], axis=-1)
ones = np.ones([keypoints.shape[0], 1, 3], dtype=np.float)
affine = np.concatenate([affine, ones], axis=1)
aff += [affine.reshape(-1, 9)]
desc += [patches]
np.savetxt(img_file.split('images')[0] + "image_index.txt",
img_name,
fmt='%s')
return (x, kp, desc, aff, K, R, t)
def _create_pairs(self, task, use_cache=True, overwrite=False):
"""Generate the pairs from ID
This function should return the pair indices given the task. The return
type is expected to be a dictionary, where you can access by doing
res["P1"], for example.
"""
# Use the cache file if asked
if use_cache:
pairs_file = os.path.join(
self.pair_dir, "{}.h5".format(task))
if not os.path.exists(self.pair_dir):
os.makedirs(self.pair_dir)
if os.path.exists(pairs_file) and not overwrite:
if self.config.use_augmented_set:
# Add rotation augmentation if it does not exist (compat)
_f = h5py.File(pairs_file, "r+")
for name in ["P1", "P2", "P3", "P4"]:
if (name + "_rot_aug") not in _f:
_f.create_dataset(
name + "_rot_aug", data=2 *
np.pi * self.rng.rand(_f[name].size))
_f.close()
return loadh5(pairs_file)
# Make a lookup table for getting the indices of each sample. This can
# be easily done by sorting by ID, and storing the indices to a
# dictionary
if self.LUT_kp[task] is None:
# Create empty dictionary
self.LUT_kp[task] = {}
self.LUT_nonkp[task] = {}
if hasattr(self, "th_dist"):
self.LUT_below_th[task] = {}
# Build list of indices
dist_idx_reverse = np.zeros(
self.data[task]["ID"].max() + 1,
dtype=np.int64)
dist_idx_reverse[self.data[task]["dist_idx"]] = range(
1, self.data[task]["dist_idx"].size + 1)
dist_idx_reverse -= 1
# Argsort the ID
print("[{}] sorting IDs...".format(task))
ID = self.data[task]["ID"]
ind = np.argsort(ID)
# Go through them one by one -- the easy way
print("[{}] building LUT...".format(task))
start = 0
start_ID = ID[ind[start]]
for end in xrange(1, len(ind)):
if end % 100 == 0:
print("\r --- working on {}/{}".format(
end + 1, len(ind)), end="")
sys.stdout.flush()
# Check if indices have changed or we reached the end
if start_ID != ID[ind[end]] or end == len(ind) - 1:
# Store them in proper LUT
if start_ID < 0:
self.LUT_nonkp[task][-1] = ind[start:end]
else:
self.LUT_kp[task][start_ID] = ind[start:end]
if hasattr(self, "th_dist"):
v = dist_idx_reverse[start_ID]
assert v >= 0, "Invalid index"
d = self.data[task]["dist_mat"][v]
# 3D points are 1-indexed
self.LUT_below_th[task][start_ID] = 1 + \
np.where((d > 0) & (d < self.th_dist))[0]
# Update start position
start = end
start_ID = ID[ind[start]]
print("\r --- done. ")
# The dictionary to return
cur_pairs = {}
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name] = []
# For each kp item, create a random pair
#
# Note that this is different from what we used to do. This way seems
# better as we will look at each keypoint once.
print("[{}] creating pairs...".format(task))
num_kp = len(self.LUT_kp[task])
for i in xrange(num_kp):
if i % 100 == 0:
print("\r --- working on {}/{}".format(
i + 1, num_kp), end="")
sys.stdout.flush()
_kp = list(self.LUT_kp[task].keys())[i]
# Check if we have enough views of this point and skip
if len(self.LUT_kp[task][_kp]) < 2:
continue
# For P1 and P2 -- Select two random patches
P1, P2 = self.rng.choice(
self.LUT_kp[task][_kp], 2, replace=False)
# For P3 -- Select a different keypoint randomly
# Generate a list of keys
valid_keys = set(self.LUT_kp[task])
# Remove self
valid_keys -= set([_kp])
# Remove points below the distance threshold
if hasattr(self, "th_dist"):
valid_keys -= set(valid_keys.intersection(
self.LUT_below_th[task][_kp]))
_kp2 = self.rng.choice(list(valid_keys))
P3 = self.rng.choice(self.LUT_kp[task][_kp2], 1)[0]
# For P4 -- Select a random nonkp
P4 = self.rng.choice(self.LUT_nonkp[task][-1], 1)[0]
# Append to the list
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name].append(eval(name))
print("\r --- done. ")
# Convert them to np arrays
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name] = np.asarray(cur_pairs[name])
# Augment pairs with rotation data
if self.config.use_augmented_set:
for name in ["P1", "P2", "P3", "P4"]:
cur_pairs[name + "_rot_aug"] = 2 * np.pi * \
self.rng.rand(cur_pairs[name].size)
# Save to cache if asked
if use_cache:
if not os.path.exists(self.pair_dir):
os.makedirs(self.pair_dir)
saveh5(cur_pairs, pairs_file)
return cur_pairs
def make_xy(num_sample, pairs, kp, z, desc, img, geom, vis, depth, geom_type,
cur_folder):
xs = []
ys = []
Rs = []
ts = []
mutuals = []
ratios = []
img1s = []
img2s = []
cx1s = []
cy1s = []
f1s = []
cx2s = []
cy2s = []
f2s = []
# Create a random folder in scratch
dump_dir = os.path.join(cur_folder, "dump")
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
# randomly suffle the pairs and select num_sample amount
np.random.seed(1234)
cur_pairs = [
pairs[_i] for _i in np.random.permutation(len(pairs))[:num_sample]
]
idx = 0
for ii, jj in cur_pairs:
idx += 1
print(
"\rExtracting keypoints {} / {}".format(idx, len(cur_pairs)),
end="")
sys.stdout.flush()
# Check and extract keypoints if necessary
for i in [ii, jj]:
dump_file = os.path.join(dump_dir, "kp-z-desc-{}.h5".format(i))
if not os.path.exists(dump_file):
if kp[i] is None:
cv_kp, cv_desc = sift.detectAndCompute(img[i].transpose(
1, 2, 0), None)
cx = (img[i][0].shape[1] - 1.0) * 0.5
cy = (img[i][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][i, 0, 2]
cy += parse_geom(geom, geom_type)["K"][i, 1, 2]
xy = np.array([_kp.pt for _kp in cv_kp])
# Correct focals
fx = parse_geom(geom, geom_type)["K"][i, 0, 0]
fy = parse_geom(geom, geom_type)["K"][i, 1, 1]
kp[i] = (
xy - np.array([[cx, cy]])
) / np.asarray([[fx, fy]])
desc[i] = cv_desc
if z[i] is None:
cx = (img[i][0].shape[1] - 1.0) * 0.5
cy = (img[i][0].shape[0] - 1.0) * 0.5
fx = parse_geom(geom, geom_type)["K"][i, 0, 0]
fy = parse_geom(geom, geom_type)["K"][i, 1, 1]
xy = kp[i] * np.asarray([[fx, fy]]) + np.array([[cx, cy]])
if len(depth) > 0:
z[i] = depth[i][
0,
np.round(xy[:, 1]).astype(int),
np.round(xy[:, 0]).astype(int)][..., None]
else:
z[i] = np.ones((xy.shape[0], 1))
# Write descs to harddisk to parallize
dump_dict = {}
dump_dict["kp"] = kp[i]
dump_dict["z"] = z[i]
dump_dict["desc"] = desc[i]
saveh5(dump_dict, dump_file)
else:
dump_dict = loadh5(dump_file)
kp[i] = dump_dict["kp"]
z[i] = dump_dict["z"]
desc[i] = dump_dict["desc"]
print("")
# Create arguments
pool_arg = []
idx = 0
for ii, jj in cur_pairs:
idx += 1
pool_arg += [(dump_dir, idx, ii, jj)]
# Run mp job
ratio_CPU = 0.8
number_of_process = int(ratio_CPU * mp.cpu_count())
pool = mp.Pool(processes=number_of_process)
manager = mp.Manager()
queue = manager.Queue()
# for debugging in dump_data_pair
# dump_data_pair(pool_arg[1] + (queue,))
for idx_arg in xrange(len(pool_arg)):
pool_arg[idx_arg] = pool_arg[idx_arg] + (queue,)
# map async
pool_res = pool.map_async(dump_data_pair, pool_arg)
# monitor loop
while True:
if pool_res.ready():
break
else:
size = queue.qsize()
print("\rDistMat {} / {}".format(size, len(pool_arg)), end="")
sys.stdout.flush()
time.sleep(1)
pool.close()
pool.join()
print("")
# Pack data
idx = 0
total_num = 0
good_num = 0
bad_num = 0
for ii, jj in cur_pairs:
idx += 1
print("\rWorking on {} / {}".format(idx, len(cur_pairs)), end="")
sys.stdout.flush()
# ------------------------------
# Get dR
R_i = parse_geom(geom, geom_type)["R"][ii]
R_j = parse_geom(geom, geom_type)["R"][jj]
dR = np.dot(R_j, R_i.T)
# Get dt
t_i = parse_geom(geom, geom_type)["t"][ii].reshape([3, 1])
t_j = parse_geom(geom, geom_type)["t"][jj].reshape([3, 1])
dt = t_j - np.dot(dR, t_i)
# ------------------------------
# Get sift points for the first image
x1 = kp[ii]
y1 = np.concatenate([kp[ii] * z[ii], z[ii]], axis=1)
# Project the first points into the second image
y1p = np.matmul(dR[None], y1[..., None]) + dt[None]
# move back to the canonical plane
x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None]
# ------------------------------
# Get sift points for the second image
x2 = kp[jj]
# # DEBUG ------------------------------
# # Check if the image projections make sense
# draw_val_res(
# img[ii],
# img[jj],
# x1, x1p, np.random.rand(x1.shape[0]) < 0.1,
# (img[ii][0].shape[1] - 1.0) * 0.5,
# (img[ii][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][ii, 0, 0],
# (img[jj][0].shape[1] - 1.0) * 0.5,
# (img[jj][0].shape[0] - 1.0) * 0.5,
# parse_geom(geom, geom_type)["K"][jj, 0, 0],
# "./debug_imgs/",
# "debug_img{:04d}.png".format(idx)
# )
# ------------------------------
# create x1, y1, x2, y2 as a matrix combo
x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1)
y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1)
x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1)
y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1)
x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0)
y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0)
# Load precomputed nearest neighbors, ratios and mutual
idx_sort = loadh5(os.path.join(
dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"]
mutual = loadh5(os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["mutual"]
ratio = loadh5(os.path.join(
dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["ratio"]
mutuals += [mutual]
ratios += [ratio]
# Move back to tuples
idx_sort = (idx_sort[0], idx_sort[1])
x1mat = x1mat[idx_sort]
y1mat = y1mat[idx_sort]
x1pmat = x1pmat[idx_sort]
y1pmat = y1pmat[idx_sort]
x2mat = x2mat[idx_sort]
y2mat = y2mat[idx_sort]
# Turn into x1, x1p, x2
x1 = np.concatenate(
[x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1)
x1p = np.concatenate(
[x1pmat.reshape(-1, 1),
y1pmat.reshape(-1, 1)], axis=1)
x2 = np.concatenate(
[x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1)
# make xs in NHWC
xs += [
np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose(
(1, 2, 0))
]
# ------------------------------
# Get the geodesic distance using with x1, x2, dR, dt
if config.obj_geod_type == "sampson":
geod_d = get_sampsons(x1, x2, dR, dt)
elif config.obj_geod_type == "episqr":
geod_d = get_episqr(x1, x2, dR, dt)
elif config.obj_geod_type == "episym":
geod_d = get_episym(x1, x2, dR, dt)
# Get *rough* reprojection errors. Note that the depth may be noisy. We
# ended up not using this...
reproj_d = np.sum((x2 - x1p)**2, axis=1)
# count inliers and outliers
total_num += len(geod_d)
good_num += np.sum((geod_d < config.obj_geod_th))
bad_num += np.sum((geod_d >= config.obj_geod_th))
ys += [np.stack([geod_d, reproj_d], axis=1)]
# Save R, t for evaluation
Rs += [np.array(dR).reshape(3, 3)]
# normalize t before saving
dtnorm = np.sqrt(np.sum(dt**2))
assert (dtnorm > 1e-5)
dt /= dtnorm
ts += [np.array(dt).flatten()]
# Save img1 and img2 for display
img1s += [img[ii]]
img2s += [img[jj]]
cx = (img[ii][0].shape[1] - 1.0) * 0.5
cy = (img[ii][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][ii, 0, 2]
cy += parse_geom(geom, geom_type)["K"][ii, 1, 2]
fx = parse_geom(geom, geom_type)["K"][ii, 0, 0]
fy = parse_geom(geom, geom_type)["K"][ii, 1, 1]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
cx1s += [cx]
cy1s += [cy]
f1s += [f]
cx = (img[jj][0].shape[1] - 1.0) * 0.5
cy = (img[jj][0].shape[0] - 1.0) * 0.5
# Correct coordinates using K
cx += parse_geom(geom, geom_type)["K"][jj, 0, 2]
cy += parse_geom(geom, geom_type)["K"][jj, 1, 2]
fx = parse_geom(geom, geom_type)["K"][jj, 0, 0]
fy = parse_geom(geom, geom_type)["K"][jj, 1, 1]
if np.isclose(fx, fy):
f = fx
else:
f = (fx, fy)
cx2s += [cx]
cy2s += [cy]
f2s += [f]
# Do *not* convert to numpy arrays, as the number of keypoints may differ
# now. Simply return it
print(".... done")
if total_num > 0:
print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format(
good_num, total_num, float(good_num) / float(total_num)))
print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format(
bad_num, total_num, float(bad_num) / float(total_num)))
res_dict = {}
res_dict["xs"] = xs
res_dict["ys"] = ys
res_dict["Rs"] = Rs
res_dict["ts"] = ts
res_dict["img1s"] = img1s
res_dict["cx1s"] = cx1s
res_dict["cy1s"] = cy1s
res_dict["f1s"] = f1s
res_dict["img2s"] = img2s
res_dict["cx2s"] = cx2s
res_dict["cy2s"] = cy2s
res_dict["f2s"] = f2s
res_dict["mutuals"] = mutuals
res_dict["ratios"] = ratios
res_dict["pairs"] = cur_pairs
return res_dict
def get_list_of_img(train_data_dir, dump_data_dir, param, mode):
# Check if split file exists
split_prefix = train_data_dir + 'split-'
split_prefix += str(param.dataset.nTrainPercent) + '-'
split_prefix += str(param.dataset.nValidPercent) + '-'
split_prefix += str(param.dataset.nTestPercent) + '-'
# If it does not exist, create one
if not os.path.exists(split_prefix + mode + '.txt'):
# Read list of images
list_png_file = []
for files in os.listdir(train_data_dir):
if files.endswith(".png"):
list_png_file = list_png_file + [files]
# Shuffle the image list
if not os.path.exists(dump_data_dir + 'permute_png_idx.h5'):
print(' -- ' + mode + ': '
'Creating new shuffle for reading images')
permute_png_idx = np.random.permutation(len(list_png_file))
to_save = {"saveval": permute_png_idx}
saveh5(to_save, dump_data_dir + 'permute_png_idx.h5')
# dt.save(permute_png_idx,
# dump_data_dir + 'permute_png_idx.h5')
else:
print(' -- ' + mode + ': '
'Loading shuffle for reading images from '
'{}'.format(dump_data_dir))
to_load = loadh5(dump_data_dir + 'permute_png_idx.h5')
permute_png_idx = to_load["saveval"]
# permute_png_idx = dt.load(dump_data_dir +
# 'permute_png_idx.h5')
list_png_file = [
list_png_file[permute_png_idx[idx]]
for idx in range(len(list_png_file))
]
# Write to file (all three)
f_train = open(split_prefix + 'train.txt', 'w')
f_valid = open(split_prefix + 'valid.txt', 'w')
f_test = open(split_prefix + 'test.txt', 'w')
train_end = int(
float(param.dataset.nTrainPercent) / 100.0 * len(list_png_file))
valid_end = int(
float(param.dataset.nTrainPercent + param.dataset.nValidPercent) /
100.0 * len(list_png_file))
for idx_png in six.moves.xrange(len(list_png_file)):
if idx_png > valid_end:
print(list_png_file[idx_png], file=f_test)
elif idx_png > train_end:
print(list_png_file[idx_png], file=f_valid)
else:
print(list_png_file[idx_png], file=f_train)
f_train.close()
f_valid.close()
f_test.close()
# Read the list
list_png_file = list(np.loadtxt(split_prefix + mode + '.txt', dtype='str'))
return list_png_file
def createDump(args):
# print("createDump called")
idx_jpg, jpg_file, train_data_dir, dump_data_dir, tmp_patch_dir, scale_hist, scale_hist_c, out_dim, param, queue = args
# print(idx_jpg)
# print(jpg_file)
# print(args)
# queue for monitoring
if queue is not None:
queue.put(idx_jpg)
final_dump_file_name = os.path.join(
tmp_patch_dir,
os.path.basename(jpg_file).split(".")[0] + ".h5")
# print(final_dump_file_name)
if not os.path.exists(final_dump_file_name):
# load image
bUseColorImage = getattr(param.patch, "bUseColorImage", False)
if not bUseColorImage:
bUseDebugImage = getattr(param.patch, "bUseDebugImage", False)
if not bUseDebugImage:
img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY)
else:
# debug image contains keypoints survive the SfM
# pipeline (blue) and the rest (in red)
img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY)
# debug_jpg = train_data_dir + jpg_file.replace(".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) +".jpg")
debug_jpg = os.path.splitext(jpg_file)[0] + "-kp-minsc-" + str(
param.dataset.fMinKpSize) + ".jpg"
imgd = cv2.cvtColor(cv2.imread(debug_jpg), cv2.COLOR_BGR2GRAY)
img = cv2.resize(imgd, img.shape[:2][::-1])
in_dim = 1
else:
img = cv2.imread(train_data_dir + jpg_file)
in_dim = 3
assert (img.shape[-1] == in_dim)
# ----------------------------------------
# load kp data
# Note that data is in order of [x,y,scale,angle,pointid,setid]]
# guessing here: pointid is the point id number for the keypoint in the two 2d image
# or maybe it's a unique idx for a point in a list of all points?
# of maybe its the index of a point for all points in a set?(I think it's this last one, maybe)
# setid is the id num for the pair which is created from the image keypoints (this would keep a uniform )
# For positive: select randomly from "valid_keypoints"
pos_kp_file_name = jpg_file.replace(
".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) + ".h5")
# print(pos_kp_file_name)
with h5py.File(pos_kp_file_name, "r") as h5file:
sfm_kp = h5file["valid_keypoints"][()]
non_sfm_kp = h5file["other_keypoints"][()]
# add two dummy fields to non_sfm since they don"t have id
# and group. THis means pointid,setid are set to -1 for non_sfm_kp
# non_sfm_kp = np.concatenate([non_sfm_kp, -np.ones((non_sfm_kp.shape[0], 2))],axis=1)
# if there are no keypoints which were leveraged in SFM
if (len(sfm_kp) == 0):
sfm_kp = np.empty([0, 6])
if (len(non_sfm_kp) == 0):
non_sfm_kp = np.empty([0, 6])
# return None
# print(sfm_kp)
# print(type(sfm_kp))
# print(sfm_kp.shape)
# print(non_sfm_kp)
# print(type(non_sfm_kp))
# print(non_sfm_kp.shape)
pos_kp = np.concatenate(
(sfm_kp, np.ones((sfm_kp.shape[0], 1), dtype="float")), axis=1)
if (len(sfm_kp) == 0):
pos_kp = pos_kp[1:, :]
if (len(non_sfm_kp) == 0):
non_sfm_kp = non_sfm_kp[1:, :]
# Select subset for positives (assuming we have same coordinates for all image)
# dump_file_name = dump_data_dir + jpg_file.replace(".jpg","_idxPosSel.h5")
dump_file_name = os.path.join(
dump_data_dir,
os.path.basename(jpg_file).split(".")[0] + "_idxPosSel.h5")
# if dump file not exist, create it; otherwise load it
if not os.path.exists(dump_file_name):
# idxPosShuffle = np.argsort(pos_kp[3,:])[::-1] # sort backwards
idxPosShuffle = np.random.permutation(
len(pos_kp)) # random shuffle
pos_2_keep = len(idxPosShuffle)
if param.dataset.nPosPerImg > 0:
pos_2_keep = min(pos_2_keep, param.dataset.nPosPerImg)
idxPosSel = idxPosShuffle[:pos_2_keep] # shuffle the points
to_save = {"saveval": idxPosSel}
# print(to_save)
saveh5(to_save, dump_file_name)
else:
to_load = loadh5(dump_file_name)
idxPosSel = to_load["saveval"]
pos_kp = pos_kp[idxPosSel]
# print(pos_kp)
# negative sampling:
#
# 1) only use Sfm points, too close keypoints will be rejected as
# negative pair (because of high potential overlapping)
#
# 2) Use all SIFT points, when the overlapping of two feature context
# windows is larger than a threshold, it will be rejected as a negative
# pair (because it shares too much common regions..). In this step, we
# concatenate sfm_kp and non_sfm_kp to form keypoint class.
neg_mine_method = getattr(param.patch, "sNegMineMethod",
"use_only_SfM_points")
if neg_mine_method == "use_only_SfM_points":
# print("mining sfm points...")
# exit()
# where is
neg_kp = random_mine_non_kp_with_2d_distance(
img, sfm_kp, scale_hist, scale_hist_c, param)
elif neg_mine_method == "use_all_SIFT_points":
# print("mining SIFT points...")
# exit()
max_iter = getattr(param.patch, "nMaxRandomNegMineIter", 100)
# print(sfm_kp)
# print(type(sfm_kp))
# print(sfm_kp.shape)
# print(non_sfm_kp)
# print(type(non_sfm_kp))
# print(non_sfm_kp.shape)
# print(pos_kp)
# print(type(pos_kp))
# print(pos_kp.shape)
try:
sift_kp = np.concatenate([sfm_kp, non_sfm_kp], axis=0)
neg_kp = random_mine_non_kp_with_3d_blocking(img,
sift_kp,
scale_hist,
scale_hist_c,
param,
max_iter=max_iter)
# neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, neg_per_iter = 10, max_iter=max_iter)
except ValueError as err:
# print(err)
# print("Possibly no non_sfm_keypoints?")
neg_kp = np.empty([0, 6])
else:
raise ValueError(
"Mining method {} is not supported!".format(neg_mine_method))
# add another dim indicating good or bad
neg_kp = np.concatenate(
(neg_kp, np.zeros((len(neg_kp), 1), dtype="float")), axis=1)
# concatenate negative and positives
# print(pos_kp)
# print(type(pos_kp))
# print(pos_kp.shape)
# print(neg_kp)
# print(type(neg_kp))
# print(neg_kp.shape)
kp = np.concatenate((pos_kp, neg_kp), axis=0)
# print(kp)
# Retrive target values, 1 for pos and 0 for neg
y = kp[:, 6]
# Retrieve angles
angle = kp[:, 3]
# Assign ID to keypoints (for sfm points, ID = 3d point ind; for non
# sfm kp, ID = -1)
ID = kp[:, 4]
# load patches with id (drop out of boundary)
bPerturb = getattr(param.patch, "bPerturb", False)
fPerturbInfo = getattr(param.patch, "fPerturbInfo", np.zeros((3, )))
nAugmentedRotations = getattr(param.patch, "nAugmentedRotations", 1)
fAugmentRange = getattr(param.patch, "fAugmentRange", 0)
fAugmentCenterRandStrength = getattr(param.patch,
"fAugmentCenterRandStrength", 0)
sAugmentCenterRandMethod = getattr(param.patch,
"sAugmentCenterRandMethod",
"uniform")
cur_data_set = load_patches(
img,
kp,
y,
ID,
angle,
param.patch.fRatioScale,
param.patch.fMaxScale,
param.patch.nPatchSize,
param.model.nDescInputSize,
in_dim,
bPerturb,
fPerturbInfo,
bReturnCoords=True,
nAugmentedRotations=nAugmentedRotations,
fAugmentRange=fAugmentRange,
fAugmentCenterRandStrength=fAugmentCenterRandStrength,
sAugmentCenterRandMethod=sAugmentCenterRandMethod,
nPatchSizeAug=param.patch.nPatchSizeAug,
)
# save dump as dictionary using saveh5
# from here Kernel died, because of not finding "MKL_intel_thread.dll"
# pdb.set_trace()
# here the data are saved in tmp_dump file, keys are numbers [0,1..]
tmpdict = dict(
(str(_idx), np.asarray(_data))
for _idx, _data in zip(np.arange(len(cur_data_set)), cur_data_set))
saveh5(tmpdict, final_dump_file_name)
return idx_jpg
def process(inputs, bypass, name, skip, config, is_training):
"""WRITEME
inputs: input to the network
bypass: gt to by used when trying to bypass
name: name of the siamese branch
skip: whether to apply the bypass information
Note
----
We don't have to worry about the reuse flag here, since it is already dealt
with in the higher level. We just need to inherit it.
"""
# We never skip descriptor
assert skip is False
# we always expect a dictionary as return value to be more explicit
res = {}
# let's look at the inputs that get fed into this layer
image_summary_nhwc(name + "-input", inputs)
# Import the lift_desc_sub_kernel.h5 to get the kernel file
script_dir = os.path.dirname(os.path.realpath(__file__))
sub_kernel = loadh5(script_dir + "/lift_desc_sub_kernel.h5")["kernel"]
# activation
if config.desc_activ == "tanh":
activ = tf.nn.tanh
elif config.desc_activ == "relu":
activ = tf.nn.relu
else:
raise RuntimeError('Unknown activation type')
# pooling
def pool(cur_in, desc_pool, ksize):
if desc_pool == "l2_pool":
return pool_l2(cur_in, ksize, ksize, "VALID")
elif desc_pool == "max_pool":
return tf.nn.max_pool(cur_in, (1, ksize, ksize, 1),
(1, ksize, ksize, 1), "VALID")
elif desc_pool == "avg_pool":
return tf.nn.avg_pool(cur_in, (1, ksize, ksize, 1),
(1, ksize, ksize, 1), "VALID")
else:
raise RuntimeError('Unknown pooling type')
# now abuse cur_in so that we can simply copy paste
cur_in = inputs
# lets apply batch normalization on the input - we did not normalize the
# input range!
with tf.variable_scope("input-bn"):
if config.use_input_batch_norm:
cur_in = batch_norm(cur_in, training=is_training)
with tf.variable_scope("conv-act-pool-norm-1"):
cur_in = conv_2d(cur_in, 7, 32, 1, "VALID")
if config.use_batch_norm:
cur_in = batch_norm(cur_in, training=is_training)
cur_in = activ(cur_in)
cur_in = pool(cur_in, config.desc_pool, 2)
if config.use_subtractive_norm:
cur_in = norm_spatial_subtractive(cur_in, sub_kernel)
with tf.variable_scope("conv-act-pool-norm-2"):
cur_in = conv_2d(cur_in, 6, 64, 1, "VALID")
if config.use_batch_norm:
cur_in = batch_norm(cur_in, training=is_training)
cur_in = activ(cur_in)
cur_in = pool(cur_in, config.desc_pool, 3)
if config.use_subtractive_norm:
cur_in = norm_spatial_subtractive(cur_in, sub_kernel)
with tf.variable_scope("conv-act-pool-3"):
cur_in = conv_2d(cur_in, 5, 128, 1, "VALID")
if config.use_batch_norm:
cur_in = batch_norm(cur_in, training=is_training)
cur_in = activ(cur_in)
cur_in = pool(cur_in, config.desc_pool, 4)
res["desc"] = tf.reshape(cur_in, (-1, 128))
return res
def comp_process(mode, data, res_dir, config):
# Unpack some references
xs = data["xs"]
ys = data["ys"]
Rs = data["Rs"]
ts = data["ts"]
img1s = data["img1s"]
cx1s = data["cx1s"]
cy1s = data["cy1s"]
f1s = data["f1s"]
img2s = data["img2s"]
cx2s = data["cx2s"]
cy2s = data["cy2s"]
f2s = data["f2s"]
# Make fs numpy array
f1s = np.array(f1s)
f2s = np.array(f2s)
# Prepare directory
if not os.path.exists(res_dir):
os.makedirs(res_dir)
print("[{}] {}: Start testing".format(config.data_tr, time.asctime()))
# Validation
num_sample = len(xs)
if config.use_lift:
comp_list = [
"lmeds",
"ransac",
"mlesac",
# "usac5point", "usac8point",
# "usacnolo5point", "usacnolo8point"
]
else:
# comp_list = [
# "lmeds", "ransac", "top8_8point", "top50_lmeds", "top50_ransac",
# "gms", "gms_orb", "gms_default",
# "gms_orb_resize", "gms_orb_resize_ransac",
# "gms_orb_resize_tt", "gms_orb_resize_ransac_tt",
# "mlesac",
# ]
comp_list = [
"lmeds",
"ransac",
"mlesac",
# "usac5point", "usac8point",
# "usacnolo5point", "usacnolo8point"
]
if config.obj_num_kp == 2000:
comp_list += [
"gms_orb_resize_ransac_tt",
"gms_orb_resize_tt",
]
# Initialize arrays that will store measurements
err_q = {}
err_t = {}
num = {}
for _comp in comp_list:
err_q[_comp] = np.zeros(num_sample)
err_t[_comp] = np.zeros(num_sample)
num[_comp] = np.zeros(num_sample)
NUM_KP = config.obj_num_kp
# batch_size = config.val_batch_size
# num_batch = int(len(xs) / batch_size)
from gms_matcher import GmsMatcher
# SIFT
sift = cv2.xfeatures2d.SIFT_create(nfeatures=NUM_KP,
contrastThreshold=1e-5)
if cv2.__version__.startswith('3'):
sift_matcher = cv2.BFMatcher(cv2.NORM_L2)
else:
sift_matcher = cv2.BFMatcher_create(cv2.NORM_L2)
sift_gms = GmsMatcher(sift, sift_matcher)
# ORB
orb = cv2.ORB_create(10000)
orb.setFastThreshold(0)
if cv2.__version__.startswith('3'):
orb_matcher = cv2.BFMatcher(cv2.NORM_HAMMING)
else:
orb_matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING)
orb_gms = GmsMatcher(orb, orb_matcher)
for method_name in comp_list:
# Check res_dir if we have the dump ready
dump_dir = os.path.join(res_dir, mode, method_name)
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
# Check for full dump
full_dump_file = os.path.join(dump_dir, "qtn_all.h5")
if not os.path.exists(full_dump_file) or config.vis_dump:
for _idx in xrange(num_sample):
print("\rWorking on {} / {}".format(_idx + 1, num_sample),
end="")
sys.stdout.flush()
dump_file = os.path.join(dump_dir, "qtn_{}.txt".format(_idx))
# If dump exists, just load it
if os.path.exists(dump_file) and not config.vis_dump:
with open(dump_file, "r") as ifp:
dump_res = ifp.read()
dump_res = parse("{err_q:e}, {err_t:e}, {num_inlier:d}\n",
dump_res)
_err_q = dump_res["err_q"]
_err_t = dump_res["err_t"]
_num_inlier = dump_res["num_inlier"]
else:
# Otherwise compute
_xs = xs[_idx][:, :, :].reshape(1, 1, -1, 4)
_ys = ys[_idx][:, :].reshape(1, -1, 2)
_dR = Rs[_idx]
_dt = ts[_idx]
# Eval decompose for all pairs
_xs = _xs.reshape(-1, 4)
# x coordinates
_x1 = _xs[:, :2]
_x2 = _xs[:, 2:]
# Prepare input
if method_name == "lmeds":
eval_func = eval_decompose
_method = cv2.LMEDS
_mask = None
elif method_name == "ransac":
eval_func = eval_decompose
_method = cv2.RANSAC
_mask = None
elif method_name == "mlesac":
eval_func = eval_decompose
_method = "MLESAC"
_mask = None
elif method_name == "gms":
eval_func = eval_decompose_8points
_method = None
sift_gms.empty_matches()
_x1, _x2, _mask = sift_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=True,
with_rotation=True)
elif method_name == "gms_default":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_x1, _x2, _mask = orb_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb_resize_ransac_tt":
eval_func = eval_decompose
_method = cv2.RANSAC
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=True,
with_rotation=True)
elif method_name == "gms_orb_resize_tt":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=True,
with_rotation=True)
elif method_name == "gms_orb_resize_ransac":
eval_func = eval_decompose
_method = cv2.RANSAC
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb_resize":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_img1 = np.transpose(img1s[_idx], (1, 2, 0))
_img2 = np.transpose(img2s[_idx], (1, 2, 0))
_h1, _w1 = _img1.shape[:2]
_h2, _w2 = _img2.shape[:2]
_s1 = 480.0 / _h1
_s2 = 480.0 / _h2
_h1 = int(_h1 * _s1)
_w1 = int(np.round(_w1 * _s1))
_h2 = int(_h2 * _s2)
_w2 = int(np.round(_w2 * _s2))
_img1 = cv2.resize(_img1, (_w1, _h1))
_img2 = cv2.resize(_img2, (_w2, _h2))
_x1, _x2, _mask = orb_gms.compute_matches(
_img1,
_img2,
cx1s[_idx] * _s1,
cx2s[_idx] * _s2,
cy1s[_idx] * _s1,
cy2s[_idx] * _s2,
f1s[_idx] * _s1,
f2s[_idx] * _s2,
with_scale=False,
with_rotation=False)
elif method_name == "gms_orb":
eval_func = eval_decompose_8points
_method = None
orb_gms.empty_matches()
_x1, _x2, _mask = orb_gms.compute_matches(
np.transpose(img1s[_idx], (1, 2, 0)),
np.transpose(img2s[_idx], (1, 2, 0)),
cx1s[_idx],
cx2s[_idx],
cy1s[_idx],
cy2s[_idx],
f1s[_idx],
f2s[_idx],
with_scale=True,
with_rotation=True)
# Compute errors
_err_q, _err_t, _, _, _num_inlier, _mask_after = eval_func(
_x1, _x2, _dR, _dt, mask=_mask, method=_method)
if config.vis_dump:
dump_val_res(
img1s[_idx],
img2s[_idx],
_x1,
_x2,
_mask,
_mask_after,
cx1s[_idx],
cy1s[_idx],
f1s[_idx],
cx2s[_idx],
cy2s[_idx],
f2s[_idx],
Rs[_idx],
ts[_idx],
os.path.join(res_dir, mode, "match", method_name,
"pair{:08d}".format(_idx)),
)
else:
# Write dump
with open(dump_file, "w") as ofp:
ofp.write("{:e}, {:e}, {:d}\n".format(
_err_q, _err_t, _num_inlier))
# Load them in list
err_q[method_name][_idx] = _err_q
err_t[method_name][_idx] = _err_t
num[method_name][_idx] = _num_inlier
if not config.vis_dump:
# Save to full dump
dump_dict = {}
dump_dict["err_q"] = err_q[method_name]
dump_dict["err_t"] = err_t[method_name]
dump_dict["num"] = num[method_name]
saveh5(dump_dict, full_dump_file)
# Remove all intermediate cache
for _f in os.listdir(dump_dir):
if _f.startswith("qtn_") and _f.endswith(".txt"):
os.remove(os.path.join(dump_dir, _f))
# Load the full dump file
else:
dump_dict = loadh5(full_dump_file)
err_q[method_name] = dump_dict["err_q"]
err_t[method_name] = dump_dict["err_t"]
num[method_name] = dump_dict["num"]
# Remove all intermediate cache
for _f in os.listdir(dump_dir):
if _f.startswith("qtn_") and _f.endswith(".txt"):
os.remove(os.path.join(dump_dir, _f))
print("")
if config.vis_dump:
return
# Report results
for _tag in comp_list:
# For median error
ofn = os.path.join(res_dir, mode, "median_err_q_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(err_q[_tag])))
ofn = os.path.join(res_dir, mode, "median_err_t_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(err_t[_tag])))
ofn = os.path.join(res_dir, mode, "median_num_{}.txt".format(_tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.median(num[_tag])))
# For accuracy AUC
ths = np.arange(7) * 5
cur_err_q = np.array(err_q[_tag]) * 180.0 / np.pi
cur_err_t = np.array(err_t[_tag]) * 180.0 / np.pi
# Get histogram
q_acc_hist, _ = np.histogram(cur_err_q, ths)
t_acc_hist, _ = np.histogram(cur_err_t, ths)
qt_acc_hist, _ = np.histogram(np.maximum(cur_err_q, cur_err_t), ths)
num_pair = float(len(cur_err_q))
q_acc_hist = q_acc_hist.astype(float) / num_pair
t_acc_hist = t_acc_hist.astype(float) / num_pair
qt_acc_hist = qt_acc_hist.astype(float) / num_pair
q_acc = np.cumsum(q_acc_hist)
t_acc = np.cumsum(t_acc_hist)
qt_acc = np.cumsum(qt_acc_hist)
for _idx_th in xrange(1, len(ths)):
# for q_auc
ofn = os.path.join(res_dir, mode,
"acc_q_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(q_acc[:_idx_th])))
# for t_auc
ofn = os.path.join(res_dir, mode,
"acc_t_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(t_acc[:_idx_th])))
# for qt_auc
ofn = os.path.join(
res_dir, mode,
"acc_qt_auc{}_{}.txt".format(ths[_idx_th], _tag))
with open(ofn, "w") as ofp:
ofp.write("{}\n".format(np.mean(qt_acc[:_idx_th])))
print("[{}] {}: End testing".format(config.data_tr, time.asctime()))
