def __init__(self, config, subtask, dataset, tfconfig): logdir = os.path.join(config.logdir, subtask) self.config = copy.deepcopy(config) self.config.subtask = subtask self.graph = tf.Graph() self.sess = tf.Session(graph=self.graph, config=tfconfig) with self.graph.as_default(): if os.path.exists(os.path.join(logdir, "mean.h5")): training_mean = loadh5(os.path.join(logdir, "mean.h5")) training_std = loadh5(os.path.join(logdir, "std.h5")) print("[{}] Loaded input normalizers for testing".format( subtask)) # Create the model instance self.network = Network(self.sess, self.config, dataset, { 'mean': training_mean, 'std': training_std}) else: self.network = Network(self.sess, self.config, dataset) self.saver = {} self.best_val_loss = {} self.best_step = {} # Create the saver instance for both joint and the current subtask for _key in ["joint", subtask]: self.saver[_key] = tf.train.Saver(self.network.allparams[_key]) # We have everything ready. We finalize and initialie the network here. self.sess.run(tf.global_variables_initializer()) restore_res = self.restore_network() if not restore_res: raise RuntimeError("Could not load network weights!")
def make_xy(self, ii, jj): geom_i, geom_j = parse_geom(self.geom[ii]), parse_geom(self.geom[jj]) # should check the image size here #load img and check img_size image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj] kp_i = loadh5(image_i+'.'+self.desc_name+'.hdf5')["keypoints"][:, :2] kp_j = loadh5(image_j+'.'+self.desc_name+'.hdf5')["keypoints"][:, :2] cx1, cy1, f1 = self.unpack_K(geom_i) cx2, cy2, f2 = self.unpack_K(geom_j) x1 = self.norm_kp(cx1, cy1, f1[0], f1[1], kp_i) x2 = self.norm_kp(cx2, cy2, f2[0], f2[1], kp_j) R_i, R_j = geom_i["R"], geom_j["R"] dR = np.dot(R_j, R_i.T) t_i, t_j = geom_i["t"].reshape([3, 1]), geom_j["t"].reshape([3, 1]) dt = t_j - np.dot(dR, t_i) if np.sqrt(np.sum(dt**2)) <= 1e-5: return [] dtnorm = np.sqrt(np.sum(dt**2)) dt /= dtnorm nn_info = loadh5(os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj))) idx_sort, ratio_test, mutual_nearest = nn_info["idx_sort"], nn_info["ratio_test"], nn_info["mutual_nearest"] x2 = x2[idx_sort[1],:] xs = np.concatenate([x1, x2], axis=1).reshape(1,-1,4) geod_d = get_episym(x1, x2, dR, dt) ys = geod_d.reshape(-1,1) return xs, ys, dR, dt, ratio_test, mutual_nearest, cx1, cy1, f1, cx2, cy2, f2
def load_legacy_network(self, load_dir): """Load function for our old framework""" print("[{}] Checking if old pre-trained weights exists in {}" "".format(self.config.subtask, load_dir)) model_file = os.path.join(load_dir, "model.h5") norm_file = os.path.join(load_dir, "norm.h5") base_file = os.path.join(load_dir, "base.h5") if os.path.exists(model_file) and os.path.exists(norm_file) and \ os.path.exists(base_file): model = loadh5(model_file) norm = loadh5(norm_file) base = loadh5(base_file) # Load the input normalization parameters. with self.graph.as_default(): self.network.mean["kp"] = float(norm["mean_x"]) self.network.mean["ori"] = float(norm["mean_x"]) self.network.mean["desc"] = float(base["patch-mean"]) self.network.std["kp"] = float(norm["std_x"]) self.network.std["ori"] = float(norm["std_x"]) self.network.std["desc"] = float(base["patch-std"]) # Load weights for the component self.network.legacy_load_func[self.config.subtask](self.sess, model) print("[{}] Loaded previously trained weights".format(self.config.subtask)) return True else: print("[{}] No pretrained weights from the old framework" "".format(self.config.subtask)) return False
def load_legacy_network(supervisor, subtask, load_dir): """Load function for our old framework""" # Lazy loading to prevent import issues from utils import loadh5 print("[{}] Checking if old pre-trained weights exists in {}" "".format(subtask, load_dir)) model_file = os.path.join(load_dir, "model.h5") norm_file = os.path.join(load_dir, "norm.h5") base_file = os.path.join(load_dir, "base.h5") if os.path.exists(model_file) and os.path.exists(norm_file) and \ os.path.exists(base_file): model = loadh5(model_file) norm = loadh5(norm_file) base = loadh5(base_file) # Load the input normalization parameters supervisor.network.mean["kp"] = float(norm["mean_x"]) supervisor.network.mean["ori"] = float(norm["mean_x"]) supervisor.network.mean["desc"] = float(base["patch-mean"]) supervisor.network.std["kp"] = float(norm["std_x"]) supervisor.network.std["ori"] = float(norm["std_x"]) supervisor.network.std["desc"] = float(base["patch-std"]) # Load weights for the component supervisor.network.legacy_load_func[subtask](supervisor.sess, model) print("[{}] Loaded previously trained weights".format(subtask)) return True else: print("[{}] No pretrained weights from the old framework" "".format(subtask)) return False
def dump_data_pair(args): dump_dir, idx, ii, jj, queue = args # queue for monitoring if queue is not None: queue.put(idx) dump_file = os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)) # txt_dict = os.path.join(dump_dir, "idx_sort-{}-{}.txt".format(ii, jj)) if not os.path.exists(dump_file): # Load descriptors for ii desc_ii = loadh5(os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(ii)))["desc"] desc_jj = loadh5(os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(jj)))["desc"] # compute decriptor distance matrix distmat = np.sqrt( np.sum( (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2, axis=2)) # Choose K best from N idx_sort = np.argsort(distmat, axis=1)[:, :config.obj_num_nn] idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None], idx_sort.shape[1], axis=1), idx_sort) distmat = distmat[idx_sort] # Dump to disk dump_dict = {} dump_dict["idx_sort"] = idx_sort saveh5(dump_dict, dump_file)
def dump_data_pair(args): dump_dir, idx, ii, jj, queue = args # queue for monitoring if queue is not None: queue.put(idx) dump_file = os.path.join( dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)) dump_file_mutual_ratio = os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)) if not os.path.exists(dump_file) or not os.path.exists(dump_file_mutual_ratio): # if 1==1: # Load descriptors for ii desc_ii = loadh5( os.path.join(dump_dir, "kp-z-desc-{}.h5".format(ii)))["desc"] desc_jj = loadh5( os.path.join(dump_dir, "kp-z-desc-{}.h5".format(jj)))["desc"] # compute decriptor distance matrix # distmat = np.sqrt( # np.sum( # (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2, # axis=2)) ##### replace with faster distmat computation distmat = np.sqrt(np.sum(desc_ii**2, axis=1, keepdims=True) + np.sum(desc_jj**2, axis=1) - 2 * np.dot(desc_ii, desc_jj.T)) # Choose K best from N idx_sort0 = np.argsort(distmat, axis=1)[:, :1] idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort0.shape[1], axis=1),idx_sort0) # saving the ratio idx_sort_2nd = np.argsort(distmat, axis=1)[:, :2] idx_sort_2nd = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort_2nd.shape[1], axis=1),idx_sort_2nd) dist2nd = distmat[idx_sort_2nd] ratio = dist2nd[:, 0] / dist2nd[:, 1] # saving mutual neighbor information # Please note that crossCheck in opencv doesn't work properly and thus is deprecated in this code. idx_sort1 = np.argsort(distmat, axis=0)[0, :] mutual_neigh = idx_sort1[idx_sort0.squeeze()] == np.arange(idx_sort0.shape[0]) # Dump to disk dump_dict = {} dump_dict_mutual_ratio = {} dump_dict["idx_sort"] = idx_sort dump_dict_mutual_ratio["ratio"] = ratio dump_dict_mutual_ratio["mutual"] = mutual_neigh # Looks ugly because we just add ratio and mutual neighboring information to CNe dataset if not os.path.exists(dump_file): saveh5(dump_dict, dump_file) if not os.path.exists(dump_file_mutual_ratio): saveh5(dump_dict_mutual_ratio, dump_file_mutual_ratio)
def dump_nn(self, ii, jj): dump_file = os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj)) if not os.path.exists(dump_file): image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj] desc_ii = loadh5(image_i+'.'+self.desc_name+'.hdf5')["descriptors"] desc_jj = loadh5(image_j+'.'+self.desc_name+'.hdf5')["descriptors"] idx_sort, ratio_test, mutual_nearest = computeNN(desc_ii, desc_jj) # Dump to disk dump_dict = {} dump_dict["idx_sort"] = idx_sort dump_dict["ratio_test"] = ratio_test dump_dict["mutual_nearest"] = mutual_nearest saveh5(dump_dict, dump_file)
def load_geom(geom_file, scale_factor=1.0, flip_R=False): # load geometry file geom_dict = loadh5(geom_file) # Check if principal point is at the center K = geom_dict["K"] # assert(abs(K[0, 2]) < 1e-3 and abs(K[1, 2]) < 1e-3) # Rescale calbration according to previous resizing S = np.asarray([[scale_factor, 0, 0], [0, scale_factor, 0], [0, 0, 1]]) K = np.dot(S, K) geom_dict["K"] = K # Transpose Rotation Matrix if needed if flip_R: R = geom_dict["R"].T.copy() geom_dict["R"] = R # append things to list geom_list = [] geom_info_name_list = ["K", "R", "T", "imsize"] for geom_info_name in geom_info_name_list: geom_list += [geom_dict[geom_info_name].flatten()] # Finally do K_inv since inverting K is tricky with theano geom_list += [np.linalg.inv(geom_dict["K"]).flatten()] # Get the quaternion from Rotation matrices as well q = quaternion_from_matrix(geom_dict["R"]) geom_list += [q.flatten()] # Also add the inverse of the quaternion q_inv = q.copy() np.negative(q_inv[1:], q_inv[1:]) geom_list += [q_inv.flatten()] # Add to list geom = np.concatenate(geom_list) return geom
def __init__(self, config, rng): self.config = config self.rng = rng # Open a tensorflow session. I like keeping things simple, so I don't # use a supervisor. I'm just going to do everything manually. I also # will just allow the gpu memory to grow tfconfig = tf.ConfigProto() tfconfig.gpu_options.allow_growth = True self.sess = tf.Session(config=tfconfig) # Create the dataset instance self.dataset = Dataset(self.config, rng) # Retrieve mean/std (yes it is hacky) logdir = os.path.join(self.config.logdir, self.config.subtask) if os.path.exists(os.path.join(logdir, "mean.h5")): training_mean = loadh5(os.path.join(logdir, "mean.h5")) training_std = loadh5(os.path.join(logdir, "std.h5")) print("[{}] Loaded input normalizers for testing".format( self.config.subtask)) # Create the model instance self.network = Network(self.sess, self.config, self.dataset, { 'mean': training_mean, 'std': training_std }) else: self.network = Network(self.sess, self.config, self.dataset) # Make individual saver instances for each module. self.saver = {} self.best_val_loss = {} self.best_step = {} # Create the saver instance for both joint and the current subtask for _key in ["joint", self.config.subtask]: self.saver[_key] = tf.train.Saver(self.network.allparams[_key]) #print('\nNETWORK PARAMETERS') #for item in self.network.allparams[_key]: # print(item) # We have everything ready. We finalize and initialie the network here. self.sess.run(tf.global_variables_initializer())
def load_network(self, load_dir): """Load function for our new framework""" print("[{}] Checking if previous Tensorflow run exists in {}" "".format(self.config.subtask, load_dir)) latest_checkpoint = tf.train.latest_checkpoint(load_dir) if latest_checkpoint is not None: # Load parameters with self.graph.as_default(): self.saver[self.config.subtask].restore( self.sess, latest_checkpoint ) print("[{}] Loaded previously trained weights".format(self.config.subtask)) # Save mean std (falls back to default if non-existent) if os.path.exists(os.path.join(load_dir, "mean.h5")): self.network.mean = loadh5(os.path.join(load_dir, "mean.h5")) self.network.std = loadh5(os.path.join(load_dir, "std.h5")) print("[{}] Loaded input normalizers".format(self.config.subtask)) # Load best validation result self.best_val_loss[self.config.subtask] = loadh5( os.path.join(load_dir, best_val_loss_filename) )[self.config.subtask] print("[{}] Loaded best validation result = {}".format( self.config.subtask,self.best_val_loss[self.config.subtask])) # Load best validation result self.best_step[self.config.subtask] = loadh5( os.path.join(load_dir, best_step_filename) )[self.config.subtask] print("[{}] Loaded best step = {}".format( self.config.subtask, self.best_step[self.config.subtask])) return True else: print("[{}] No previous Tensorflow result".format(self.config.subtask)) return False
def load_data_for_set(self, pathconf, param, mode): # ---------------------------------------------------------------------- # Train, Validation, and Test # mlab = matlab.engine.start_matlab() # Read from pathconf # Original implementation train_data_dir = os.path.normpath(pathconf.dataset) dump_data_dir = os.path.normpath(pathconf.train_dump) dump_patch_dir = os.path.normpath(pathconf.patch_dump) # local (or volatile) copy of the dump data tmp_patch_dir = os.path.normpath(pathconf.volatile_patch_dump) # print("train_data_dir = {}".format(train_data_dir)) # print("dump_data_dir = {}".format(dump_data_dir)) # print("dump_patch_dir = {}".format(dump_patch_dir)) # print("tmp_patch_dir = {}".format(tmp_patch_dir)) if not os.path.exists(dump_data_dir): os.makedirs(dump_data_dir) if not os.path.exists(dump_patch_dir): os.makedirs(dump_patch_dir) if not os.path.exists(tmp_patch_dir): os.makedirs(tmp_patch_dir) # Check if we have the big h5 file ready big_file_name = dump_patch_dir + mode + "-data-chunked.h5" # if os.getenv("MLTEST_DEBUG", default=""): # import pdb # pdb.set_trace() # Mutex lock # # We will create an nfs-safe lock file in a temporary directory to # prevent our script from using corrupted, or data that is still being # generated. This allows us to launch multiple instances at the same # time, and allow only a single instance to generate the big_file. if not os.path.exists(".locks"): os.makedirs(".locks") check_lock_file = ".locks/" + \ hashlib.md5(big_file_name.encode()).hexdigest() if os.name == "posix": check_lock = Lock(check_lock_file) check_lock.lifetime = timedelta(days=2) frameinfo = getframeinfo(currentframe()) print("-- {}/{}: waiting to obtain lock --".format( frameinfo.filename, frameinfo.lineno)) check_lock.lock() print(">> obtained lock for posix system<<") elif os.name == "nt": import filelock check_lock = filelock.FileLock(check_lock_file) check_lock.timeout = 2000 check_lock.acquire() if check_lock.is_locked: print(">> obtained lock for windows system <<") else: print("Unknown operating system, lock unavailable") # if the large training data file does not exist if not os.path.exists(big_file_name): print("big data file does not exist...") # if the patch-mode-data file does not exist if not os.path.exists( os.path.join(dump_patch_dir, mode + "-data.h5")): print("{0} does not exist...".format( os.path.join(dump_patch_dir, mode + "-data.h5"))) # Read scale histogram hist_file_path = train_data_dir + "scales-histogram-minsc-" + str( param.dataset.fMinKpSize) + ".h5" if not os.path.exists(hist_file_path): print("Hist file does not exist, creating...") get_scale_hist(train_data_dir, param) # print("Loading hist file...") hist_file = h5py.File(hist_file_path, "r") scale_hist = np.asarray(hist_file["histogram_bins"], dtype=float).flatten() # print(scale_hist) scale_hist /= np.sum(scale_hist) scale_hist_c = np.asarray( hist_file["histogram_centers"]).flatten() # Read list of images from split files split_name = "" split_name += str(param.dataset.nTrainPercent) + "-" split_name += str(param.dataset.nValidPercent) + "-" split_name += str(param.dataset.nTestPercent) + "-" if mode == "train": # split_name += "train-" split_name += "train" elif mode == "valid": # split_name += "val-" split_name += "val" elif mode == "test": # split_name += "test-" split_name += "test" print("split_name: {}".format(split_name)) # split_file_name = train_data_dir + "split-" \ # + split_name + "minsc-" \ # + str(param.dataset.fMinKpSize) + ".h.txt" # split_file_name = "split-" + split_name + "minsc-" + str(param.dataset.fMinKpSize) + ".h.txt" split_file_name = "split-" + split_name + ".txt" split_file_name = train_data_dir + split_file_name # split_file_name = os.path.join(train_data_dir, split_file_name) print("split_file_name: {}".format(split_file_name)) if not os.path.exists(split_file_name): print("split_file_name does not exist...") list_jpg_file = get_list_of_img(train_data_dir, dump_data_dir, param, mode) else: print("split_file_name exists...") list_jpg_file = [] for file_name in list( np.loadtxt(split_file_name, dtype=bytes)): list_jpg_file += [ file_name.decode("utf-8").replace( "-kp-minsc-" + str(param.dataset.fMinKpSize), ".jpg") ] # ------------------------------------------------- # Create dumps in parallel # I am lazy so create arguments in loop lol pool_arg = [None] * len(list_jpg_file) for idx_jpg in six.moves.xrange(len(list_jpg_file)): pool_arg[idx_jpg] = (idx_jpg, list_jpg_file[idx_jpg], train_data_dir, dump_data_dir, tmp_patch_dir, scale_hist, scale_hist_c, self.out_dim, param) # # if true, use multi thread, otherwise use only single thread prod = True if prod: number_of_process = int(ratio_CPU * mp.cpu_count()) pool = mp.Pool(processes=number_of_process) manager = mp.Manager() queue = manager.Queue() for idx_jpg in six.moves.xrange(len(list_jpg_file)): pool_arg[idx_jpg] = pool_arg[idx_jpg] + (queue, ) # map async pool_res = pool.map_async(createDump, pool_arg) # pool_res = pool.map_async(createDump, pool_arg, chunksize = int(len(list_jpg_file)/(number_of_process* mp.cpu_count()))) # monitor loop while True: if pool_res.ready(): print("Pool_res ready?") break else: size = queue.qsize() print("\r -- " + mode + ": Processing image {}/{}".format( size, len(list_jpg_file)), end="") # print(list_jpg_file[size]) sys.stdout.flush() time.sleep(1) pool.close() pool.join() print("\r -- " + mode + ": Finished Processing Images!") # for debugging, if multi thread is used, then it is difficult # to debug else: for idx_jpg in six.moves.xrange(len(list_jpg_file)): pool_arg[idx_jpg] = pool_arg[idx_jpg] + (None, ) for idx_jpg in six.moves.xrange(len(list_jpg_file)): createDump(pool_arg[idx_jpg]) print("\r -- " + mode + ": Processing image " "{}/{}".format(idx_jpg + 1, len(list_jpg_file)), end="") sys.stdout.flush() print("\r -- " + mode + ": Finished Processing Images!") # ------------------------------------------------- # # -------------------- # use single thread for simplify debugging # for idx_jpg in six.moves.xrange(len(list_jpg_file)): # pool_arg[idx_jpg] = pool_arg[idx_jpg] + (None,) # for idx_jpg in six.moves.xrange(len(list_jpg_file)): # createDump(pool_arg[idx_jpg]) # print("\r -- " + mode + ": Processing image " # "{}/{}".format(idx_jpg + 1, len(list_jpg_file)), # end="") # sys.stdout.flush() # print("\r -- " + mode + ": Finished Processing Images!") # ------------------------------------------------------------------ # Use only valid indices to ascertain mutual exclusiveness id_file_name = train_data_dir + "split-" id_file_name += str(param.dataset.nTrainPercent) + "-" id_file_name += str(param.dataset.nValidPercent) + "-" id_file_name += str(param.dataset.nTestPercent) + "-" id_file_name += ("minsc-" + str(param.dataset.fMinKpSize) + ".h5") if mode == "train": id_key = "indices_train" elif mode == "valid": id_key = "indices_val" elif mode == "test": id_key = "indices_test" # print(id_file_name) try: with h5py.File(id_file_name, "r") as id_file: id_2_keep = np.asarray(id_file[id_key]) except OSError as err: print(err) print("Creating idx file...") # if "unable to open file" in err: createsplitindexh5file(id_file_name, train_data_dir, param) with h5py.File(id_file_name, "r") as id_file: id_2_keep = np.asarray(id_file[id_key]) # print(id_2_keep) print("{0} has {1} sfmid points to keep...".format( id_key, len(id_2_keep))) # exit() # ind_2_keep = np.in1d(dataset[2], id_2_keep) # ind_2_keep += dataset[2] < 0 # loop through files to figure out how many valid items we have # pdb.set_trace() # for tracking of the dataset num_valid = 0 # print(len(list_jpg_file)) # exit() for idx_jpg in six.moves.xrange(len(list_jpg_file)): jpg_file = list_jpg_file[idx_jpg] print("\r -- " + mode + ": " "Reading dumps to figure out number of valid " "{}/{}".format(idx_jpg + 1, len(list_jpg_file)), end="") sys.stdout.flush() # Load created dump # final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5") # print(tmp_patch_dir) # print(jpg_file) final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename( jpg_file)[:-4] + ".h5" # print(final_dump_file_name) # Use loadh5 and turn it back to original cur_data_set try: with h5py.File(final_dump_file_name, "r") as dump_file: # print(list(dump_file.keys())) cur_ids = dump_file["2"].value # kps = dump_file["valid_keypoints"][()] # cur_ids = np.asarray(kps[:, 4]) # print(cur_ids) except OSError as err: # print(err) continue # Find cur valid by looking at id_2_keep cur_valid = np.in1d(cur_ids, id_2_keep) # print(cur_valid) # Add all negative labels as valid (neg data) cur_valid += cur_ids < 0 # Sum it up num_valid += np.sum(cur_valid) # print(num_valid) print("\n -- " + mode + ": " "Found {} valid data points from {} files" "".format(num_valid, len(list_jpg_file))) # Get the first data to simply check the shape tmp_dump_file_name = tmp_patch_dir + "\\" + os.path.basename( list_jpg_file[-1])[:-4] + ".h5" with h5py.File(tmp_dump_file_name, "r") as dump_file: dataset_shape = [] dataset_type = [] for _idx in six.moves.xrange(len(dump_file.keys())): dataset_shape += [dump_file[str(_idx)].shape] dataset_type += [dump_file[str(_idx)].dtype] # create and save the large dataset chunk with h5py.File(big_file_name, "w-") as big_file: big_file["time_stamp"] = np.asarray(time.localtime()) name_list = ["x", "y", "ID", "pos", "angle", "coords"] # create the dataset storage chunk for __i in six.moves.xrange(len(dataset_shape)): big_file.create_dataset( name_list[__i], (num_valid, ) + dataset_shape[__i][1:], chunks=(1, ) + dataset_shape[__i][1:], maxshape=((num_valid, ) + dataset_shape[__i][1:]), dtype=dataset_type[__i]) # loop through the file to save to a big chunk save_base = 0 for idx_jpg in six.moves.xrange(len(list_jpg_file)): jpg_file = list_jpg_file[idx_jpg] print("\r -- " + mode + ": " "Saving the data to the big dump " "{}/{}".format(idx_jpg + 1, len(list_jpg_file)), end="") sys.stdout.flush() # Load created dump # final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5") final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename( jpg_file)[:-4] + ".h5" # print(final_dump_file_name) # Use loadh5 and turn it back to original cur_data_set try: tmpdict = loadh5(final_dump_file_name) cur_data_set = tuple([ tmpdict[str(_idx)] for _idx in range(len(tmpdict.keys())) ]) # Find cur valid by looking at id_2_keep cur_valid = np.in1d(cur_data_set[2], id_2_keep) # Add all negative labels as valid (neg data) cur_valid += cur_data_set[2] < 0 for __i in six.moves.xrange(len(dataset_shape)): big_file[name_list[__i]][ save_base:save_base + np.sum(cur_valid )] = cur_data_set[__i][cur_valid] # Move base to the next chunk save_base += np.sum(cur_valid) except OSError as err: # print(err) # print("{0} skipped due to invalidity...".format(final_dump_file_name)) # sys.stdout.flush() continue # Assert that we saved all assert save_base == num_valid print("\n -- " + mode + ": " "Done saving {} valid data points from {} files" "".format(num_valid, len(list_jpg_file))) # -------------------------------------------------- # Cleanup dump for idx_jpg in six.moves.xrange(len(list_jpg_file)): jpg_file = list_jpg_file[idx_jpg] print("\r -- " + mode + ": " "Removing dump " "{}/{}".format(idx_jpg + 1, len(list_jpg_file)), end="") sys.stdout.flush() # Delete dump # final_dump_file_name = tmp_patch_dir + jpg_file.replace(".jpg", ".h5") final_dump_file_name = tmp_patch_dir + "\\" + os.path.basename( jpg_file)[:-4] + ".h5" try: os.remove(final_dump_file_name) except FileNotFoundError as err: pass print("\r -- " + mode + ": " "Cleaned up dumps! " "Local dump is now clean!") else: print(" -- Found old file without chunks. " "Copying to new file with chunks...") old_big_file_name = dump_patch_dir + mode + "-data.h5" with h5py.File(old_big_file_name, "r") as old_big_file, \ h5py.File(big_file_name, "w-") as big_file: dataset = [] # load old train into array name_list = ["x", "y", "ID", "pos", "angle", "coords"] for __i in six.moves.xrange(len(name_list)): dataset += [np.asarray(old_big_file[name_list[__i]])] # save train big_file["time_stamp"] = np.asarray(time.localtime()) # allocate and write for __i in six.moves.xrange(len(name_list)): if name_list[__i] == "x": chunk_shape = (1, ) + dataset[__i].shape[1:] else: chunk_shape = None big_file.create_dataset( name_list[__i], dataset[__i].shape, data=dataset[__i], chunks=chunk_shape, maxshape=dataset[__i].shape, ) print(" -- Finished creating chunked file, removing old...") os.remove(old_big_file_name) # ---------------------------------------------------------------------- # Copy to local tmp if necessary if not os.path.exists(tmp_patch_dir + mode + "-data-chunked.h5"): print(" -- " + mode + ": " "Local dump does not exist! " "Copying big dump to local drive... ") shutil.copy(dump_patch_dir + mode + "-data-chunked.h5", tmp_patch_dir + mode + "-data-chunked.h5") else: print(" -- " + mode + ": " "Local dump exists. Checking timestamp...") # get timestamp from nfs with h5py.File(dump_patch_dir + mode + "-data-chunked.h5", "r") \ as nfs_file: nfs_time = np.asarray(nfs_file["time_stamp"]) # get timestamp from local with h5py.File(tmp_patch_dir + mode + "-data-chunked.h5", "r") \ as local_file: local_time = np.asarray(local_file["time_stamp"]) # if the two files have different time stamps if any(nfs_time != local_time): print(" -- " + mode + ": " "Time stamps are different! " "Copying big dump to local drive... ") shutil.copy(dump_patch_dir + mode + "-data-chunked.h5", tmp_patch_dir + mode + "-data-chunked.h5") else: print(" -- " + mode + ": " "Time stamps are identical! Re-using local dump") # Free lock if os.name == "posix": check_lock.unlock() print("-- free lock --") elif os.name == "nt": check_lock.release() print("-- free lock --") else: pass # ---------------------------------------------------------------------- # Use local copy for faster speed print(" -- " + mode + ": Loading from local drive... ") big_file_name = tmp_patch_dir + mode + "-data-chunked.h5" # open big_file and don"t close big_file = h5py.File(big_file_name, "r") x = big_file["x"] # work arround for h5py loading all things to memory read_batch_size = 10000 read_batch_num = int( np.ceil(float(big_file["x"].shape[0]) / float(read_batch_size))) # Manual, since I don't want to bother debugging the below # fields = ["y", "ID", "pos", "angle", "coords"] # for var_name in fields: # # allocate data # exec("{0} = np.zeros(big_file['{0}'].shape, " # "dtype=big_file['{0}'].dtype)".format(var_name)) # # copy data in batches # for idx_batch in six.moves.xrange(read_batch_num): # idx_s = idx_batch * read_batch_size # idx_e = (idx_batch + 1) * read_batch_size # idx_e = np.minimum(idx_e, big_file["x"].shape[0]) # exec("{0}[idx_s:idx_e] = np.asarray(big_file['{0}'][idx_s:idx_e])" # "". format(var_name)) # Allocate y = np.zeros(big_file["y"].shape, dtype=big_file["y"].dtype) ID = np.zeros(big_file["ID"].shape, dtype=big_file["ID"].dtype) pos = np.zeros(big_file["pos"].shape, dtype=big_file["pos"].dtype) angle = np.zeros(big_file["angle"].shape, dtype=big_file["angle"].dtype) coords = np.zeros(big_file["coords"].shape, dtype=big_file["coords"].dtype) # Copy data in batches for idx_batch in six.moves.xrange(read_batch_num): idx_s = idx_batch * read_batch_size idx_e = (idx_batch + 1) * read_batch_size idx_e = np.minimum(idx_e, big_file["x"].shape[0]) y[idx_s:idx_e] = np.asarray(big_file['y'][idx_s:idx_e]) ID[idx_s:idx_e] = np.asarray(big_file['ID'][idx_s:idx_e]) pos[idx_s:idx_e] = np.asarray(big_file['pos'][idx_s:idx_e]) angle[idx_s:idx_e] = np.asarray(big_file['angle'][idx_s:idx_e]) coords[idx_s:idx_e] = np.asarray(big_file['coords'][idx_s:idx_e]) # import pdb # pdb.set_trace() # # Make sure data is contiguos # y = np.ascontiguousarray(y) # ID = np.ascontiguousarray(ID) # pos = np.ascontiguousarray(pos) # angle = np.ascontiguousarray(angle) # coords = np.ascontiguousarray(coords) print(" -- " + mode + ": Done... ") return x, y, ID, pos, angle, coords
def make_xy(pair_index, img, kp, desc, aff, K, R, t, cur_folder): kp_initial = copy.deepcopy(kp) xs = [] xs_initial = [] ys = [] Rs = [] ts = [] img1s = [] img2s = [] cx1s = [] cy1s = [] f1s = [] cx2s = [] cy2s = [] f2s = [] affine = [] # Create a random folder in scratch dump_dir = os.path.join(cur_folder, "dump") if not os.path.exists(dump_dir): os.makedirs(dump_dir) pool_arg = [] for idx in range(pair_index.shape[0]): ii = pair_index[idx, 0] jj = pair_index[idx, 1] print("\rExtracting keypoints {} / {}".format(idx, pair_index.shape[0]), end="") sys.stdout.flush() # Check and extract keypoints if necessary for i in [ii, jj]: i = int(i) dump_file = os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(i)) if not os.path.exists(dump_file): # Correct coordinates using K cx = K[i, 2] cy = K[i, 5] # Correct focals fx = K[i, 0] fy = K[i, 4] kp[i] = (kp[i] - np.array([[cx, cy]])) / np.asarray([[fx, fy]]) # Write descs to harddisk to parallize dump_dict = {} dump_dict["kp_normal"] = kp[i] dump_dict["kp"] = kp_initial[i] dump_dict["desc"] = desc[i] dump_dict["aff"] = aff[i] saveh5(dump_dict, dump_file) else: dump_dict = loadh5(dump_file) kp[i] = dump_dict["kp_normal"] kp_initial[i] = dump_dict["kp"] desc[i] = dump_dict["desc"] aff[i] = dump_dict["aff"] pool_arg += [(dump_dir, idx, int(ii), int(jj))] print("") # Run mp job ratio_CPU = 0.9 number_of_process = int(ratio_CPU * mp.cpu_count()) pool = mp.Pool(processes=number_of_process) manager = mp.Manager() queue = manager.Queue() for idx_arg in xrange(len(pool_arg)): pool_arg[idx_arg] = pool_arg[idx_arg] + (queue, ) # map async pool_res = pool.map_async(dump_data_pair, pool_arg) # monitor loop while True: if pool_res.ready(): break else: size = queue.qsize() print("\rDistMat {} / {}".format(size, len(pool_arg)), end="") sys.stdout.flush() time.sleep(1) pool.close() pool.join() print("") # Pack data idx = 0 total_num = 0 good_num = 0 bad_num = 0 ratio1 = [] ratio2 = [] for idx in range(pair_index.shape[0]): ii = int(pair_index[idx, 0]) jj = int(pair_index[idx, 1]) print("\rWorking on {} / {}".format(idx, pair_index.shape[0]), end="") sys.stdout.flush() K1 = K[ii].reshape(3, 3) K2 = K[jj].reshape(3, 3) # ------------------------------ # Get dR R_i = R[ii].reshape(3, 3) R_j = R[jj].reshape(3, 3) dR = np.dot(R_j, R_i.T) # Get dt t_i = t[ii].reshape(3, 1) t_j = t[jj].reshape(3, 1) dt = t_j - np.dot(dR, t_i) # ------------------------------ # Get keypoints for the first image x1 = kp[ii] y1 = np.concatenate((kp[ii], np.ones((kp[ii].shape[0], 1))), axis=1) # Project the first points into the second image y1p = np.matmul(dR[None], y1[..., None]) + dt[None] # move back to the canonical plane x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None] # ------------------------------ # Get keypoints for the second image x2 = kp[jj] # # DEBUG ------------------------------ # # Check if the image projections make sense # draw_val_res( # img[ii], # img[jj], # x1, x1p, np.random.rand(x1.shape[0]) < 0.1, # (img[ii][0].shape[1] - 1.0) * 0.5, # (img[ii][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][ii, 0, 0], # (img[jj][0].shape[1] - 1.0) * 0.5, # (img[jj][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][jj, 0, 0], # "./debug_imgs/", # "debug_img{:04d}.png".format(idx) # ) # ------------------------------ # create x1, y1, x2, y2 as a matrix combo x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1) y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) # Load precomputed nearest neighbors idx_sort = loadh5( os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x1pmat = x1pmat[idx_sort] y1pmat = y1pmat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x1p = np.concatenate([x1pmat.reshape(-1, 1), y1pmat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) # make xs in NHWC xs += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] # ------------------------------ # Get the geodesic distance using with x1, x2, dR, dt if config.obj_geod_type == "sampson": geod_d = get_sampsons(x1, x2, dR, dt) elif config.obj_geod_type == "episqr": geod_d = get_episqr(x1, x2, dR, dt) elif config.obj_geod_type == "episym": geod_d = get_episym(x1, x2, dR, dt) # geod_d = get_episym(x1, x2, dR, dt, K1_inv, K2_inv, K1, K2) # Get *rough* reprojection errors. Note that the depth may be noisy. We # ended up not using this... reproj_d = np.sum((x2 - x1p)**2, axis=1) # count inliers and outliers total_num += len(geod_d) good_num += np.sum((geod_d < config.obj_geod_th)) bad_num += np.sum((geod_d >= config.obj_geod_th)) ''' mask = np.zeros(len(geod_d)) for i in range(len(geod_d)): if geod_d[i] < config.obj_geod_th: mask[i] = 1 np.savetxt(os.path.join(dump_dir, "mask-{}-{}.txt".format(ii, jj)), mask) ''' ys += [np.stack([geod_d, reproj_d], axis=1)] # Save R, t for evaluation Rs += [np.array(dR).reshape(3, 3)] # normalize t before saving dtnorm = np.sqrt(np.sum(dt**2)) assert (dtnorm > 1e-5) dt /= dtnorm ts += [np.array(dt).flatten()] # Save img1 and img2 for display img1s += [img[ii]] img2s += [img[jj]] cx = K[ii, 2] cy = K[ii, 5] cx1s += [cx] cy1s += [cy] cx = K[jj, 2] cy = K[jj, 5] cx2s += [cx] cy2s += [cy] fx = K[ii, 0] fy = K[ii, 4] if np.isclose(fx, fy): f = fx else: f = (fx, fy) f1s += [f] fx = K[jj, 0] fy = K[jj, 4] if np.isclose(fx, fy): f = fx else: f = (fx, fy) f2s += [f] # Generate xs_initial and T-transform x1 = kp_initial[ii] x2 = kp_initial[jj] aff1 = aff[ii] aff2 = aff[jj] aff1 = np.repeat(aff1[:, :][..., None], len(aff2), axis=-1).transpose(0, 2, 1) aff2 = np.repeat(aff2[:, :][None], len(aff1), axis=0) x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) idx_sort = loadh5( os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] aff1 = aff1[idx_sort] aff2 = aff2[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) affine += [ np.concatenate( [aff1.reshape(-1, 9), aff2.reshape(-1, 9)], axis=1) ] xs_initial += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] print("") # Do *not* convert to numpy arrays, as the number of keypoints may differ # now. Simply return it print(".... done") if total_num > 0: print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format( good_num, total_num, float(good_num) / float(total_num))) print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format( bad_num, total_num, float(bad_num) / float(total_num))) res_dict = {} res_dict["xs"] = xs res_dict["xs_initial"] = xs_initial res_dict["affine"] = affine res_dict["ys"] = ys res_dict["Rs"] = Rs res_dict["ts"] = ts res_dict["img1s"] = img1s res_dict["cx1s"] = cx1s res_dict["cy1s"] = cy1s res_dict["f1s"] = f1s res_dict["img2s"] = img2s res_dict["cx2s"] = cx2s res_dict["cy2s"] = cy2s res_dict["f2s"] = f2s return res_dict
def loadFromDir(train_data_dir, cur_folder, gt_div_str="", bUseColorImage=True, input_width=512, crop_center=False, load_hessian=True): """Loads data from directory. train_data_dir : Directory containing data gt_div_str : suffix for depth (e.g. -8x8) bUseColorImage : whether to use color or gray (default false) input_width : input image rescaling size """ # read the list of imgs and the homography train_data_dir = train_data_dir.rstrip("/") + "/" img_list_file = train_data_dir + "list.txt" K_list_file = train_data_dir + "models/K/K.txt" R_list_file = train_data_dir + "models/R/R.txt" t_list_file = train_data_dir + "models/t/t.txt" # parse the file image_fullpath_list = [] with open(img_list_file, "r") as img_list: while True: # read a single line tmp = img_list.readline() if type(tmp) != str: line2parse = tmp.decode("utf-8") else: line2parse = tmp if not line2parse: break # strip the newline at the end and add to list with full path image_fullpath_list += [train_data_dir + line2parse.rstrip("\n")] K = np.loadtxt(K_list_file) R = np.loadtxt(R_list_file) t = np.loadtxt(t_list_file) # For each image and geom file in the list, read the image onto # memory. We may later on want to simply save it to a hdf5 file x = [] kp = [] desc = [] img_name = [] aff = [] idxImg = 0 for img_file in zip(image_fullpath_list): idxImg += 1 print('\r -- Loading Image {} / {}'.format(idxImg, len(image_fullpath_list)), end="") img_file = img_file[0] img_file = img_file.split('\\')[0] + '/' + img_file.split('\\')[1] # --------------------------------------------------------------------- # Read the color image if not bUseColorImage: # If there is not gray image, load the color one and convert to # gray if os.path.exists(img_file.replace("image_color", "image_gray")): img = cv2.imread(img_file.replace("image_color", "image_gray"), 0) assert len(img.shape) == 2 else: # read the image img = cv2.cvtColor(cv2.imread(img_file), cv2.COLOR_BGR2GRAY) if len(img.shape) == 2: img = img[..., None] in_dim = 1 else: img = cv2.imread(img_file) in_dim = 3 assert (img.shape[-1] == in_dim) # Crop center and resize image into something reasonable if crop_center: rows, cols = img.shape[:2] if rows > cols: cut = (rows - cols) // 2 img_cropped = img[cut:cut + cols, :] else: cut = (cols - rows) // 2 img_cropped = img[:, cut:cut + rows] scale_factor = float(input_width) / float(img_cropped.shape[0]) img = cv2.resize(img_cropped, (input_width, input_width)) else: scale_factor = 1.0 if load_hessian == True: if os.path.exists( os.path.join(cur_folder, "dump/kp-aff-desc-{}.h5".format(idxImg - 1))): dump_dict = loadh5( os.path.join(cur_folder, "dump/kp-aff-desc-{}.h5".format(idxImg - 1))) x += [img.transpose(2, 0, 1)] kp += [dump_dict["kp"]] aff += [dump_dict["aff"]] desc += [dump_dict["desc"]] else: eng = matlab.engine.start_matlab() frames = eng.hessian(img_file, nargout=2) keypoints = np.array(frames[0]).transpose() patches = np.array(frames[1]).transpose() x += [img.transpose(2, 0, 1)] keypoint = keypoints[:, :2] kp += [keypoint] affine = keypoints[:, 2:6].reshape(-1, 2, 2) affine = np.concatenate( [affine, np.expand_dims(keypoint, axis=-1)], axis=-1) ones = np.ones([keypoints.shape[0], 1, 3], dtype=np.float) affine = np.concatenate([affine, ones], axis=1) aff += [affine.reshape(-1, 9)] desc += [patches] np.savetxt(img_file.split('images')[0] + "image_index.txt", img_name, fmt='%s') return (x, kp, desc, aff, K, R, t)
def _create_pairs(self, task, use_cache=True, overwrite=False): """Generate the pairs from ID This function should return the pair indices given the task. The return type is expected to be a dictionary, where you can access by doing res["P1"], for example. """ # Use the cache file if asked if use_cache: pairs_file = os.path.join( self.pair_dir, "{}.h5".format(task)) if not os.path.exists(self.pair_dir): os.makedirs(self.pair_dir) if os.path.exists(pairs_file) and not overwrite: if self.config.use_augmented_set: # Add rotation augmentation if it does not exist (compat) _f = h5py.File(pairs_file, "r+") for name in ["P1", "P2", "P3", "P4"]: if (name + "_rot_aug") not in _f: _f.create_dataset( name + "_rot_aug", data=2 * np.pi * self.rng.rand(_f[name].size)) _f.close() return loadh5(pairs_file) # Make a lookup table for getting the indices of each sample. This can # be easily done by sorting by ID, and storing the indices to a # dictionary if self.LUT_kp[task] is None: # Create empty dictionary self.LUT_kp[task] = {} self.LUT_nonkp[task] = {} if hasattr(self, "th_dist"): self.LUT_below_th[task] = {} # Build list of indices dist_idx_reverse = np.zeros( self.data[task]["ID"].max() + 1, dtype=np.int64) dist_idx_reverse[self.data[task]["dist_idx"]] = range( 1, self.data[task]["dist_idx"].size + 1) dist_idx_reverse -= 1 # Argsort the ID print("[{}] sorting IDs...".format(task)) ID = self.data[task]["ID"] ind = np.argsort(ID) # Go through them one by one -- the easy way print("[{}] building LUT...".format(task)) start = 0 start_ID = ID[ind[start]] for end in xrange(1, len(ind)): if end % 100 == 0: print("\r --- working on {}/{}".format( end + 1, len(ind)), end="") sys.stdout.flush() # Check if indices have changed or we reached the end if start_ID != ID[ind[end]] or end == len(ind) - 1: # Store them in proper LUT if start_ID < 0: self.LUT_nonkp[task][-1] = ind[start:end] else: self.LUT_kp[task][start_ID] = ind[start:end] if hasattr(self, "th_dist"): v = dist_idx_reverse[start_ID] assert v >= 0, "Invalid index" d = self.data[task]["dist_mat"][v] # 3D points are 1-indexed self.LUT_below_th[task][start_ID] = 1 + \ np.where((d > 0) & (d < self.th_dist))[0] # Update start position start = end start_ID = ID[ind[start]] print("\r --- done. ") # The dictionary to return cur_pairs = {} for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name] = [] # For each kp item, create a random pair # # Note that this is different from what we used to do. This way seems # better as we will look at each keypoint once. print("[{}] creating pairs...".format(task)) num_kp = len(self.LUT_kp[task]) for i in xrange(num_kp): if i % 100 == 0: print("\r --- working on {}/{}".format( i + 1, num_kp), end="") sys.stdout.flush() _kp = list(self.LUT_kp[task].keys())[i] # Check if we have enough views of this point and skip if len(self.LUT_kp[task][_kp]) < 2: continue # For P1 and P2 -- Select two random patches P1, P2 = self.rng.choice( self.LUT_kp[task][_kp], 2, replace=False) # For P3 -- Select a different keypoint randomly # Generate a list of keys valid_keys = set(self.LUT_kp[task]) # Remove self valid_keys -= set([_kp]) # Remove points below the distance threshold if hasattr(self, "th_dist"): valid_keys -= set(valid_keys.intersection( self.LUT_below_th[task][_kp])) _kp2 = self.rng.choice(list(valid_keys)) P3 = self.rng.choice(self.LUT_kp[task][_kp2], 1)[0] # For P4 -- Select a random nonkp P4 = self.rng.choice(self.LUT_nonkp[task][-1], 1)[0] # Append to the list for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name].append(eval(name)) print("\r --- done. ") # Convert them to np arrays for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name] = np.asarray(cur_pairs[name]) # Augment pairs with rotation data if self.config.use_augmented_set: for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name + "_rot_aug"] = 2 * np.pi * \ self.rng.rand(cur_pairs[name].size) # Save to cache if asked if use_cache: if not os.path.exists(self.pair_dir): os.makedirs(self.pair_dir) saveh5(cur_pairs, pairs_file) return cur_pairs
def make_xy(num_sample, pairs, kp, z, desc, img, geom, vis, depth, geom_type, cur_folder): xs = [] ys = [] Rs = [] ts = [] mutuals = [] ratios = [] img1s = [] img2s = [] cx1s = [] cy1s = [] f1s = [] cx2s = [] cy2s = [] f2s = [] # Create a random folder in scratch dump_dir = os.path.join(cur_folder, "dump") if not os.path.exists(dump_dir): os.makedirs(dump_dir) # randomly suffle the pairs and select num_sample amount np.random.seed(1234) cur_pairs = [ pairs[_i] for _i in np.random.permutation(len(pairs))[:num_sample] ] idx = 0 for ii, jj in cur_pairs: idx += 1 print( "\rExtracting keypoints {} / {}".format(idx, len(cur_pairs)), end="") sys.stdout.flush() # Check and extract keypoints if necessary for i in [ii, jj]: dump_file = os.path.join(dump_dir, "kp-z-desc-{}.h5".format(i)) if not os.path.exists(dump_file): if kp[i] is None: cv_kp, cv_desc = sift.detectAndCompute(img[i].transpose( 1, 2, 0), None) cx = (img[i][0].shape[1] - 1.0) * 0.5 cy = (img[i][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][i, 0, 2] cy += parse_geom(geom, geom_type)["K"][i, 1, 2] xy = np.array([_kp.pt for _kp in cv_kp]) # Correct focals fx = parse_geom(geom, geom_type)["K"][i, 0, 0] fy = parse_geom(geom, geom_type)["K"][i, 1, 1] kp[i] = ( xy - np.array([[cx, cy]]) ) / np.asarray([[fx, fy]]) desc[i] = cv_desc if z[i] is None: cx = (img[i][0].shape[1] - 1.0) * 0.5 cy = (img[i][0].shape[0] - 1.0) * 0.5 fx = parse_geom(geom, geom_type)["K"][i, 0, 0] fy = parse_geom(geom, geom_type)["K"][i, 1, 1] xy = kp[i] * np.asarray([[fx, fy]]) + np.array([[cx, cy]]) if len(depth) > 0: z[i] = depth[i][ 0, np.round(xy[:, 1]).astype(int), np.round(xy[:, 0]).astype(int)][..., None] else: z[i] = np.ones((xy.shape[0], 1)) # Write descs to harddisk to parallize dump_dict = {} dump_dict["kp"] = kp[i] dump_dict["z"] = z[i] dump_dict["desc"] = desc[i] saveh5(dump_dict, dump_file) else: dump_dict = loadh5(dump_file) kp[i] = dump_dict["kp"] z[i] = dump_dict["z"] desc[i] = dump_dict["desc"] print("") # Create arguments pool_arg = [] idx = 0 for ii, jj in cur_pairs: idx += 1 pool_arg += [(dump_dir, idx, ii, jj)] # Run mp job ratio_CPU = 0.8 number_of_process = int(ratio_CPU * mp.cpu_count()) pool = mp.Pool(processes=number_of_process) manager = mp.Manager() queue = manager.Queue() # for debugging in dump_data_pair # dump_data_pair(pool_arg[1] + (queue,)) for idx_arg in xrange(len(pool_arg)): pool_arg[idx_arg] = pool_arg[idx_arg] + (queue,) # map async pool_res = pool.map_async(dump_data_pair, pool_arg) # monitor loop while True: if pool_res.ready(): break else: size = queue.qsize() print("\rDistMat {} / {}".format(size, len(pool_arg)), end="") sys.stdout.flush() time.sleep(1) pool.close() pool.join() print("") # Pack data idx = 0 total_num = 0 good_num = 0 bad_num = 0 for ii, jj in cur_pairs: idx += 1 print("\rWorking on {} / {}".format(idx, len(cur_pairs)), end="") sys.stdout.flush() # ------------------------------ # Get dR R_i = parse_geom(geom, geom_type)["R"][ii] R_j = parse_geom(geom, geom_type)["R"][jj] dR = np.dot(R_j, R_i.T) # Get dt t_i = parse_geom(geom, geom_type)["t"][ii].reshape([3, 1]) t_j = parse_geom(geom, geom_type)["t"][jj].reshape([3, 1]) dt = t_j - np.dot(dR, t_i) # ------------------------------ # Get sift points for the first image x1 = kp[ii] y1 = np.concatenate([kp[ii] * z[ii], z[ii]], axis=1) # Project the first points into the second image y1p = np.matmul(dR[None], y1[..., None]) + dt[None] # move back to the canonical plane x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None] # ------------------------------ # Get sift points for the second image x2 = kp[jj] # # DEBUG ------------------------------ # # Check if the image projections make sense # draw_val_res( # img[ii], # img[jj], # x1, x1p, np.random.rand(x1.shape[0]) < 0.1, # (img[ii][0].shape[1] - 1.0) * 0.5, # (img[ii][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][ii, 0, 0], # (img[jj][0].shape[1] - 1.0) * 0.5, # (img[jj][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][jj, 0, 0], # "./debug_imgs/", # "debug_img{:04d}.png".format(idx) # ) # ------------------------------ # create x1, y1, x2, y2 as a matrix combo x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1) y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) # Load precomputed nearest neighbors, ratios and mutual idx_sort = loadh5(os.path.join( dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] mutual = loadh5(os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["mutual"] ratio = loadh5(os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["ratio"] mutuals += [mutual] ratios += [ratio] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x1pmat = x1pmat[idx_sort] y1pmat = y1pmat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x1p = np.concatenate( [x1pmat.reshape(-1, 1), y1pmat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) # make xs in NHWC xs += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] # ------------------------------ # Get the geodesic distance using with x1, x2, dR, dt if config.obj_geod_type == "sampson": geod_d = get_sampsons(x1, x2, dR, dt) elif config.obj_geod_type == "episqr": geod_d = get_episqr(x1, x2, dR, dt) elif config.obj_geod_type == "episym": geod_d = get_episym(x1, x2, dR, dt) # Get *rough* reprojection errors. Note that the depth may be noisy. We # ended up not using this... reproj_d = np.sum((x2 - x1p)**2, axis=1) # count inliers and outliers total_num += len(geod_d) good_num += np.sum((geod_d < config.obj_geod_th)) bad_num += np.sum((geod_d >= config.obj_geod_th)) ys += [np.stack([geod_d, reproj_d], axis=1)] # Save R, t for evaluation Rs += [np.array(dR).reshape(3, 3)] # normalize t before saving dtnorm = np.sqrt(np.sum(dt**2)) assert (dtnorm > 1e-5) dt /= dtnorm ts += [np.array(dt).flatten()] # Save img1 and img2 for display img1s += [img[ii]] img2s += [img[jj]] cx = (img[ii][0].shape[1] - 1.0) * 0.5 cy = (img[ii][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][ii, 0, 2] cy += parse_geom(geom, geom_type)["K"][ii, 1, 2] fx = parse_geom(geom, geom_type)["K"][ii, 0, 0] fy = parse_geom(geom, geom_type)["K"][ii, 1, 1] if np.isclose(fx, fy): f = fx else: f = (fx, fy) cx1s += [cx] cy1s += [cy] f1s += [f] cx = (img[jj][0].shape[1] - 1.0) * 0.5 cy = (img[jj][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][jj, 0, 2] cy += parse_geom(geom, geom_type)["K"][jj, 1, 2] fx = parse_geom(geom, geom_type)["K"][jj, 0, 0] fy = parse_geom(geom, geom_type)["K"][jj, 1, 1] if np.isclose(fx, fy): f = fx else: f = (fx, fy) cx2s += [cx] cy2s += [cy] f2s += [f] # Do *not* convert to numpy arrays, as the number of keypoints may differ # now. Simply return it print(".... done") if total_num > 0: print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format( good_num, total_num, float(good_num) / float(total_num))) print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format( bad_num, total_num, float(bad_num) / float(total_num))) res_dict = {} res_dict["xs"] = xs res_dict["ys"] = ys res_dict["Rs"] = Rs res_dict["ts"] = ts res_dict["img1s"] = img1s res_dict["cx1s"] = cx1s res_dict["cy1s"] = cy1s res_dict["f1s"] = f1s res_dict["img2s"] = img2s res_dict["cx2s"] = cx2s res_dict["cy2s"] = cy2s res_dict["f2s"] = f2s res_dict["mutuals"] = mutuals res_dict["ratios"] = ratios res_dict["pairs"] = cur_pairs return res_dict
def get_list_of_img(train_data_dir, dump_data_dir, param, mode): # Check if split file exists split_prefix = train_data_dir + 'split-' split_prefix += str(param.dataset.nTrainPercent) + '-' split_prefix += str(param.dataset.nValidPercent) + '-' split_prefix += str(param.dataset.nTestPercent) + '-' # If it does not exist, create one if not os.path.exists(split_prefix + mode + '.txt'): # Read list of images list_png_file = [] for files in os.listdir(train_data_dir): if files.endswith(".png"): list_png_file = list_png_file + [files] # Shuffle the image list if not os.path.exists(dump_data_dir + 'permute_png_idx.h5'): print(' -- ' + mode + ': ' 'Creating new shuffle for reading images') permute_png_idx = np.random.permutation(len(list_png_file)) to_save = {"saveval": permute_png_idx} saveh5(to_save, dump_data_dir + 'permute_png_idx.h5') # dt.save(permute_png_idx, # dump_data_dir + 'permute_png_idx.h5') else: print(' -- ' + mode + ': ' 'Loading shuffle for reading images from ' '{}'.format(dump_data_dir)) to_load = loadh5(dump_data_dir + 'permute_png_idx.h5') permute_png_idx = to_load["saveval"] # permute_png_idx = dt.load(dump_data_dir + # 'permute_png_idx.h5') list_png_file = [ list_png_file[permute_png_idx[idx]] for idx in range(len(list_png_file)) ] # Write to file (all three) f_train = open(split_prefix + 'train.txt', 'w') f_valid = open(split_prefix + 'valid.txt', 'w') f_test = open(split_prefix + 'test.txt', 'w') train_end = int( float(param.dataset.nTrainPercent) / 100.0 * len(list_png_file)) valid_end = int( float(param.dataset.nTrainPercent + param.dataset.nValidPercent) / 100.0 * len(list_png_file)) for idx_png in six.moves.xrange(len(list_png_file)): if idx_png > valid_end: print(list_png_file[idx_png], file=f_test) elif idx_png > train_end: print(list_png_file[idx_png], file=f_valid) else: print(list_png_file[idx_png], file=f_train) f_train.close() f_valid.close() f_test.close() # Read the list list_png_file = list(np.loadtxt(split_prefix + mode + '.txt', dtype='str')) return list_png_file
def createDump(args): # print("createDump called") idx_jpg, jpg_file, train_data_dir, dump_data_dir, tmp_patch_dir, scale_hist, scale_hist_c, out_dim, param, queue = args # print(idx_jpg) # print(jpg_file) # print(args) # queue for monitoring if queue is not None: queue.put(idx_jpg) final_dump_file_name = os.path.join( tmp_patch_dir, os.path.basename(jpg_file).split(".")[0] + ".h5") # print(final_dump_file_name) if not os.path.exists(final_dump_file_name): # load image bUseColorImage = getattr(param.patch, "bUseColorImage", False) if not bUseColorImage: bUseDebugImage = getattr(param.patch, "bUseDebugImage", False) if not bUseDebugImage: img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY) else: # debug image contains keypoints survive the SfM # pipeline (blue) and the rest (in red) img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY) # debug_jpg = train_data_dir + jpg_file.replace(".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) +".jpg") debug_jpg = os.path.splitext(jpg_file)[0] + "-kp-minsc-" + str( param.dataset.fMinKpSize) + ".jpg" imgd = cv2.cvtColor(cv2.imread(debug_jpg), cv2.COLOR_BGR2GRAY) img = cv2.resize(imgd, img.shape[:2][::-1]) in_dim = 1 else: img = cv2.imread(train_data_dir + jpg_file) in_dim = 3 assert (img.shape[-1] == in_dim) # ---------------------------------------- # load kp data # Note that data is in order of [x,y,scale,angle,pointid,setid]] # guessing here: pointid is the point id number for the keypoint in the two 2d image # or maybe it's a unique idx for a point in a list of all points? # of maybe its the index of a point for all points in a set?(I think it's this last one, maybe) # setid is the id num for the pair which is created from the image keypoints (this would keep a uniform ) # For positive: select randomly from "valid_keypoints" pos_kp_file_name = jpg_file.replace( ".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) + ".h5") # print(pos_kp_file_name) with h5py.File(pos_kp_file_name, "r") as h5file: sfm_kp = h5file["valid_keypoints"][()] non_sfm_kp = h5file["other_keypoints"][()] # add two dummy fields to non_sfm since they don"t have id # and group. THis means pointid,setid are set to -1 for non_sfm_kp # non_sfm_kp = np.concatenate([non_sfm_kp, -np.ones((non_sfm_kp.shape[0], 2))],axis=1) # if there are no keypoints which were leveraged in SFM if (len(sfm_kp) == 0): sfm_kp = np.empty([0, 6]) if (len(non_sfm_kp) == 0): non_sfm_kp = np.empty([0, 6]) # return None # print(sfm_kp) # print(type(sfm_kp)) # print(sfm_kp.shape) # print(non_sfm_kp) # print(type(non_sfm_kp)) # print(non_sfm_kp.shape) pos_kp = np.concatenate( (sfm_kp, np.ones((sfm_kp.shape[0], 1), dtype="float")), axis=1) if (len(sfm_kp) == 0): pos_kp = pos_kp[1:, :] if (len(non_sfm_kp) == 0): non_sfm_kp = non_sfm_kp[1:, :] # Select subset for positives (assuming we have same coordinates for all image) # dump_file_name = dump_data_dir + jpg_file.replace(".jpg","_idxPosSel.h5") dump_file_name = os.path.join( dump_data_dir, os.path.basename(jpg_file).split(".")[0] + "_idxPosSel.h5") # if dump file not exist, create it; otherwise load it if not os.path.exists(dump_file_name): # idxPosShuffle = np.argsort(pos_kp[3,:])[::-1] # sort backwards idxPosShuffle = np.random.permutation( len(pos_kp)) # random shuffle pos_2_keep = len(idxPosShuffle) if param.dataset.nPosPerImg > 0: pos_2_keep = min(pos_2_keep, param.dataset.nPosPerImg) idxPosSel = idxPosShuffle[:pos_2_keep] # shuffle the points to_save = {"saveval": idxPosSel} # print(to_save) saveh5(to_save, dump_file_name) else: to_load = loadh5(dump_file_name) idxPosSel = to_load["saveval"] pos_kp = pos_kp[idxPosSel] # print(pos_kp) # negative sampling: # # 1) only use Sfm points, too close keypoints will be rejected as # negative pair (because of high potential overlapping) # # 2) Use all SIFT points, when the overlapping of two feature context # windows is larger than a threshold, it will be rejected as a negative # pair (because it shares too much common regions..). In this step, we # concatenate sfm_kp and non_sfm_kp to form keypoint class. neg_mine_method = getattr(param.patch, "sNegMineMethod", "use_only_SfM_points") if neg_mine_method == "use_only_SfM_points": # print("mining sfm points...") # exit() # where is neg_kp = random_mine_non_kp_with_2d_distance( img, sfm_kp, scale_hist, scale_hist_c, param) elif neg_mine_method == "use_all_SIFT_points": # print("mining SIFT points...") # exit() max_iter = getattr(param.patch, "nMaxRandomNegMineIter", 100) # print(sfm_kp) # print(type(sfm_kp)) # print(sfm_kp.shape) # print(non_sfm_kp) # print(type(non_sfm_kp)) # print(non_sfm_kp.shape) # print(pos_kp) # print(type(pos_kp)) # print(pos_kp.shape) try: sift_kp = np.concatenate([sfm_kp, non_sfm_kp], axis=0) neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, max_iter=max_iter) # neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, neg_per_iter = 10, max_iter=max_iter) except ValueError as err: # print(err) # print("Possibly no non_sfm_keypoints?") neg_kp = np.empty([0, 6]) else: raise ValueError( "Mining method {} is not supported!".format(neg_mine_method)) # add another dim indicating good or bad neg_kp = np.concatenate( (neg_kp, np.zeros((len(neg_kp), 1), dtype="float")), axis=1) # concatenate negative and positives # print(pos_kp) # print(type(pos_kp)) # print(pos_kp.shape) # print(neg_kp) # print(type(neg_kp)) # print(neg_kp.shape) kp = np.concatenate((pos_kp, neg_kp), axis=0) # print(kp) # Retrive target values, 1 for pos and 0 for neg y = kp[:, 6] # Retrieve angles angle = kp[:, 3] # Assign ID to keypoints (for sfm points, ID = 3d point ind; for non # sfm kp, ID = -1) ID = kp[:, 4] # load patches with id (drop out of boundary) bPerturb = getattr(param.patch, "bPerturb", False) fPerturbInfo = getattr(param.patch, "fPerturbInfo", np.zeros((3, ))) nAugmentedRotations = getattr(param.patch, "nAugmentedRotations", 1) fAugmentRange = getattr(param.patch, "fAugmentRange", 0) fAugmentCenterRandStrength = getattr(param.patch, "fAugmentCenterRandStrength", 0) sAugmentCenterRandMethod = getattr(param.patch, "sAugmentCenterRandMethod", "uniform") cur_data_set = load_patches( img, kp, y, ID, angle, param.patch.fRatioScale, param.patch.fMaxScale, param.patch.nPatchSize, param.model.nDescInputSize, in_dim, bPerturb, fPerturbInfo, bReturnCoords=True, nAugmentedRotations=nAugmentedRotations, fAugmentRange=fAugmentRange, fAugmentCenterRandStrength=fAugmentCenterRandStrength, sAugmentCenterRandMethod=sAugmentCenterRandMethod, nPatchSizeAug=param.patch.nPatchSizeAug, ) # save dump as dictionary using saveh5 # from here Kernel died, because of not finding "MKL_intel_thread.dll" # pdb.set_trace() # here the data are saved in tmp_dump file, keys are numbers [0,1..] tmpdict = dict( (str(_idx), np.asarray(_data)) for _idx, _data in zip(np.arange(len(cur_data_set)), cur_data_set)) saveh5(tmpdict, final_dump_file_name) return idx_jpg
def process(inputs, bypass, name, skip, config, is_training): """WRITEME inputs: input to the network bypass: gt to by used when trying to bypass name: name of the siamese branch skip: whether to apply the bypass information Note ---- We don't have to worry about the reuse flag here, since it is already dealt with in the higher level. We just need to inherit it. """ # We never skip descriptor assert skip is False # we always expect a dictionary as return value to be more explicit res = {} # let's look at the inputs that get fed into this layer image_summary_nhwc(name + "-input", inputs) # Import the lift_desc_sub_kernel.h5 to get the kernel file script_dir = os.path.dirname(os.path.realpath(__file__)) sub_kernel = loadh5(script_dir + "/lift_desc_sub_kernel.h5")["kernel"] # activation if config.desc_activ == "tanh": activ = tf.nn.tanh elif config.desc_activ == "relu": activ = tf.nn.relu else: raise RuntimeError('Unknown activation type') # pooling def pool(cur_in, desc_pool, ksize): if desc_pool == "l2_pool": return pool_l2(cur_in, ksize, ksize, "VALID") elif desc_pool == "max_pool": return tf.nn.max_pool(cur_in, (1, ksize, ksize, 1), (1, ksize, ksize, 1), "VALID") elif desc_pool == "avg_pool": return tf.nn.avg_pool(cur_in, (1, ksize, ksize, 1), (1, ksize, ksize, 1), "VALID") else: raise RuntimeError('Unknown pooling type') # now abuse cur_in so that we can simply copy paste cur_in = inputs # lets apply batch normalization on the input - we did not normalize the # input range! with tf.variable_scope("input-bn"): if config.use_input_batch_norm: cur_in = batch_norm(cur_in, training=is_training) with tf.variable_scope("conv-act-pool-norm-1"): cur_in = conv_2d(cur_in, 7, 32, 1, "VALID") if config.use_batch_norm: cur_in = batch_norm(cur_in, training=is_training) cur_in = activ(cur_in) cur_in = pool(cur_in, config.desc_pool, 2) if config.use_subtractive_norm: cur_in = norm_spatial_subtractive(cur_in, sub_kernel) with tf.variable_scope("conv-act-pool-norm-2"): cur_in = conv_2d(cur_in, 6, 64, 1, "VALID") if config.use_batch_norm: cur_in = batch_norm(cur_in, training=is_training) cur_in = activ(cur_in) cur_in = pool(cur_in, config.desc_pool, 3) if config.use_subtractive_norm: cur_in = norm_spatial_subtractive(cur_in, sub_kernel) with tf.variable_scope("conv-act-pool-3"): cur_in = conv_2d(cur_in, 5, 128, 1, "VALID") if config.use_batch_norm: cur_in = batch_norm(cur_in, training=is_training) cur_in = activ(cur_in) cur_in = pool(cur_in, config.desc_pool, 4) res["desc"] = tf.reshape(cur_in, (-1, 128)) return res
def comp_process(mode, data, res_dir, config): # Unpack some references xs = data["xs"] ys = data["ys"] Rs = data["Rs"] ts = data["ts"] img1s = data["img1s"] cx1s = data["cx1s"] cy1s = data["cy1s"] f1s = data["f1s"] img2s = data["img2s"] cx2s = data["cx2s"] cy2s = data["cy2s"] f2s = data["f2s"] # Make fs numpy array f1s = np.array(f1s) f2s = np.array(f2s) # Prepare directory if not os.path.exists(res_dir): os.makedirs(res_dir) print("[{}] {}: Start testing".format(config.data_tr, time.asctime())) # Validation num_sample = len(xs) if config.use_lift: comp_list = [ "lmeds", "ransac", "mlesac", # "usac5point", "usac8point", # "usacnolo5point", "usacnolo8point" ] else: # comp_list = [ # "lmeds", "ransac", "top8_8point", "top50_lmeds", "top50_ransac", # "gms", "gms_orb", "gms_default", # "gms_orb_resize", "gms_orb_resize_ransac", # "gms_orb_resize_tt", "gms_orb_resize_ransac_tt", # "mlesac", # ] comp_list = [ "lmeds", "ransac", "mlesac", # "usac5point", "usac8point", # "usacnolo5point", "usacnolo8point" ] if config.obj_num_kp == 2000: comp_list += [ "gms_orb_resize_ransac_tt", "gms_orb_resize_tt", ] # Initialize arrays that will store measurements err_q = {} err_t = {} num = {} for _comp in comp_list: err_q[_comp] = np.zeros(num_sample) err_t[_comp] = np.zeros(num_sample) num[_comp] = np.zeros(num_sample) NUM_KP = config.obj_num_kp # batch_size = config.val_batch_size # num_batch = int(len(xs) / batch_size) from gms_matcher import GmsMatcher # SIFT sift = cv2.xfeatures2d.SIFT_create(nfeatures=NUM_KP, contrastThreshold=1e-5) if cv2.__version__.startswith('3'): sift_matcher = cv2.BFMatcher(cv2.NORM_L2) else: sift_matcher = cv2.BFMatcher_create(cv2.NORM_L2) sift_gms = GmsMatcher(sift, sift_matcher) # ORB orb = cv2.ORB_create(10000) orb.setFastThreshold(0) if cv2.__version__.startswith('3'): orb_matcher = cv2.BFMatcher(cv2.NORM_HAMMING) else: orb_matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING) orb_gms = GmsMatcher(orb, orb_matcher) for method_name in comp_list: # Check res_dir if we have the dump ready dump_dir = os.path.join(res_dir, mode, method_name) if not os.path.exists(dump_dir): os.makedirs(dump_dir) # Check for full dump full_dump_file = os.path.join(dump_dir, "qtn_all.h5") if not os.path.exists(full_dump_file) or config.vis_dump: for _idx in xrange(num_sample): print("\rWorking on {} / {}".format(_idx + 1, num_sample), end="") sys.stdout.flush() dump_file = os.path.join(dump_dir, "qtn_{}.txt".format(_idx)) # If dump exists, just load it if os.path.exists(dump_file) and not config.vis_dump: with open(dump_file, "r") as ifp: dump_res = ifp.read() dump_res = parse("{err_q:e}, {err_t:e}, {num_inlier:d}\n", dump_res) _err_q = dump_res["err_q"] _err_t = dump_res["err_t"] _num_inlier = dump_res["num_inlier"] else: # Otherwise compute _xs = xs[_idx][:, :, :].reshape(1, 1, -1, 4) _ys = ys[_idx][:, :].reshape(1, -1, 2) _dR = Rs[_idx] _dt = ts[_idx] # Eval decompose for all pairs _xs = _xs.reshape(-1, 4) # x coordinates _x1 = _xs[:, :2] _x2 = _xs[:, 2:] # Prepare input if method_name == "lmeds": eval_func = eval_decompose _method = cv2.LMEDS _mask = None elif method_name == "ransac": eval_func = eval_decompose _method = cv2.RANSAC _mask = None elif method_name == "mlesac": eval_func = eval_decompose _method = "MLESAC" _mask = None elif method_name == "gms": eval_func = eval_decompose_8points _method = None sift_gms.empty_matches() _x1, _x2, _mask = sift_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=True, with_rotation=True) elif method_name == "gms_default": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _x1, _x2, _mask = orb_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=False, with_rotation=False) elif method_name == "gms_orb_resize_ransac_tt": eval_func = eval_decompose _method = cv2.RANSAC orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=True, with_rotation=True) elif method_name == "gms_orb_resize_tt": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=True, with_rotation=True) elif method_name == "gms_orb_resize_ransac": eval_func = eval_decompose _method = cv2.RANSAC orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=False, with_rotation=False) elif method_name == "gms_orb_resize": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=False, with_rotation=False) elif method_name == "gms_orb": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _x1, _x2, _mask = orb_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=True, with_rotation=True) # Compute errors _err_q, _err_t, _, _, _num_inlier, _mask_after = eval_func( _x1, _x2, _dR, _dt, mask=_mask, method=_method) if config.vis_dump: dump_val_res( img1s[_idx], img2s[_idx], _x1, _x2, _mask, _mask_after, cx1s[_idx], cy1s[_idx], f1s[_idx], cx2s[_idx], cy2s[_idx], f2s[_idx], Rs[_idx], ts[_idx], os.path.join(res_dir, mode, "match", method_name, "pair{:08d}".format(_idx)), ) else: # Write dump with open(dump_file, "w") as ofp: ofp.write("{:e}, {:e}, {:d}\n".format( _err_q, _err_t, _num_inlier)) # Load them in list err_q[method_name][_idx] = _err_q err_t[method_name][_idx] = _err_t num[method_name][_idx] = _num_inlier if not config.vis_dump: # Save to full dump dump_dict = {} dump_dict["err_q"] = err_q[method_name] dump_dict["err_t"] = err_t[method_name] dump_dict["num"] = num[method_name] saveh5(dump_dict, full_dump_file) # Remove all intermediate cache for _f in os.listdir(dump_dir): if _f.startswith("qtn_") and _f.endswith(".txt"): os.remove(os.path.join(dump_dir, _f)) # Load the full dump file else: dump_dict = loadh5(full_dump_file) err_q[method_name] = dump_dict["err_q"] err_t[method_name] = dump_dict["err_t"] num[method_name] = dump_dict["num"] # Remove all intermediate cache for _f in os.listdir(dump_dir): if _f.startswith("qtn_") and _f.endswith(".txt"): os.remove(os.path.join(dump_dir, _f)) print("") if config.vis_dump: return # Report results for _tag in comp_list: # For median error ofn = os.path.join(res_dir, mode, "median_err_q_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(err_q[_tag]))) ofn = os.path.join(res_dir, mode, "median_err_t_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(err_t[_tag]))) ofn = os.path.join(res_dir, mode, "median_num_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(num[_tag]))) # For accuracy AUC ths = np.arange(7) * 5 cur_err_q = np.array(err_q[_tag]) * 180.0 / np.pi cur_err_t = np.array(err_t[_tag]) * 180.0 / np.pi # Get histogram q_acc_hist, _ = np.histogram(cur_err_q, ths) t_acc_hist, _ = np.histogram(cur_err_t, ths) qt_acc_hist, _ = np.histogram(np.maximum(cur_err_q, cur_err_t), ths) num_pair = float(len(cur_err_q)) q_acc_hist = q_acc_hist.astype(float) / num_pair t_acc_hist = t_acc_hist.astype(float) / num_pair qt_acc_hist = qt_acc_hist.astype(float) / num_pair q_acc = np.cumsum(q_acc_hist) t_acc = np.cumsum(t_acc_hist) qt_acc = np.cumsum(qt_acc_hist) for _idx_th in xrange(1, len(ths)): # for q_auc ofn = os.path.join(res_dir, mode, "acc_q_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(q_acc[:_idx_th]))) # for t_auc ofn = os.path.join(res_dir, mode, "acc_t_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(t_acc[:_idx_th]))) # for qt_auc ofn = os.path.join( res_dir, mode, "acc_qt_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(qt_acc[:_idx_th]))) print("[{}] {}: End testing".format(config.data_tr, time.asctime()))