def dump_val_res(img1, img2, x1, x2, mask_before, mask_after, cx1, cy1, f1, cx2, cy2, f2, R, t, dump): if not os.path.exists(dump): os.makedirs(dump) # Images img1 = img1.transpose(1, 2, 0) img2 = img2.transpose(1, 2, 0) cv2.imwrite(os.path.join(dump, "img1.png"), img1) cv2.imwrite(os.path.join(dump, "img2.png"), img2) dump_dict = {} dump_dict["x1"] = x1 dump_dict["cx1"] = cx1 dump_dict["cy1"] = cy1 dump_dict["f1"] = f1 dump_dict["x2"] = x2 dump_dict["cx2"] = cx2 dump_dict["cy2"] = cy2 dump_dict["f2"] = f2 dump_dict["R"] = R dump_dict["t"] = t if mask_before is not None: dump_dict["mask_before"] = mask_before if mask_after is not None: dump_dict["mask_after"] = mask_after saveh5(dump_dict, os.path.join(dump, "dump.h5"))
def dump_data_pair(args): dump_dir, idx, ii, jj, queue = args # queue for monitoring if queue is not None: queue.put(idx) dump_file = os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)) # txt_dict = os.path.join(dump_dir, "idx_sort-{}-{}.txt".format(ii, jj)) if not os.path.exists(dump_file): # Load descriptors for ii desc_ii = loadh5(os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(ii)))["desc"] desc_jj = loadh5(os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(jj)))["desc"] # compute decriptor distance matrix distmat = np.sqrt( np.sum( (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2, axis=2)) # Choose K best from N idx_sort = np.argsort(distmat, axis=1)[:, :config.obj_num_nn] idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None], idx_sort.shape[1], axis=1), idx_sort) distmat = distmat[idx_sort] # Dump to disk dump_dict = {} dump_dict["idx_sort"] = idx_sort saveh5(dump_dict, dump_file)
def dump_data_pair(args): dump_dir, idx, ii, jj, queue = args # queue for monitoring if queue is not None: queue.put(idx) dump_file = os.path.join( dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)) dump_file_mutual_ratio = os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)) if not os.path.exists(dump_file) or not os.path.exists(dump_file_mutual_ratio): # if 1==1: # Load descriptors for ii desc_ii = loadh5( os.path.join(dump_dir, "kp-z-desc-{}.h5".format(ii)))["desc"] desc_jj = loadh5( os.path.join(dump_dir, "kp-z-desc-{}.h5".format(jj)))["desc"] # compute decriptor distance matrix # distmat = np.sqrt( # np.sum( # (np.expand_dims(desc_ii, 1) - np.expand_dims(desc_jj, 0))**2, # axis=2)) ##### replace with faster distmat computation distmat = np.sqrt(np.sum(desc_ii**2, axis=1, keepdims=True) + np.sum(desc_jj**2, axis=1) - 2 * np.dot(desc_ii, desc_jj.T)) # Choose K best from N idx_sort0 = np.argsort(distmat, axis=1)[:, :1] idx_sort = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort0.shape[1], axis=1),idx_sort0) # saving the ratio idx_sort_2nd = np.argsort(distmat, axis=1)[:, :2] idx_sort_2nd = (np.repeat(np.arange(distmat.shape[0])[..., None],idx_sort_2nd.shape[1], axis=1),idx_sort_2nd) dist2nd = distmat[idx_sort_2nd] ratio = dist2nd[:, 0] / dist2nd[:, 1] # saving mutual neighbor information # Please note that crossCheck in opencv doesn't work properly and thus is deprecated in this code. idx_sort1 = np.argsort(distmat, axis=0)[0, :] mutual_neigh = idx_sort1[idx_sort0.squeeze()] == np.arange(idx_sort0.shape[0]) # Dump to disk dump_dict = {} dump_dict_mutual_ratio = {} dump_dict["idx_sort"] = idx_sort dump_dict_mutual_ratio["ratio"] = ratio dump_dict_mutual_ratio["mutual"] = mutual_neigh # Looks ugly because we just add ratio and mutual neighboring information to CNe dataset if not os.path.exists(dump_file): saveh5(dump_dict, dump_file) if not os.path.exists(dump_file_mutual_ratio): saveh5(dump_dict_mutual_ratio, dump_file_mutual_ratio)
def dump_nn(self, ii, jj): dump_file = os.path.join(self.intermediate_dir, "nn-{}-{}.h5".format(ii, jj)) if not os.path.exists(dump_file): image_i, image_j = self.image_fullpath_list[ii], self.image_fullpath_list[jj] desc_ii = loadh5(image_i+'.'+self.desc_name+'.hdf5')["descriptors"] desc_jj = loadh5(image_j+'.'+self.desc_name+'.hdf5')["descriptors"] idx_sort, ratio_test, mutual_nearest = computeNN(desc_ii, desc_jj) # Dump to disk dump_dict = {} dump_dict["idx_sort"] = idx_sort dump_dict["ratio_test"] = ratio_test dump_dict["mutual_nearest"] = mutual_nearest saveh5(dump_dict, dump_file)
def _compute_desc(self): """Compute Descriptors """ total_time = 0.0 # Read image start_time = time.clock() cur_data = self.dataset.load_data() end_time = time.clock() load_time = (end_time - start_time) * 1000.0 if self.config.verbose: print("Time taken to load patches is {} ms".format(load_time)) total_time += load_time # Test using the test function start_time = time.clock() descs = self._test_multibatch(cur_data) end_time = time.clock() compute_time = (end_time - start_time) * 1000.0 if self.config.verbose: print("Time taken to compute is {} ms".format(compute_time)) total_time += compute_time if self.config.verbose: print("Total time for descriptor is {} ms".format(total_time)) # Overwrite angle kps = cur_data["kps"].copy() kps[:, 3] = cur_data["angle"][:, 0] # Save as h5 file save_dict = {} # save_dict['keypoints'] = cur_data["kps"] save_dict['keypoints'] = kps save_dict['descriptors'] = descs saveh5(save_dict, self.config.test_out_file)
def make_xy(pair_index, img, kp, desc, aff, K, R, t, cur_folder): kp_initial = copy.deepcopy(kp) xs = [] xs_initial = [] ys = [] Rs = [] ts = [] img1s = [] img2s = [] cx1s = [] cy1s = [] f1s = [] cx2s = [] cy2s = [] f2s = [] affine = [] # Create a random folder in scratch dump_dir = os.path.join(cur_folder, "dump") if not os.path.exists(dump_dir): os.makedirs(dump_dir) pool_arg = [] for idx in range(pair_index.shape[0]): ii = pair_index[idx, 0] jj = pair_index[idx, 1] print("\rExtracting keypoints {} / {}".format(idx, pair_index.shape[0]), end="") sys.stdout.flush() # Check and extract keypoints if necessary for i in [ii, jj]: i = int(i) dump_file = os.path.join(dump_dir, "kp-aff-desc-{}.h5".format(i)) if not os.path.exists(dump_file): # Correct coordinates using K cx = K[i, 2] cy = K[i, 5] # Correct focals fx = K[i, 0] fy = K[i, 4] kp[i] = (kp[i] - np.array([[cx, cy]])) / np.asarray([[fx, fy]]) # Write descs to harddisk to parallize dump_dict = {} dump_dict["kp_normal"] = kp[i] dump_dict["kp"] = kp_initial[i] dump_dict["desc"] = desc[i] dump_dict["aff"] = aff[i] saveh5(dump_dict, dump_file) else: dump_dict = loadh5(dump_file) kp[i] = dump_dict["kp_normal"] kp_initial[i] = dump_dict["kp"] desc[i] = dump_dict["desc"] aff[i] = dump_dict["aff"] pool_arg += [(dump_dir, idx, int(ii), int(jj))] print("") # Run mp job ratio_CPU = 0.9 number_of_process = int(ratio_CPU * mp.cpu_count()) pool = mp.Pool(processes=number_of_process) manager = mp.Manager() queue = manager.Queue() for idx_arg in xrange(len(pool_arg)): pool_arg[idx_arg] = pool_arg[idx_arg] + (queue, ) # map async pool_res = pool.map_async(dump_data_pair, pool_arg) # monitor loop while True: if pool_res.ready(): break else: size = queue.qsize() print("\rDistMat {} / {}".format(size, len(pool_arg)), end="") sys.stdout.flush() time.sleep(1) pool.close() pool.join() print("") # Pack data idx = 0 total_num = 0 good_num = 0 bad_num = 0 ratio1 = [] ratio2 = [] for idx in range(pair_index.shape[0]): ii = int(pair_index[idx, 0]) jj = int(pair_index[idx, 1]) print("\rWorking on {} / {}".format(idx, pair_index.shape[0]), end="") sys.stdout.flush() K1 = K[ii].reshape(3, 3) K2 = K[jj].reshape(3, 3) # ------------------------------ # Get dR R_i = R[ii].reshape(3, 3) R_j = R[jj].reshape(3, 3) dR = np.dot(R_j, R_i.T) # Get dt t_i = t[ii].reshape(3, 1) t_j = t[jj].reshape(3, 1) dt = t_j - np.dot(dR, t_i) # ------------------------------ # Get keypoints for the first image x1 = kp[ii] y1 = np.concatenate((kp[ii], np.ones((kp[ii].shape[0], 1))), axis=1) # Project the first points into the second image y1p = np.matmul(dR[None], y1[..., None]) + dt[None] # move back to the canonical plane x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None] # ------------------------------ # Get keypoints for the second image x2 = kp[jj] # # DEBUG ------------------------------ # # Check if the image projections make sense # draw_val_res( # img[ii], # img[jj], # x1, x1p, np.random.rand(x1.shape[0]) < 0.1, # (img[ii][0].shape[1] - 1.0) * 0.5, # (img[ii][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][ii, 0, 0], # (img[jj][0].shape[1] - 1.0) * 0.5, # (img[jj][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][jj, 0, 0], # "./debug_imgs/", # "debug_img{:04d}.png".format(idx) # ) # ------------------------------ # create x1, y1, x2, y2 as a matrix combo x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1) y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) # Load precomputed nearest neighbors idx_sort = loadh5( os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x1pmat = x1pmat[idx_sort] y1pmat = y1pmat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x1p = np.concatenate([x1pmat.reshape(-1, 1), y1pmat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) # make xs in NHWC xs += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] # ------------------------------ # Get the geodesic distance using with x1, x2, dR, dt if config.obj_geod_type == "sampson": geod_d = get_sampsons(x1, x2, dR, dt) elif config.obj_geod_type == "episqr": geod_d = get_episqr(x1, x2, dR, dt) elif config.obj_geod_type == "episym": geod_d = get_episym(x1, x2, dR, dt) # geod_d = get_episym(x1, x2, dR, dt, K1_inv, K2_inv, K1, K2) # Get *rough* reprojection errors. Note that the depth may be noisy. We # ended up not using this... reproj_d = np.sum((x2 - x1p)**2, axis=1) # count inliers and outliers total_num += len(geod_d) good_num += np.sum((geod_d < config.obj_geod_th)) bad_num += np.sum((geod_d >= config.obj_geod_th)) ''' mask = np.zeros(len(geod_d)) for i in range(len(geod_d)): if geod_d[i] < config.obj_geod_th: mask[i] = 1 np.savetxt(os.path.join(dump_dir, "mask-{}-{}.txt".format(ii, jj)), mask) ''' ys += [np.stack([geod_d, reproj_d], axis=1)] # Save R, t for evaluation Rs += [np.array(dR).reshape(3, 3)] # normalize t before saving dtnorm = np.sqrt(np.sum(dt**2)) assert (dtnorm > 1e-5) dt /= dtnorm ts += [np.array(dt).flatten()] # Save img1 and img2 for display img1s += [img[ii]] img2s += [img[jj]] cx = K[ii, 2] cy = K[ii, 5] cx1s += [cx] cy1s += [cy] cx = K[jj, 2] cy = K[jj, 5] cx2s += [cx] cy2s += [cy] fx = K[ii, 0] fy = K[ii, 4] if np.isclose(fx, fy): f = fx else: f = (fx, fy) f1s += [f] fx = K[jj, 0] fy = K[jj, 4] if np.isclose(fx, fy): f = fx else: f = (fx, fy) f2s += [f] # Generate xs_initial and T-transform x1 = kp_initial[ii] x2 = kp_initial[jj] aff1 = aff[ii] aff2 = aff[jj] aff1 = np.repeat(aff1[:, :][..., None], len(aff2), axis=-1).transpose(0, 2, 1) aff2 = np.repeat(aff2[:, :][None], len(aff1), axis=0) x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) idx_sort = loadh5( os.path.join(dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] aff1 = aff1[idx_sort] aff2 = aff2[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) affine += [ np.concatenate( [aff1.reshape(-1, 9), aff2.reshape(-1, 9)], axis=1) ] xs_initial += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] print("") # Do *not* convert to numpy arrays, as the number of keypoints may differ # now. Simply return it print(".... done") if total_num > 0: print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format( good_num, total_num, float(good_num) / float(total_num))) print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format( bad_num, total_num, float(bad_num) / float(total_num))) res_dict = {} res_dict["xs"] = xs res_dict["xs_initial"] = xs_initial res_dict["affine"] = affine res_dict["ys"] = ys res_dict["Rs"] = Rs res_dict["ts"] = ts res_dict["img1s"] = img1s res_dict["cx1s"] = cx1s res_dict["cy1s"] = cy1s res_dict["f1s"] = f1s res_dict["img2s"] = img2s res_dict["cx2s"] = cx2s res_dict["cy2s"] = cy2s res_dict["f2s"] = f2s return res_dict
def _compute_desc(self): """Compute Descriptors """ total_time = 0.0 global_start_time = time.time() for scene_name in cst.SCENE_LIST: duration = time.time() - global_start_time print('****** %s ****** %d:%02d' % (scene_name, duration / 60, duration % 60)) if not os.path.exists(os.path.join(cst.DES_DIR, scene_name)): os.makedirs(os.path.join(cst.DES_DIR, scene_name)) for img_key in range(1, cst.MAX_IMG_NUM + 1): if ((scene_name == 'v_feast' and img_key == 6) or (scene_name == 'v_wall' and img_key == 6)): if cst.DATA == 'hpatches': continue img_fn = os.path.join(cst.DATA_DIR, scene_name, '%d.ppm' % (img_key)) new_kp_fn = os.path.join(cst.ORI_DIR, scene_name, '%d_kp.txt' % (img_key)) des_fn = os.path.join(cst.DES_DIR, scene_name, '%d.h5' % (img_key)) # Read image start_time = time.clock() cur_data = self.dataset.my_load_data(img_fn, new_kp_fn, cst.NEW_SIZE) end_time = time.clock() load_time = (end_time - start_time) * 1000.0 #print("Time taken to load patches is {} ms".format( # load_time #)) total_time += load_time # import IPython # IPython.embed() # ------------------------------------------------------------------------- # Test using the test function start_time = time.clock() descs = self._test_multibatch(cur_data) end_time = time.clock() compute_time = (end_time - start_time) * 1000.0 #print("Time taken to compute is {} ms".format( # compute_time #)) total_time += compute_time #print("Total time for descriptor is {} ms".format(total_time)) # Overwrite angle kps = cur_data["kps"].copy() kps[:, 3] = cur_data["angle"][:, 0] # Save as h5 file save_dict = {} # save_dict['keypoints'] = cur_data["kps"] save_dict['keypoints'] = kps save_dict['descriptors'] = descs saveh5(save_dict, des_fn)
def make_xy(num_sample, pairs, kp, z, desc, img, geom, vis, depth, geom_type, cur_folder): xs = [] ys = [] Rs = [] ts = [] mutuals = [] ratios = [] img1s = [] img2s = [] cx1s = [] cy1s = [] f1s = [] cx2s = [] cy2s = [] f2s = [] # Create a random folder in scratch dump_dir = os.path.join(cur_folder, "dump") if not os.path.exists(dump_dir): os.makedirs(dump_dir) # randomly suffle the pairs and select num_sample amount np.random.seed(1234) cur_pairs = [ pairs[_i] for _i in np.random.permutation(len(pairs))[:num_sample] ] idx = 0 for ii, jj in cur_pairs: idx += 1 print( "\rExtracting keypoints {} / {}".format(idx, len(cur_pairs)), end="") sys.stdout.flush() # Check and extract keypoints if necessary for i in [ii, jj]: dump_file = os.path.join(dump_dir, "kp-z-desc-{}.h5".format(i)) if not os.path.exists(dump_file): if kp[i] is None: cv_kp, cv_desc = sift.detectAndCompute(img[i].transpose( 1, 2, 0), None) cx = (img[i][0].shape[1] - 1.0) * 0.5 cy = (img[i][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][i, 0, 2] cy += parse_geom(geom, geom_type)["K"][i, 1, 2] xy = np.array([_kp.pt for _kp in cv_kp]) # Correct focals fx = parse_geom(geom, geom_type)["K"][i, 0, 0] fy = parse_geom(geom, geom_type)["K"][i, 1, 1] kp[i] = ( xy - np.array([[cx, cy]]) ) / np.asarray([[fx, fy]]) desc[i] = cv_desc if z[i] is None: cx = (img[i][0].shape[1] - 1.0) * 0.5 cy = (img[i][0].shape[0] - 1.0) * 0.5 fx = parse_geom(geom, geom_type)["K"][i, 0, 0] fy = parse_geom(geom, geom_type)["K"][i, 1, 1] xy = kp[i] * np.asarray([[fx, fy]]) + np.array([[cx, cy]]) if len(depth) > 0: z[i] = depth[i][ 0, np.round(xy[:, 1]).astype(int), np.round(xy[:, 0]).astype(int)][..., None] else: z[i] = np.ones((xy.shape[0], 1)) # Write descs to harddisk to parallize dump_dict = {} dump_dict["kp"] = kp[i] dump_dict["z"] = z[i] dump_dict["desc"] = desc[i] saveh5(dump_dict, dump_file) else: dump_dict = loadh5(dump_file) kp[i] = dump_dict["kp"] z[i] = dump_dict["z"] desc[i] = dump_dict["desc"] print("") # Create arguments pool_arg = [] idx = 0 for ii, jj in cur_pairs: idx += 1 pool_arg += [(dump_dir, idx, ii, jj)] # Run mp job ratio_CPU = 0.8 number_of_process = int(ratio_CPU * mp.cpu_count()) pool = mp.Pool(processes=number_of_process) manager = mp.Manager() queue = manager.Queue() # for debugging in dump_data_pair # dump_data_pair(pool_arg[1] + (queue,)) for idx_arg in xrange(len(pool_arg)): pool_arg[idx_arg] = pool_arg[idx_arg] + (queue,) # map async pool_res = pool.map_async(dump_data_pair, pool_arg) # monitor loop while True: if pool_res.ready(): break else: size = queue.qsize() print("\rDistMat {} / {}".format(size, len(pool_arg)), end="") sys.stdout.flush() time.sleep(1) pool.close() pool.join() print("") # Pack data idx = 0 total_num = 0 good_num = 0 bad_num = 0 for ii, jj in cur_pairs: idx += 1 print("\rWorking on {} / {}".format(idx, len(cur_pairs)), end="") sys.stdout.flush() # ------------------------------ # Get dR R_i = parse_geom(geom, geom_type)["R"][ii] R_j = parse_geom(geom, geom_type)["R"][jj] dR = np.dot(R_j, R_i.T) # Get dt t_i = parse_geom(geom, geom_type)["t"][ii].reshape([3, 1]) t_j = parse_geom(geom, geom_type)["t"][jj].reshape([3, 1]) dt = t_j - np.dot(dR, t_i) # ------------------------------ # Get sift points for the first image x1 = kp[ii] y1 = np.concatenate([kp[ii] * z[ii], z[ii]], axis=1) # Project the first points into the second image y1p = np.matmul(dR[None], y1[..., None]) + dt[None] # move back to the canonical plane x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None] # ------------------------------ # Get sift points for the second image x2 = kp[jj] # # DEBUG ------------------------------ # # Check if the image projections make sense # draw_val_res( # img[ii], # img[jj], # x1, x1p, np.random.rand(x1.shape[0]) < 0.1, # (img[ii][0].shape[1] - 1.0) * 0.5, # (img[ii][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][ii, 0, 0], # (img[jj][0].shape[1] - 1.0) * 0.5, # (img[jj][0].shape[0] - 1.0) * 0.5, # parse_geom(geom, geom_type)["K"][jj, 0, 0], # "./debug_imgs/", # "debug_img{:04d}.png".format(idx) # ) # ------------------------------ # create x1, y1, x2, y2 as a matrix combo x1mat = np.repeat(x1[:, 0][..., None], len(x2), axis=-1) y1mat = np.repeat(x1[:, 1][..., None], len(x2), axis=1) x1pmat = np.repeat(x1p[:, 0][..., None], len(x2), axis=-1) y1pmat = np.repeat(x1p[:, 1][..., None], len(x2), axis=1) x2mat = np.repeat(x2[:, 0][None], len(x1), axis=0) y2mat = np.repeat(x2[:, 1][None], len(x1), axis=0) # Load precomputed nearest neighbors, ratios and mutual idx_sort = loadh5(os.path.join( dump_dir, "idx_sort-{}-{}.h5".format(ii, jj)))["idx_sort"] mutual = loadh5(os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["mutual"] ratio = loadh5(os.path.join( dump_dir, "mutual_ratio-{}-{}.h5".format(ii, jj)))["ratio"] mutuals += [mutual] ratios += [ratio] # Move back to tuples idx_sort = (idx_sort[0], idx_sort[1]) x1mat = x1mat[idx_sort] y1mat = y1mat[idx_sort] x1pmat = x1pmat[idx_sort] y1pmat = y1pmat[idx_sort] x2mat = x2mat[idx_sort] y2mat = y2mat[idx_sort] # Turn into x1, x1p, x2 x1 = np.concatenate( [x1mat.reshape(-1, 1), y1mat.reshape(-1, 1)], axis=1) x1p = np.concatenate( [x1pmat.reshape(-1, 1), y1pmat.reshape(-1, 1)], axis=1) x2 = np.concatenate( [x2mat.reshape(-1, 1), y2mat.reshape(-1, 1)], axis=1) # make xs in NHWC xs += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] # ------------------------------ # Get the geodesic distance using with x1, x2, dR, dt if config.obj_geod_type == "sampson": geod_d = get_sampsons(x1, x2, dR, dt) elif config.obj_geod_type == "episqr": geod_d = get_episqr(x1, x2, dR, dt) elif config.obj_geod_type == "episym": geod_d = get_episym(x1, x2, dR, dt) # Get *rough* reprojection errors. Note that the depth may be noisy. We # ended up not using this... reproj_d = np.sum((x2 - x1p)**2, axis=1) # count inliers and outliers total_num += len(geod_d) good_num += np.sum((geod_d < config.obj_geod_th)) bad_num += np.sum((geod_d >= config.obj_geod_th)) ys += [np.stack([geod_d, reproj_d], axis=1)] # Save R, t for evaluation Rs += [np.array(dR).reshape(3, 3)] # normalize t before saving dtnorm = np.sqrt(np.sum(dt**2)) assert (dtnorm > 1e-5) dt /= dtnorm ts += [np.array(dt).flatten()] # Save img1 and img2 for display img1s += [img[ii]] img2s += [img[jj]] cx = (img[ii][0].shape[1] - 1.0) * 0.5 cy = (img[ii][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][ii, 0, 2] cy += parse_geom(geom, geom_type)["K"][ii, 1, 2] fx = parse_geom(geom, geom_type)["K"][ii, 0, 0] fy = parse_geom(geom, geom_type)["K"][ii, 1, 1] if np.isclose(fx, fy): f = fx else: f = (fx, fy) cx1s += [cx] cy1s += [cy] f1s += [f] cx = (img[jj][0].shape[1] - 1.0) * 0.5 cy = (img[jj][0].shape[0] - 1.0) * 0.5 # Correct coordinates using K cx += parse_geom(geom, geom_type)["K"][jj, 0, 2] cy += parse_geom(geom, geom_type)["K"][jj, 1, 2] fx = parse_geom(geom, geom_type)["K"][jj, 0, 0] fy = parse_geom(geom, geom_type)["K"][jj, 1, 1] if np.isclose(fx, fy): f = fx else: f = (fx, fy) cx2s += [cx] cy2s += [cy] f2s += [f] # Do *not* convert to numpy arrays, as the number of keypoints may differ # now. Simply return it print(".... done") if total_num > 0: print(" Good pairs = {}, Total pairs = {}, Ratio = {}".format( good_num, total_num, float(good_num) / float(total_num))) print(" Bad pairs = {}, Total pairs = {}, Ratio = {}".format( bad_num, total_num, float(bad_num) / float(total_num))) res_dict = {} res_dict["xs"] = xs res_dict["ys"] = ys res_dict["Rs"] = Rs res_dict["ts"] = ts res_dict["img1s"] = img1s res_dict["cx1s"] = cx1s res_dict["cy1s"] = cy1s res_dict["f1s"] = f1s res_dict["img2s"] = img2s res_dict["cx2s"] = cx2s res_dict["cy2s"] = cy2s res_dict["f2s"] = f2s res_dict["mutuals"] = mutuals res_dict["ratios"] = ratios res_dict["pairs"] = cur_pairs return res_dict
def createDump(args): # print("createDump called") idx_jpg, jpg_file, train_data_dir, dump_data_dir, tmp_patch_dir, scale_hist, scale_hist_c, out_dim, param, queue = args # print(idx_jpg) # print(jpg_file) # print(args) # queue for monitoring if queue is not None: queue.put(idx_jpg) final_dump_file_name = os.path.join( tmp_patch_dir, os.path.basename(jpg_file).split(".")[0] + ".h5") # print(final_dump_file_name) if not os.path.exists(final_dump_file_name): # load image bUseColorImage = getattr(param.patch, "bUseColorImage", False) if not bUseColorImage: bUseDebugImage = getattr(param.patch, "bUseDebugImage", False) if not bUseDebugImage: img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY) else: # debug image contains keypoints survive the SfM # pipeline (blue) and the rest (in red) img = cv2.cvtColor(cv2.imread(jpg_file), cv2.COLOR_BGR2GRAY) # debug_jpg = train_data_dir + jpg_file.replace(".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) +".jpg") debug_jpg = os.path.splitext(jpg_file)[0] + "-kp-minsc-" + str( param.dataset.fMinKpSize) + ".jpg" imgd = cv2.cvtColor(cv2.imread(debug_jpg), cv2.COLOR_BGR2GRAY) img = cv2.resize(imgd, img.shape[:2][::-1]) in_dim = 1 else: img = cv2.imread(train_data_dir + jpg_file) in_dim = 3 assert (img.shape[-1] == in_dim) # ---------------------------------------- # load kp data # Note that data is in order of [x,y,scale,angle,pointid,setid]] # guessing here: pointid is the point id number for the keypoint in the two 2d image # or maybe it's a unique idx for a point in a list of all points? # of maybe its the index of a point for all points in a set?(I think it's this last one, maybe) # setid is the id num for the pair which is created from the image keypoints (this would keep a uniform ) # For positive: select randomly from "valid_keypoints" pos_kp_file_name = jpg_file.replace( ".jpg", "-kp-minsc-" + str(param.dataset.fMinKpSize) + ".h5") # print(pos_kp_file_name) with h5py.File(pos_kp_file_name, "r") as h5file: sfm_kp = h5file["valid_keypoints"][()] non_sfm_kp = h5file["other_keypoints"][()] # add two dummy fields to non_sfm since they don"t have id # and group. THis means pointid,setid are set to -1 for non_sfm_kp # non_sfm_kp = np.concatenate([non_sfm_kp, -np.ones((non_sfm_kp.shape[0], 2))],axis=1) # if there are no keypoints which were leveraged in SFM if (len(sfm_kp) == 0): sfm_kp = np.empty([0, 6]) if (len(non_sfm_kp) == 0): non_sfm_kp = np.empty([0, 6]) # return None # print(sfm_kp) # print(type(sfm_kp)) # print(sfm_kp.shape) # print(non_sfm_kp) # print(type(non_sfm_kp)) # print(non_sfm_kp.shape) pos_kp = np.concatenate( (sfm_kp, np.ones((sfm_kp.shape[0], 1), dtype="float")), axis=1) if (len(sfm_kp) == 0): pos_kp = pos_kp[1:, :] if (len(non_sfm_kp) == 0): non_sfm_kp = non_sfm_kp[1:, :] # Select subset for positives (assuming we have same coordinates for all image) # dump_file_name = dump_data_dir + jpg_file.replace(".jpg","_idxPosSel.h5") dump_file_name = os.path.join( dump_data_dir, os.path.basename(jpg_file).split(".")[0] + "_idxPosSel.h5") # if dump file not exist, create it; otherwise load it if not os.path.exists(dump_file_name): # idxPosShuffle = np.argsort(pos_kp[3,:])[::-1] # sort backwards idxPosShuffle = np.random.permutation( len(pos_kp)) # random shuffle pos_2_keep = len(idxPosShuffle) if param.dataset.nPosPerImg > 0: pos_2_keep = min(pos_2_keep, param.dataset.nPosPerImg) idxPosSel = idxPosShuffle[:pos_2_keep] # shuffle the points to_save = {"saveval": idxPosSel} # print(to_save) saveh5(to_save, dump_file_name) else: to_load = loadh5(dump_file_name) idxPosSel = to_load["saveval"] pos_kp = pos_kp[idxPosSel] # print(pos_kp) # negative sampling: # # 1) only use Sfm points, too close keypoints will be rejected as # negative pair (because of high potential overlapping) # # 2) Use all SIFT points, when the overlapping of two feature context # windows is larger than a threshold, it will be rejected as a negative # pair (because it shares too much common regions..). In this step, we # concatenate sfm_kp and non_sfm_kp to form keypoint class. neg_mine_method = getattr(param.patch, "sNegMineMethod", "use_only_SfM_points") if neg_mine_method == "use_only_SfM_points": # print("mining sfm points...") # exit() # where is neg_kp = random_mine_non_kp_with_2d_distance( img, sfm_kp, scale_hist, scale_hist_c, param) elif neg_mine_method == "use_all_SIFT_points": # print("mining SIFT points...") # exit() max_iter = getattr(param.patch, "nMaxRandomNegMineIter", 100) # print(sfm_kp) # print(type(sfm_kp)) # print(sfm_kp.shape) # print(non_sfm_kp) # print(type(non_sfm_kp)) # print(non_sfm_kp.shape) # print(pos_kp) # print(type(pos_kp)) # print(pos_kp.shape) try: sift_kp = np.concatenate([sfm_kp, non_sfm_kp], axis=0) neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, max_iter=max_iter) # neg_kp = random_mine_non_kp_with_3d_blocking(img, sift_kp, scale_hist, scale_hist_c, param, neg_per_iter = 10, max_iter=max_iter) except ValueError as err: # print(err) # print("Possibly no non_sfm_keypoints?") neg_kp = np.empty([0, 6]) else: raise ValueError( "Mining method {} is not supported!".format(neg_mine_method)) # add another dim indicating good or bad neg_kp = np.concatenate( (neg_kp, np.zeros((len(neg_kp), 1), dtype="float")), axis=1) # concatenate negative and positives # print(pos_kp) # print(type(pos_kp)) # print(pos_kp.shape) # print(neg_kp) # print(type(neg_kp)) # print(neg_kp.shape) kp = np.concatenate((pos_kp, neg_kp), axis=0) # print(kp) # Retrive target values, 1 for pos and 0 for neg y = kp[:, 6] # Retrieve angles angle = kp[:, 3] # Assign ID to keypoints (for sfm points, ID = 3d point ind; for non # sfm kp, ID = -1) ID = kp[:, 4] # load patches with id (drop out of boundary) bPerturb = getattr(param.patch, "bPerturb", False) fPerturbInfo = getattr(param.patch, "fPerturbInfo", np.zeros((3, ))) nAugmentedRotations = getattr(param.patch, "nAugmentedRotations", 1) fAugmentRange = getattr(param.patch, "fAugmentRange", 0) fAugmentCenterRandStrength = getattr(param.patch, "fAugmentCenterRandStrength", 0) sAugmentCenterRandMethod = getattr(param.patch, "sAugmentCenterRandMethod", "uniform") cur_data_set = load_patches( img, kp, y, ID, angle, param.patch.fRatioScale, param.patch.fMaxScale, param.patch.nPatchSize, param.model.nDescInputSize, in_dim, bPerturb, fPerturbInfo, bReturnCoords=True, nAugmentedRotations=nAugmentedRotations, fAugmentRange=fAugmentRange, fAugmentCenterRandStrength=fAugmentCenterRandStrength, sAugmentCenterRandMethod=sAugmentCenterRandMethod, nPatchSizeAug=param.patch.nPatchSizeAug, ) # save dump as dictionary using saveh5 # from here Kernel died, because of not finding "MKL_intel_thread.dll" # pdb.set_trace() # here the data are saved in tmp_dump file, keys are numbers [0,1..] tmpdict = dict( (str(_idx), np.asarray(_data)) for _idx, _data in zip(np.arange(len(cur_data_set)), cur_data_set)) saveh5(tmpdict, final_dump_file_name) return idx_jpg
def comp_process(mode, data, res_dir, config): # Unpack some references xs = data["xs"] ys = data["ys"] Rs = data["Rs"] ts = data["ts"] img1s = data["img1s"] cx1s = data["cx1s"] cy1s = data["cy1s"] f1s = data["f1s"] img2s = data["img2s"] cx2s = data["cx2s"] cy2s = data["cy2s"] f2s = data["f2s"] # Make fs numpy array f1s = np.array(f1s) f2s = np.array(f2s) # Prepare directory if not os.path.exists(res_dir): os.makedirs(res_dir) print("[{}] {}: Start testing".format(config.data_tr, time.asctime())) # Validation num_sample = len(xs) if config.use_lift: comp_list = [ "lmeds", "ransac", "mlesac", # "usac5point", "usac8point", # "usacnolo5point", "usacnolo8point" ] else: # comp_list = [ # "lmeds", "ransac", "top8_8point", "top50_lmeds", "top50_ransac", # "gms", "gms_orb", "gms_default", # "gms_orb_resize", "gms_orb_resize_ransac", # "gms_orb_resize_tt", "gms_orb_resize_ransac_tt", # "mlesac", # ] comp_list = [ "lmeds", "ransac", "mlesac", # "usac5point", "usac8point", # "usacnolo5point", "usacnolo8point" ] if config.obj_num_kp == 2000: comp_list += [ "gms_orb_resize_ransac_tt", "gms_orb_resize_tt", ] # Initialize arrays that will store measurements err_q = {} err_t = {} num = {} for _comp in comp_list: err_q[_comp] = np.zeros(num_sample) err_t[_comp] = np.zeros(num_sample) num[_comp] = np.zeros(num_sample) NUM_KP = config.obj_num_kp # batch_size = config.val_batch_size # num_batch = int(len(xs) / batch_size) from gms_matcher import GmsMatcher # SIFT sift = cv2.xfeatures2d.SIFT_create(nfeatures=NUM_KP, contrastThreshold=1e-5) if cv2.__version__.startswith('3'): sift_matcher = cv2.BFMatcher(cv2.NORM_L2) else: sift_matcher = cv2.BFMatcher_create(cv2.NORM_L2) sift_gms = GmsMatcher(sift, sift_matcher) # ORB orb = cv2.ORB_create(10000) orb.setFastThreshold(0) if cv2.__version__.startswith('3'): orb_matcher = cv2.BFMatcher(cv2.NORM_HAMMING) else: orb_matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING) orb_gms = GmsMatcher(orb, orb_matcher) for method_name in comp_list: # Check res_dir if we have the dump ready dump_dir = os.path.join(res_dir, mode, method_name) if not os.path.exists(dump_dir): os.makedirs(dump_dir) # Check for full dump full_dump_file = os.path.join(dump_dir, "qtn_all.h5") if not os.path.exists(full_dump_file) or config.vis_dump: for _idx in xrange(num_sample): print("\rWorking on {} / {}".format(_idx + 1, num_sample), end="") sys.stdout.flush() dump_file = os.path.join(dump_dir, "qtn_{}.txt".format(_idx)) # If dump exists, just load it if os.path.exists(dump_file) and not config.vis_dump: with open(dump_file, "r") as ifp: dump_res = ifp.read() dump_res = parse("{err_q:e}, {err_t:e}, {num_inlier:d}\n", dump_res) _err_q = dump_res["err_q"] _err_t = dump_res["err_t"] _num_inlier = dump_res["num_inlier"] else: # Otherwise compute _xs = xs[_idx][:, :, :].reshape(1, 1, -1, 4) _ys = ys[_idx][:, :].reshape(1, -1, 2) _dR = Rs[_idx] _dt = ts[_idx] # Eval decompose for all pairs _xs = _xs.reshape(-1, 4) # x coordinates _x1 = _xs[:, :2] _x2 = _xs[:, 2:] # Prepare input if method_name == "lmeds": eval_func = eval_decompose _method = cv2.LMEDS _mask = None elif method_name == "ransac": eval_func = eval_decompose _method = cv2.RANSAC _mask = None elif method_name == "mlesac": eval_func = eval_decompose _method = "MLESAC" _mask = None elif method_name == "gms": eval_func = eval_decompose_8points _method = None sift_gms.empty_matches() _x1, _x2, _mask = sift_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=True, with_rotation=True) elif method_name == "gms_default": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _x1, _x2, _mask = orb_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=False, with_rotation=False) elif method_name == "gms_orb_resize_ransac_tt": eval_func = eval_decompose _method = cv2.RANSAC orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=True, with_rotation=True) elif method_name == "gms_orb_resize_tt": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=True, with_rotation=True) elif method_name == "gms_orb_resize_ransac": eval_func = eval_decompose _method = cv2.RANSAC orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=False, with_rotation=False) elif method_name == "gms_orb_resize": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _img1 = np.transpose(img1s[_idx], (1, 2, 0)) _img2 = np.transpose(img2s[_idx], (1, 2, 0)) _h1, _w1 = _img1.shape[:2] _h2, _w2 = _img2.shape[:2] _s1 = 480.0 / _h1 _s2 = 480.0 / _h2 _h1 = int(_h1 * _s1) _w1 = int(np.round(_w1 * _s1)) _h2 = int(_h2 * _s2) _w2 = int(np.round(_w2 * _s2)) _img1 = cv2.resize(_img1, (_w1, _h1)) _img2 = cv2.resize(_img2, (_w2, _h2)) _x1, _x2, _mask = orb_gms.compute_matches( _img1, _img2, cx1s[_idx] * _s1, cx2s[_idx] * _s2, cy1s[_idx] * _s1, cy2s[_idx] * _s2, f1s[_idx] * _s1, f2s[_idx] * _s2, with_scale=False, with_rotation=False) elif method_name == "gms_orb": eval_func = eval_decompose_8points _method = None orb_gms.empty_matches() _x1, _x2, _mask = orb_gms.compute_matches( np.transpose(img1s[_idx], (1, 2, 0)), np.transpose(img2s[_idx], (1, 2, 0)), cx1s[_idx], cx2s[_idx], cy1s[_idx], cy2s[_idx], f1s[_idx], f2s[_idx], with_scale=True, with_rotation=True) # Compute errors _err_q, _err_t, _, _, _num_inlier, _mask_after = eval_func( _x1, _x2, _dR, _dt, mask=_mask, method=_method) if config.vis_dump: dump_val_res( img1s[_idx], img2s[_idx], _x1, _x2, _mask, _mask_after, cx1s[_idx], cy1s[_idx], f1s[_idx], cx2s[_idx], cy2s[_idx], f2s[_idx], Rs[_idx], ts[_idx], os.path.join(res_dir, mode, "match", method_name, "pair{:08d}".format(_idx)), ) else: # Write dump with open(dump_file, "w") as ofp: ofp.write("{:e}, {:e}, {:d}\n".format( _err_q, _err_t, _num_inlier)) # Load them in list err_q[method_name][_idx] = _err_q err_t[method_name][_idx] = _err_t num[method_name][_idx] = _num_inlier if not config.vis_dump: # Save to full dump dump_dict = {} dump_dict["err_q"] = err_q[method_name] dump_dict["err_t"] = err_t[method_name] dump_dict["num"] = num[method_name] saveh5(dump_dict, full_dump_file) # Remove all intermediate cache for _f in os.listdir(dump_dir): if _f.startswith("qtn_") and _f.endswith(".txt"): os.remove(os.path.join(dump_dir, _f)) # Load the full dump file else: dump_dict = loadh5(full_dump_file) err_q[method_name] = dump_dict["err_q"] err_t[method_name] = dump_dict["err_t"] num[method_name] = dump_dict["num"] # Remove all intermediate cache for _f in os.listdir(dump_dir): if _f.startswith("qtn_") and _f.endswith(".txt"): os.remove(os.path.join(dump_dir, _f)) print("") if config.vis_dump: return # Report results for _tag in comp_list: # For median error ofn = os.path.join(res_dir, mode, "median_err_q_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(err_q[_tag]))) ofn = os.path.join(res_dir, mode, "median_err_t_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(err_t[_tag]))) ofn = os.path.join(res_dir, mode, "median_num_{}.txt".format(_tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.median(num[_tag]))) # For accuracy AUC ths = np.arange(7) * 5 cur_err_q = np.array(err_q[_tag]) * 180.0 / np.pi cur_err_t = np.array(err_t[_tag]) * 180.0 / np.pi # Get histogram q_acc_hist, _ = np.histogram(cur_err_q, ths) t_acc_hist, _ = np.histogram(cur_err_t, ths) qt_acc_hist, _ = np.histogram(np.maximum(cur_err_q, cur_err_t), ths) num_pair = float(len(cur_err_q)) q_acc_hist = q_acc_hist.astype(float) / num_pair t_acc_hist = t_acc_hist.astype(float) / num_pair qt_acc_hist = qt_acc_hist.astype(float) / num_pair q_acc = np.cumsum(q_acc_hist) t_acc = np.cumsum(t_acc_hist) qt_acc = np.cumsum(qt_acc_hist) for _idx_th in xrange(1, len(ths)): # for q_auc ofn = os.path.join(res_dir, mode, "acc_q_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(q_acc[:_idx_th]))) # for t_auc ofn = os.path.join(res_dir, mode, "acc_t_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(t_acc[:_idx_th]))) # for qt_auc ofn = os.path.join( res_dir, mode, "acc_qt_auc{}_{}.txt".format(ths[_idx_th], _tag)) with open(ofn, "w") as ofp: ofp.write("{}\n".format(np.mean(qt_acc[:_idx_th]))) print("[{}] {}: End testing".format(config.data_tr, time.asctime()))
def _compute_desc(self): """Compute Descriptors """ total_time = 0.0 global_start_time = time.time() for scene_name in cst.SCENE_LIST: print('scene_name: %s'%scene_name) img_dir = '%s/%s/'%(cst.DATA_DIR, scene_name) img_list = [ l for l in sorted(os.listdir(img_dir)) if l[-3:]=='png'] img_num = len(img_list) duration = time.time() - global_start_time print('****** %s ****** %d:%02d'%(scene_name, duration/60, duration%60)) if not os.path.exists(os.path.join(cst.DES_DIR, scene_name)): os.makedirs(os.path.join(cst.DES_DIR, scene_name)) for img_id in range(img_num): root_name = '%04d'%img_id img_fn = os.path.join(img_dir, '%s.png'%root_name) new_kp_fn = os.path.join(cst.ORI_DIR, scene_name,'%s.txt'%(root_name)) des_fn = os.path.join(cst.DES_DIR, scene_name, '%s.h5'%(root_name)) # Read image start_time = time.clock() cur_data = self.dataset.my_load_data(img_fn, new_kp_fn, cst.NEW_SIZE) end_time = time.clock() load_time = (end_time - start_time) * 1000.0 #print("Time taken to load patches is {} ms".format( # load_time #)) total_time += load_time # import IPython # IPython.embed() # ------------------------------------------------------------------------- # Test using the test function start_time = time.clock() descs = self._test_multibatch(cur_data) end_time = time.clock() compute_time = (end_time - start_time) * 1000.0 #print("Time taken to compute is {} ms".format( # compute_time #)) total_time += compute_time #print("Total time for descriptor is {} ms".format(total_time)) # Overwrite angle kps = cur_data["kps"].copy() kps[:, 3] = cur_data["angle"][:, 0] # Save as h5 file save_dict = {} # save_dict['keypoints'] = cur_data["kps"] save_dict['keypoints'] = kps save_dict['descriptors'] = descs saveh5(save_dict, des_fn)
def get_list_of_img(train_data_dir, dump_data_dir, param, mode): # Check if split file exists split_prefix = train_data_dir + 'split-' split_prefix += str(param.dataset.nTrainPercent) + '-' split_prefix += str(param.dataset.nValidPercent) + '-' split_prefix += str(param.dataset.nTestPercent) + '-' # If it does not exist, create one if not os.path.exists(split_prefix + mode + '.txt'): # Read list of images list_png_file = [] for files in os.listdir(train_data_dir): if files.endswith(".png"): list_png_file = list_png_file + [files] # Shuffle the image list if not os.path.exists(dump_data_dir + 'permute_png_idx.h5'): print(' -- ' + mode + ': ' 'Creating new shuffle for reading images') permute_png_idx = np.random.permutation(len(list_png_file)) to_save = {"saveval": permute_png_idx} saveh5(to_save, dump_data_dir + 'permute_png_idx.h5') # dt.save(permute_png_idx, # dump_data_dir + 'permute_png_idx.h5') else: print(' -- ' + mode + ': ' 'Loading shuffle for reading images from ' '{}'.format(dump_data_dir)) to_load = loadh5(dump_data_dir + 'permute_png_idx.h5') permute_png_idx = to_load["saveval"] # permute_png_idx = dt.load(dump_data_dir + # 'permute_png_idx.h5') list_png_file = [ list_png_file[permute_png_idx[idx]] for idx in range(len(list_png_file)) ] # Write to file (all three) f_train = open(split_prefix + 'train.txt', 'w') f_valid = open(split_prefix + 'valid.txt', 'w') f_test = open(split_prefix + 'test.txt', 'w') train_end = int( float(param.dataset.nTrainPercent) / 100.0 * len(list_png_file)) valid_end = int( float(param.dataset.nTrainPercent + param.dataset.nValidPercent) / 100.0 * len(list_png_file)) for idx_png in six.moves.xrange(len(list_png_file)): if idx_png > valid_end: print(list_png_file[idx_png], file=f_test) elif idx_png > train_end: print(list_png_file[idx_png], file=f_valid) else: print(list_png_file[idx_png], file=f_train) f_train.close() f_valid.close() f_test.close() # Read the list list_png_file = list(np.loadtxt(split_prefix + mode + '.txt', dtype='str')) return list_png_file
def _create_pairs(self, task, use_cache=True, overwrite=False): """Generate the pairs from ID This function should return the pair indices given the task. The return type is expected to be a dictionary, where you can access by doing res["P1"], for example. """ # Use the cache file if asked if use_cache: pairs_file = os.path.join( self.pair_dir, "{}.h5".format(task)) if not os.path.exists(self.pair_dir): os.makedirs(self.pair_dir) if os.path.exists(pairs_file) and not overwrite: if self.config.use_augmented_set: # Add rotation augmentation if it does not exist (compat) _f = h5py.File(pairs_file, "r+") for name in ["P1", "P2", "P3", "P4"]: if (name + "_rot_aug") not in _f: _f.create_dataset( name + "_rot_aug", data=2 * np.pi * self.rng.rand(_f[name].size)) _f.close() return loadh5(pairs_file) # Make a lookup table for getting the indices of each sample. This can # be easily done by sorting by ID, and storing the indices to a # dictionary if self.LUT_kp[task] is None: # Create empty dictionary self.LUT_kp[task] = {} self.LUT_nonkp[task] = {} if hasattr(self, "th_dist"): self.LUT_below_th[task] = {} # Build list of indices dist_idx_reverse = np.zeros( self.data[task]["ID"].max() + 1, dtype=np.int64) dist_idx_reverse[self.data[task]["dist_idx"]] = range( 1, self.data[task]["dist_idx"].size + 1) dist_idx_reverse -= 1 # Argsort the ID print("[{}] sorting IDs...".format(task)) ID = self.data[task]["ID"] ind = np.argsort(ID) # Go through them one by one -- the easy way print("[{}] building LUT...".format(task)) start = 0 start_ID = ID[ind[start]] for end in xrange(1, len(ind)): if end % 100 == 0: print("\r --- working on {}/{}".format( end + 1, len(ind)), end="") sys.stdout.flush() # Check if indices have changed or we reached the end if start_ID != ID[ind[end]] or end == len(ind) - 1: # Store them in proper LUT if start_ID < 0: self.LUT_nonkp[task][-1] = ind[start:end] else: self.LUT_kp[task][start_ID] = ind[start:end] if hasattr(self, "th_dist"): v = dist_idx_reverse[start_ID] assert v >= 0, "Invalid index" d = self.data[task]["dist_mat"][v] # 3D points are 1-indexed self.LUT_below_th[task][start_ID] = 1 + \ np.where((d > 0) & (d < self.th_dist))[0] # Update start position start = end start_ID = ID[ind[start]] print("\r --- done. ") # The dictionary to return cur_pairs = {} for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name] = [] # For each kp item, create a random pair # # Note that this is different from what we used to do. This way seems # better as we will look at each keypoint once. print("[{}] creating pairs...".format(task)) num_kp = len(self.LUT_kp[task]) for i in xrange(num_kp): if i % 100 == 0: print("\r --- working on {}/{}".format( i + 1, num_kp), end="") sys.stdout.flush() _kp = list(self.LUT_kp[task].keys())[i] # Check if we have enough views of this point and skip if len(self.LUT_kp[task][_kp]) < 2: continue # For P1 and P2 -- Select two random patches P1, P2 = self.rng.choice( self.LUT_kp[task][_kp], 2, replace=False) # For P3 -- Select a different keypoint randomly # Generate a list of keys valid_keys = set(self.LUT_kp[task]) # Remove self valid_keys -= set([_kp]) # Remove points below the distance threshold if hasattr(self, "th_dist"): valid_keys -= set(valid_keys.intersection( self.LUT_below_th[task][_kp])) _kp2 = self.rng.choice(list(valid_keys)) P3 = self.rng.choice(self.LUT_kp[task][_kp2], 1)[0] # For P4 -- Select a random nonkp P4 = self.rng.choice(self.LUT_nonkp[task][-1], 1)[0] # Append to the list for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name].append(eval(name)) print("\r --- done. ") # Convert them to np arrays for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name] = np.asarray(cur_pairs[name]) # Augment pairs with rotation data if self.config.use_augmented_set: for name in ["P1", "P2", "P3", "P4"]: cur_pairs[name + "_rot_aug"] = 2 * np.pi * \ self.rng.rand(cur_pairs[name].size) # Save to cache if asked if use_cache: if not os.path.exists(self.pair_dir): os.makedirs(self.pair_dir) saveh5(cur_pairs, pairs_file) return cur_pairs