def batch_extract_chips(gfpath_list, cfpath_list, roi_list, theta_list, uniform_size=None, uniform_sqrt_area=None, filter_list=[], num_procs=1, lazy=True, force_gray=False): ''' cfpath_fmt - a string with a %d embedded where the cid will go. ''' try: list_size_list = map(len, (gfpath_list, cfpath_list, roi_list, theta_list)) assert all([list_size_list[0] == list_size for list_size in list_size_list]) except AssertionError as ex: print(ex) raise # Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels if uniform_sqrt_area is not None: chipsz_list = compute_uniform_area_chip_sizes(roi_list, uniform_sqrt_area) elif uniform_size is not None: chipsz_list = [uniform_size] * len(roi_list) else: chipsz_list = [(int(w), int(h)) for (x, y, w, h) in roi_list] arg_list = [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list] pcc_kwargs = { 'arg_list': arg_list, 'lazy': lazy, 'num_procs': num_procs, 'common_args': [filter_list, force_gray] } # Compute all chips with paramatarized filters parallel_compute(compute_chip, **pcc_kwargs)
def _load_features_individualy(hs, cx_list): use_cache = not hs.args.nocache_feats feat_cfg = hs.prefs.feat_cfg feat_dir = hs.dirs.feat_dir feat_uid = feat_cfg.get_uid() print('[fc2] Loading ' + feat_uid + ' individually') # Build feature paths rchip_fpath_list = [hs.cpaths.cx2_rchip_path[cx] for cx in iter(cx_list)] cid_list = hs.tables.cx2_cid[cx_list] feat_fname_fmt = ''.join(('cid%d', feat_uid, '.npz')) feat_fpath_list = [join(feat_dir, feat_fname_fmt % cid) for cid in cid_list] # Compute features in parallel, saving them to disk kwargs_list = [feat_cfg.get_dict_args()] * len(rchip_fpath_list) precompute_args = [rchip_fpath_list, feat_fpath_list, kwargs_list] pfc_kwargs = {'num_procs': hs.args.num_procs, 'lazy': use_cache} precompute_fn = feat_type2_precompute[feat_cfg.feat_type] parallel_compute(precompute_fn, precompute_args, **pfc_kwargs) # Load precomputed features sequentially kpts_list, desc_list = sequential_feat_load(feat_cfg, feat_fpath_list) return kpts_list, desc_list
def batch_extract_chips(gfpath_list, cfpath_list, roi_list, theta_list, uniform_size=None, uniform_sqrt_area=None, filter_list=[], num_procs=1, lazy=True, force_gray=False): ''' cfpath_fmt - a string with a %d embedded where the cid will go. ''' try: list_size_list = map(len, (gfpath_list, cfpath_list, roi_list, theta_list)) assert all( [list_size_list[0] == list_size for list_size in list_size_list]) except AssertionError as ex: print(ex) raise # Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels if uniform_sqrt_area is not None: chipsz_list = compute_uniform_area_chip_sizes(roi_list, uniform_sqrt_area) elif uniform_size is not None: chipsz_list = [uniform_size] * len(roi_list) else: chipsz_list = [(int(w), int(h)) for (x, y, w, h) in roi_list] arg_list = [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list] pcc_kwargs = { 'arg_list': arg_list, 'lazy': lazy, 'num_procs': num_procs, 'common_args': [filter_list, force_gray] } # Compute all chips with paramatarized filters parallel_compute(compute_chip, **pcc_kwargs)
def load_chips(hs, cx_list=None, **kwargs): print('\n=============================') print('[cc2] Precomputing chips and loading chip paths: %r' % hs.get_db_name()) print('=============================') #---------------- # COMPUTE SETUP #---------------- chip_cfg = hs.prefs.chip_cfg chip_uid = chip_cfg.get_uid() if hs.cpaths.chip_uid != '' and hs.cpaths.chip_uid != chip_uid: print('[cc2] Disagreement: OLD_chip_uid = %r' % hs.cpaths.chip_uid) print('[cc2] Disagreement: NEW_chip_uid = %r' % chip_uid) print('[cc2] Unloading all chip information') hs.unload_all() print('[cc2] chip_uid = %r' % chip_uid) # Get the list of chips paths to load cx_list = hs.get_valid_cxs() if cx_list is None else cx_list if not np.iterable(cx_list): cx_list = [cx_list] if len(cx_list) == 0: return # HACK cx_list = np.array(cx_list) # HACK hs.cpaths.chip_uid = chip_uid #print('[cc2] Requested %d chips' % (len(cx_list))) #print('[cc2] cx_list = %r' % (cx_list,)) # Get table information try: gx_list = hs.tables.cx2_gx[cx_list] cid_list = hs.tables.cx2_cid[cx_list] theta_list = hs.tables.cx2_theta[cx_list] roi_list = hs.tables.cx2_roi[cx_list] #gname_list = hs.tables.gx2_gname[gx_list] except IndexError as ex: print(repr(ex)) print(hs.tables) print('cx_list=%r' % (cx_list,)) raise # Get ChipConfig Parameters sqrt_area = chip_cfg['chip_sqrt_area'] filter_list = [] if chip_cfg['adapteq']: filter_list.append(adapteq_fn) if chip_cfg['histeq']: filter_list.append(histeq_fn) if chip_cfg['region_norm']: filter_list.append(region_norm_fn) #if chip_cfg['maxcontrast']: #filter_list.append(maxcontr_fn) #if chip_cfg['rank_eq']: #filter_list.append(rankeq_fn) #if chip_cfg['local_eq']: #filter_list.append(localeq_fn) if chip_cfg['grabcut']: filter_list.append(grabcut_fn) #--------------------------- # ___Normalized Chip Args___ #--------------------------- # Full Image Paths: where to extract the chips from gfpath_list = hs.gx2_gname(gx_list, full=True) #img_dir = hs.dirs.img_dir #gfpath_list = [join(img_dir, gname) for gname in iter(gname_list)] # Chip Paths: where to write extracted chips to _cfname_fmt = 'cid%d' + chip_uid + '.png' _cfpath_fmt = join(hs.dirs.chip_dir, _cfname_fmt) cfpath_list = [_cfpath_fmt % cid for cid in iter(cid_list)] # Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels chipsz_list = compute_uniform_area_chip_sizes(roi_list, sqrt_area) #-------------------------- # EXTRACT AND RESIZE CHIPS #-------------------------- pcc_kwargs = { 'arg_list': [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list], 'lazy': not hs.args.nocache_chips, 'num_procs': hs.args.num_procs, 'common_args': [filter_list] } # Compute all chips with paramatarized filters parallel_compute(compute_chip, **pcc_kwargs) # Read sizes try: rsize_list = [(None, None) if path is None else Image.open(path).size for path in iter(cfpath_list)] except IOError as ex: import gc gc.collect() print('[cc] ex=%r' % ex) print('path=%r' % path) if helpers.checkpath(path, verbose=True): import time time.sleep(1) # delays for 1 seconds print('[cc] file exists but cause IOError?') print('[cc] probably corrupted. Removing it') try: helpers.remove_file(path) except OSError: print('Something bad happened') raise raise #---------------------- # UPDATE API VARIABLES #---------------------- print('[cc2] Done Precomputing chips and loading chip paths') # Extend the datastructure if needed list_size = max(cx_list) + 1 #helpers.ensure_list_size(hs.cpaths.cx2_chip_path, list_size) helpers.ensure_list_size(hs.cpaths.cx2_rchip_path, list_size) helpers.ensure_list_size(hs.cpaths.cx2_rchip_size, list_size) # Copy the values into the ChipPaths object #for lx, cx in enumerate(cx_list): #hs.cpaths.cx2_chip_path[cx] = cfpath_list[lx] for lx, cx in enumerate(cx_list): hs.cpaths.cx2_rchip_path[cx] = cfpath_list[lx] for lx, cx in enumerate(cx_list): hs.cpaths.cx2_rchip_size[cx] = rsize_list[lx] #hs.load_cx2_rchip_size() # TODO: Loading rchip size should be handled more robustly print('[cc2]=============================')
def load_chips(hs, cx_list=None, **kwargs): print('\n=============================') print('[cc2] Precomputing chips and loading chip paths: %r' % hs.get_db_name()) print('=============================') #---------------- # COMPUTE SETUP #---------------- chip_cfg = hs.prefs.chip_cfg chip_uid = chip_cfg.get_uid() if hs.cpaths.chip_uid != '' and hs.cpaths.chip_uid != chip_uid: print('[cc2] Disagreement: OLD_chip_uid = %r' % hs.cpaths.chip_uid) print('[cc2] Disagreement: NEW_chip_uid = %r' % chip_uid) print('[cc2] Unloading all chip information') hs.unload_all() print('[cc2] chip_uid = %r' % chip_uid) # Get the list of chips paths to load cx_list = hs.get_valid_cxs() if cx_list is None else cx_list if not np.iterable(cx_list): cx_list = [cx_list] if len(cx_list) == 0: return # HACK cx_list = np.array(cx_list) # HACK hs.cpaths.chip_uid = chip_uid #print('[cc2] Requested %d chips' % (len(cx_list))) #print('[cc2] cx_list = %r' % (cx_list,)) # Get table information try: gx_list = hs.tables.cx2_gx[cx_list] cid_list = hs.tables.cx2_cid[cx_list] theta_list = hs.tables.cx2_theta[cx_list] roi_list = hs.tables.cx2_roi[cx_list] #gname_list = hs.tables.gx2_gname[gx_list] except IndexError as ex: print(repr(ex)) print(hs.tables) print('cx_list=%r' % (cx_list, )) raise # Get ChipConfig Parameters sqrt_area = chip_cfg['chip_sqrt_area'] filter_list = [] if chip_cfg['adapteq']: filter_list.append(adapteq_fn) if chip_cfg['histeq']: filter_list.append(histeq_fn) if chip_cfg['region_norm']: filter_list.append(region_norm_fn) #if chip_cfg['maxcontrast']: #filter_list.append(maxcontr_fn) #if chip_cfg['rank_eq']: #filter_list.append(rankeq_fn) #if chip_cfg['local_eq']: #filter_list.append(localeq_fn) if chip_cfg['grabcut']: filter_list.append(grabcut_fn) #--------------------------- # ___Normalized Chip Args___ #--------------------------- # Full Image Paths: where to extract the chips from gfpath_list = hs.gx2_gname(gx_list, full=True) #img_dir = hs.dirs.img_dir #gfpath_list = [join(img_dir, gname) for gname in iter(gname_list)] # Chip Paths: where to write extracted chips to _cfname_fmt = 'cid%d' + chip_uid + '.png' _cfpath_fmt = join(hs.dirs.chip_dir, _cfname_fmt) cfpath_list = [_cfpath_fmt % cid for cid in iter(cid_list)] # Normalized Chip Sizes: ensure chips have about sqrt_area squared pixels chipsz_list = compute_uniform_area_chip_sizes(roi_list, sqrt_area) #-------------------------- # EXTRACT AND RESIZE CHIPS #-------------------------- pcc_kwargs = { 'arg_list': [gfpath_list, cfpath_list, roi_list, theta_list, chipsz_list], 'lazy': not hs.args.nocache_chips, 'num_procs': hs.args.num_procs, 'common_args': [filter_list] } # Compute all chips with paramatarized filters parallel_compute(compute_chip, **pcc_kwargs) # Read sizes try: rsize_list = [(None, None) if path is None else Image.open(path).size for path in iter(cfpath_list)] except IOError as ex: import gc gc.collect() print('[cc] ex=%r' % ex) print('path=%r' % path) if helpers.checkpath(path, verbose=True): import time time.sleep(1) # delays for 1 seconds print('[cc] file exists but cause IOError?') print('[cc] probably corrupted. Removing it') try: helpers.remove_file(path) except OSError: print('Something bad happened') raise raise #---------------------- # UPDATE API VARIABLES #---------------------- print('[cc2] Done Precomputing chips and loading chip paths') # Extend the datastructure if needed list_size = max(cx_list) + 1 #helpers.ensure_list_size(hs.cpaths.cx2_chip_path, list_size) helpers.ensure_list_size(hs.cpaths.cx2_rchip_path, list_size) helpers.ensure_list_size(hs.cpaths.cx2_rchip_size, list_size) # Copy the values into the ChipPaths object #for lx, cx in enumerate(cx_list): #hs.cpaths.cx2_chip_path[cx] = cfpath_list[lx] for lx, cx in enumerate(cx_list): hs.cpaths.cx2_rchip_path[cx] = cfpath_list[lx] for lx, cx in enumerate(cx_list): hs.cpaths.cx2_rchip_size[cx] = rsize_list[lx] #hs.load_cx2_rchip_size() # TODO: Loading rchip size should be handled more robustly print('[cc2]=============================')