def request_background_nnindexer(qreq_, daid_list): r""" FIXME: Duplicate code Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_background_nnindexer Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> daid_list = ibs.get_valid_aids(species=ibeis.const.TEST_SPECIES.ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> # execute function >>> request_background_nnindexer(qreq_, daid_list) >>> # verify results >>> result = str(False) >>> print(result) """ global CURRENT_THREAD print('Requesting background reindex') if not can_request_background_nnindexer(): # Make sure this function doesn't run if it is already running print('REQUEST DENIED') return False print('REQUEST ACCPETED') daids_hashid = qreq_.ibs.get_annot_hashid_visual_uuid(daid_list) cfgstr = build_nnindex_cfgstr(qreq_, daid_list) cachedir = qreq_.ibs.get_flann_cachedir() # Save inverted cache uuid mappings for min_reindex_thresh = qreq_.qparams.min_reindex_thresh # Grab the keypoints names and image ids before query time? flann_params = qreq_.qparams.flann_params # Get annot descriptors to index vecs_list, fgws_list, fxs_list = get_support_data(qreq_, daid_list) # Dont hash rowids when given enough info in nnindex_cfgstr flann_params['cores'] = 2 # Only ues a few cores in the background # Build/Load the flann index uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) # set temporary attribute for when the thread finishes finishtup = (uuid_map_fpath, daids_hashid, visual_uuid_list, min_reindex_thresh) CURRENT_THREAD = ut.spawn_background_process(background_flann_func, cachedir, daid_list, vecs_list, fgws_list, fxs_list, flann_params, cfgstr) CURRENT_THREAD.finishtup = finishtup
def request_background_nnindexer(qreq_, daid_list): r""" FIXME: Duplicate code Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_background_nnindexer Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> daid_list = ibs.get_valid_aids(species=ibeis.const.TEST_SPECIES.ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> # execute function >>> request_background_nnindexer(qreq_, daid_list) >>> # verify results >>> result = str(False) >>> print(result) """ global CURRENT_THREAD print('Requesting background reindex') if not can_request_background_nnindexer(): # Make sure this function doesn't run if it is already running print('REQUEST DENIED') return False print('REQUEST ACCPETED') daids_hashid = qreq_.ibs.get_annot_hashid_visual_uuid(daid_list) cfgstr = build_nnindex_cfgstr(qreq_, daid_list) cachedir = qreq_.ibs.get_flann_cachedir() # Save inverted cache uuid mappings for min_reindex_thresh = qreq_.qparams.min_reindex_thresh # Grab the keypoints names and image ids before query time? flann_params = qreq_.qparams.flann_params # Get annot descriptors to index vecs_list, fgws_list = get_support_data(qreq_, daid_list) # Dont hash rowids when given enough info in nnindex_cfgstr flann_params['cores'] = 2 # Only ues a few cores in the background # Build/Load the flann index uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) # set temporary attribute for when the thread finishes finishtup = (uuid_map_fpath, daids_hashid, visual_uuid_list, min_reindex_thresh) CURRENT_THREAD = ut.spawn_background_process( background_flann_func, cachedir, daid_list, vecs_list, fgws_list, flann_params, cfgstr) CURRENT_THREAD.finishtup = finishtup
def request_diskcached_ibeis_nnindexer(qreq_, daid_list, nnindex_cfgstr=None, verbose=True, force_rebuild=False, memtrack=None, prog_hook=None): r""" builds new NeighborIndexer which will try to use a disk cached flann if available Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): nnindex_cfgstr (?): verbose (bool): Returns: NeighborIndexer: nnindexer CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_diskcached_ibeis_nnindexer Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> daid_list = ibs.get_valid_aids(species=ibeis.const.TEST_SPECIES.ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) >>> verbose = True >>> # execute function >>> nnindexer = request_diskcached_ibeis_nnindexer(qreq_, daid_list, nnindex_cfgstr, verbose) >>> # verify results >>> result = str(nnindexer) >>> print(result) """ if nnindex_cfgstr is None: nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) cfgstr = nnindex_cfgstr cachedir = qreq_.ibs.get_flann_cachedir() flann_params = qreq_.qparams.flann_params flann_params['checks'] = qreq_.qparams.checks #if memtrack is not None: # memtrack.report('[PRE SUPPORT]') # Get annot descriptors to index if prog_hook is not None: prog_hook.set_progress(1, 3, 'Loading support data for indexer') print('[nnindex] Loading support data for indexer') vecs_list, fgws_list, fxs_list = get_support_data(qreq_, daid_list) if memtrack is not None: memtrack.report('[AFTER GET SUPPORT DATA]') try: nnindexer = new_neighbor_index(daid_list, vecs_list, fgws_list, fxs_list, flann_params, cachedir, cfgstr=cfgstr, verbose=verbose, force_rebuild=force_rebuild, memtrack=memtrack, prog_hook=prog_hook) except Exception as ex: ut.printex(ex, True, msg_='cannot build inverted index', key_list=['ibs.get_infostr()']) raise # Record these uuids in the disk based uuid map so they can be augmented if # needed min_reindex_thresh = qreq_.qparams.min_reindex_thresh if len(daid_list) > min_reindex_thresh: uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) daids_hashid = get_data_cfgstr(qreq_.ibs, daid_list) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) UUID_MAP_CACHE.write_uuid_map_dict(uuid_map_fpath, visual_uuid_list, daids_hashid) if memtrack is not None: memtrack.report('[AFTER WRITE_UUID_MAP]') return nnindexer
def request_augmented_ibeis_nnindexer(qreq_, daid_list, verbose=True, use_memcache=True, force_rebuild=False, memtrack=None): r""" DO NOT USE. THIS FUNCTION CAN CURRENTLY CAUSE A SEGFAULT tries to give you an indexer for the requested daids using the least amount of computation possible. By loading and adding to a partially build nnindex if possible and if that fails fallbs back to request_memcache. Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): Returns: str: nnindex_cfgstr CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_augmented_ibeis_nnindexer Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ZEB_PLAIN = ibeis.const.TEST_SPECIES.ZEB_PLAIN >>> ibs = ibeis.opendb('testdb1') >>> use_memcache, max_covers, verbose = True, None, True >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:6] >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> qreq_.qparams.min_reindex_thresh = 1 >>> min_reindex_thresh = qreq_.qparams.min_reindex_thresh >>> # CLEAR CACHE for clean test >>> clear_uuid_cache(qreq_) >>> # LOAD 3 AIDS INTO CACHE >>> aid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:3] >>> # Should fallback >>> nnindexer = request_augmented_ibeis_nnindexer(qreq_, aid_list) >>> # assert the fallback >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result2 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result2, ([4, 5, 6], [[1, 2, 3]]), 'pre augment') >>> # Should augment >>> nnindexer = request_augmented_ibeis_nnindexer(qreq_, daid_list) >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result3 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result3, ([], [[1, 2, 3, 4, 5, 6]]), 'post augment') >>> # Should fallback >>> nnindexer2 = request_augmented_ibeis_nnindexer(qreq_, daid_list) >>> assert nnindexer is nnindexer2 """ global NEIGHBOR_CACHE min_reindex_thresh = qreq_.qparams.min_reindex_thresh if not force_rebuild: new_daid_list, covered_aids_list = group_daids_by_cached_nnindexer( qreq_, daid_list, min_reindex_thresh, max_covers=1) can_augment = (len(covered_aids_list) > 0 and not ut.list_set_equal(covered_aids_list[0], daid_list)) else: can_augment = False if verbose: print('[aug] Requesting augmented nnindexer') if can_augment: covered_aids = covered_aids_list[0] if verbose: print('[aug] Augmenting index %r old daids with %d new daids' % (len(covered_aids), len(new_daid_list))) # Load the base covered indexer # THIS SHOULD LOAD NOT REBUILD IF THE UUIDS ARE COVERED base_nnindexer = request_memcached_ibeis_nnindexer( qreq_, covered_aids, verbose=verbose, use_memcache=use_memcache) # Remove this indexer from the memcache because we are going to change it if NEIGHBOR_CACHE.has_key(base_nnindexer.cfgstr): # NOQA print('Removing key from memcache') NEIGHBOR_CACHE[base_nnindexer.cfgstr] = None del NEIGHBOR_CACHE[base_nnindexer.cfgstr] support_data = get_support_data(qreq_, new_daid_list) (new_vecs_list, new_fgws_list, new_fxs_list) = support_data base_nnindexer.add_support(new_daid_list, new_vecs_list, new_fgws_list, new_fxs_list, verbose=True) # FIXME: pointer issues nnindexer = base_nnindexer # Change to the new cfgstr nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) nnindexer.cfgstr = nnindex_cfgstr cachedir = qreq_.ibs.get_flann_cachedir() nnindexer.save(cachedir) # Write to inverse uuid if len(daid_list) > min_reindex_thresh: uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) daids_hashid = get_data_cfgstr(qreq_.ibs, daid_list) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) UUID_MAP_CACHE.write_uuid_map_dict(uuid_map_fpath, visual_uuid_list, daids_hashid) # Write to memcache if ut.VERBOSE: print('[aug] Wrote to memcache=%r' % (nnindex_cfgstr, )) NEIGHBOR_CACHE[nnindex_cfgstr] = nnindexer return nnindexer else: #if ut.VERBOSE: if verbose: print('[aug] Nothing to augment, fallback to memcache') # Fallback nnindexer = request_memcached_ibeis_nnindexer( qreq_, daid_list, verbose=verbose, use_memcache=use_memcache, force_rebuild=force_rebuild, memtrack=memtrack) return nnindexer
def request_diskcached_ibeis_nnindexer(qreq_, daid_list, nnindex_cfgstr=None, verbose=True, force_rebuild=False, memtrack=None): r""" builds new NeighborIndexer which will try to use a disk cached flann if available Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): nnindex_cfgstr (?): verbose (bool): Returns: NeighborIndexer: nnindexer CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_diskcached_ibeis_nnindexer Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> daid_list = ibs.get_valid_aids(species=ibeis.const.TEST_SPECIES.ZEB_PLAIN) >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) >>> verbose = True >>> # execute function >>> nnindexer = request_diskcached_ibeis_nnindexer(qreq_, daid_list, nnindex_cfgstr, verbose) >>> # verify results >>> result = str(nnindexer) >>> print(result) """ if nnindex_cfgstr is None: nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) cfgstr = nnindex_cfgstr cachedir = qreq_.ibs.get_flann_cachedir() flann_params = qreq_.qparams.flann_params flann_params['checks'] = qreq_.qparams.checks #if memtrack is not None: # memtrack.report('[PRE SUPPORT]') # Get annot descriptors to index print('[nnindex] Loading support data to build diskcached indexer') vecs_list, fgws_list = get_support_data(qreq_, daid_list) if memtrack is not None: memtrack.report('[AFTER GET SUPPORT DATA]') try: nnindexer = new_neighbor_index( daid_list, vecs_list, fgws_list, flann_params, cachedir, cfgstr=cfgstr, verbose=verbose, force_rebuild=force_rebuild, memtrack=memtrack) except Exception as ex: ut.printex(ex, True, msg_='cannot build inverted index', key_list=['ibs.get_infostr()']) raise # Record these uuids in the disk based uuid map so they can be augmented if # needed min_reindex_thresh = qreq_.qparams.min_reindex_thresh if len(daid_list) > min_reindex_thresh: uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) daids_hashid = get_data_cfgstr(qreq_.ibs, daid_list) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) UUID_MAP_CACHE.write_uuid_map_dict(uuid_map_fpath, visual_uuid_list, daids_hashid) if memtrack is not None: memtrack.report('[AFTER WRITE_UUID_MAP]') return nnindexer
def request_augmented_ibeis_nnindexer(qreq_, daid_list, verbose=True, use_memcache=True, force_rebuild=False, memtrack=None): r""" DO NOT USE. THIS FUNCTION CAN CURRENTLY CAUSE A SEGFAULT tries to give you an indexer for the requested daids using the least amount of computation possible. By loading and adding to a partially build nnindex if possible and if that fails fallbs back to request_memcache. Args: qreq_ (QueryRequest): query request object with hyper-parameters daid_list (list): Returns: str: nnindex_cfgstr CommandLine: python -m ibeis.algo.hots.neighbor_index_cache --test-request_augmented_ibeis_nnindexer Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.neighbor_index_cache import * # NOQA >>> import ibeis >>> # build test data >>> ZEB_PLAIN = ibeis.const.TEST_SPECIES.ZEB_PLAIN >>> ibs = ibeis.opendb('testdb1') >>> use_memcache, max_covers, verbose = True, None, True >>> daid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:6] >>> qreq_ = ibs.new_query_request(daid_list, daid_list) >>> qreq_.qparams.min_reindex_thresh = 1 >>> min_reindex_thresh = qreq_.qparams.min_reindex_thresh >>> # CLEAR CACHE for clean test >>> clear_uuid_cache(qreq_) >>> # LOAD 3 AIDS INTO CACHE >>> aid_list = ibs.get_valid_aids(species=ZEB_PLAIN)[0:3] >>> # Should fallback >>> nnindexer = request_augmented_ibeis_nnindexer(qreq_, aid_list) >>> # assert the fallback >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result2 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result2, ([4, 5, 6], [[1, 2, 3]]), 'pre augment') >>> # Should augment >>> nnindexer = request_augmented_ibeis_nnindexer(qreq_, daid_list) >>> uncovered_aids, covered_aids_list = group_daids_by_cached_nnindexer( ... qreq_, daid_list, min_reindex_thresh, max_covers) >>> result3 = uncovered_aids, covered_aids_list >>> ut.assert_eq(result3, ([], [[1, 2, 3, 4, 5, 6]]), 'post augment') >>> # Should fallback >>> nnindexer2 = request_augmented_ibeis_nnindexer(qreq_, daid_list) >>> assert nnindexer is nnindexer2 """ global NEIGHBOR_CACHE min_reindex_thresh = qreq_.qparams.min_reindex_thresh if not force_rebuild: new_daid_list, covered_aids_list = group_daids_by_cached_nnindexer( qreq_, daid_list, min_reindex_thresh, max_covers=1) can_augment = ( len(covered_aids_list) > 0 and not ut.list_set_equal(covered_aids_list[0], daid_list)) else: can_augment = False if verbose: print('[aug] Requesting augmented nnindexer') if can_augment: covered_aids = covered_aids_list[0] if verbose: print('[aug] Augmenting index %r old daids with %d new daids' % (len(covered_aids), len(new_daid_list))) # Load the base covered indexer # THIS SHOULD LOAD NOT REBUILD IF THE UUIDS ARE COVERED base_nnindexer = request_memcached_ibeis_nnindexer( qreq_, covered_aids, verbose=verbose, use_memcache=use_memcache) # Remove this indexer from the memcache because we are going to change it if NEIGHBOR_CACHE.has_key(base_nnindexer.cfgstr): # NOQA print('Removing key from memcache') NEIGHBOR_CACHE[base_nnindexer.cfgstr] = None del NEIGHBOR_CACHE[base_nnindexer.cfgstr] new_vecs_list, new_fgws_list = get_support_data(qreq_, new_daid_list) base_nnindexer.add_support(new_daid_list, new_vecs_list, new_fgws_list, verbose=True) # FIXME: pointer issues nnindexer = base_nnindexer # Change to the new cfgstr nnindex_cfgstr = build_nnindex_cfgstr(qreq_, daid_list) nnindexer.cfgstr = nnindex_cfgstr cachedir = qreq_.ibs.get_flann_cachedir() nnindexer.save(cachedir) # Write to inverse uuid if len(daid_list) > min_reindex_thresh: uuid_map_fpath = get_nnindexer_uuid_map_fpath(qreq_) daids_hashid = get_data_cfgstr(qreq_.ibs, daid_list) visual_uuid_list = qreq_.ibs.get_annot_visual_uuids(daid_list) UUID_MAP_CACHE.write_uuid_map_dict(uuid_map_fpath, visual_uuid_list, daids_hashid) # Write to memcache if ut.VERBOSE: print('[aug] Wrote to memcache=%r' % (nnindex_cfgstr,)) NEIGHBOR_CACHE[nnindex_cfgstr] = nnindexer return nnindexer else: #if ut.VERBOSE: if verbose: print('[aug] Nothing to augment, fallback to memcache') # Fallback nnindexer = request_memcached_ibeis_nnindexer( qreq_, daid_list, verbose=verbose, use_memcache=use_memcache, force_rebuild=force_rebuild, memtrack=memtrack ) return nnindexer