Beispiel #1
0
    def add_ibeis_support(mxer, qreq_, new_aid_list):
        """
        Chooses indexer with smallest number of annotations and reindexes it.

        Args:
            qreq_ (QueryRequest):  query request object with hyper-parameters
            new_aid_list (list):

        CommandLine:
            python -m ibeis.algo.hots.multi_index --test-add_ibeis_support

        Example:
            >>> # DISABLE_DOCTEST
            >>> from ibeis.algo.hots.multi_index import *  # NOQA
            >>> mxer, qreq_, ibs = testdata_mindexer()
            >>> new_aid_list = ibs.get_valid_aids()[70:80]
            >>> # execute function
            >>> result = mxer.add_ibeis_support(qreq_, new_aid_list)
            >>> # verify results
            >>> print(result)
        """
        print('adding multi-indexer support')
        # Assert that the aids are indeed new
        mxer.assert_can_add_aids(new_aid_list)
        # Find the indexer to add to
        num_indexed_list = mxer.get_multi_num_indexed_annots()
        min_argx = num_indexed_list.argmin()
        nnindexer_old = mxer.nn_indexer_list[min_argx]
        # Combine old and new aids
        prev_aids = nnindexer_old.get_indexed_aids()
        new_aid_list_ = np.append(prev_aids, new_aid_list)
        # Reindexed combined aids
        nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer(
            qreq_, new_aid_list_)
        # Replace the old nnindexer with the new nnindexer
        mxer.nn_indexer_list[min_argx] = nnindexer_new
        mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh

        if neighbor_index_cache.can_request_background_nnindexer():
            # Check if background process needs to be spawned
            # FIXME: this does not belong in method code
            num_indexed_list_new = mxer.get_multi_num_indexed_annots()
            new_smalled_size = min(num_indexed_list_new)
            need_reindex = (new_smalled_size > mxer.min_reindex_thresh or
                            len(num_indexed_list_new) > mxer.max_subindexers)
            if need_reindex:
                if USE_FORGROUND_REINDEX:
                    raise NotImplementedError(
                        'no foreground reindex in stateful query')
                else:
                    # Reindex the multi-indexed trees in the background
                    aid_list = mxer.get_indexed_aids()
                    neighbor_index_cache.request_background_nnindexer(
                        qreq_, aid_list)
Beispiel #2
0
    def add_ibeis_support(mxer, qreq_, new_aid_list):
        """
        Chooses indexer with smallest number of annotations and reindexes it.

        Args:
            qreq_ (QueryRequest):  query request object with hyper-parameters
            new_aid_list (list):

        CommandLine:
            python -m ibeis.algo.hots.multi_index --test-add_ibeis_support

        Example:
            >>> # DISABLE_DOCTEST
            >>> from ibeis.algo.hots.multi_index import *  # NOQA
            >>> mxer, qreq_, ibs = testdata_mindexer()
            >>> new_aid_list = ibs.get_valid_aids()[70:80]
            >>> # execute function
            >>> result = mxer.add_ibeis_support(qreq_, new_aid_list)
            >>> # verify results
            >>> print(result)
        """
        print('adding multi-indexer support')
        # Assert that the aids are indeed new
        mxer.assert_can_add_aids(new_aid_list)
        # Find the indexer to add to
        num_indexed_list = mxer.get_multi_num_indexed_annots()
        min_argx = num_indexed_list.argmin()
        nnindexer_old = mxer.nn_indexer_list[min_argx]
        # Combine old and new aids
        prev_aids = nnindexer_old.get_indexed_aids()
        new_aid_list_ = np.append(prev_aids, new_aid_list)
        # Reindexed combined aids
        nnindexer_new = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, new_aid_list_)
        # Replace the old nnindexer with the new nnindexer
        mxer.nn_indexer_list[min_argx] = nnindexer_new
        mxer.min_reindex_thresh = qreq_.qparams.min_reindex_thresh

        if neighbor_index_cache.can_request_background_nnindexer():
            # Check if background process needs to be spawned
            # FIXME: this does not belong in method code
            num_indexed_list_new = mxer.get_multi_num_indexed_annots()
            new_smalled_size = min(num_indexed_list_new)
            need_reindex = (new_smalled_size > mxer.min_reindex_thresh or
                            len(num_indexed_list_new) > mxer.max_subindexers)
            if need_reindex:
                if USE_FORGROUND_REINDEX:
                    raise NotImplementedError('no foreground reindex in stateful query')
                else:
                    # Reindex the multi-indexed trees in the background
                    aid_list = mxer.get_indexed_aids()
                    neighbor_index_cache.request_background_nnindexer(qreq_, aid_list)
Beispiel #3
0
def request_ibeis_mindexer(qreq_, index_method='multi', verbose=True):
    """

    CommandLine:
        python -m ibeis.algo.hots.multi_index --test-request_ibeis_mindexer:2

    Example0:
        >>> # SLOW_DOCTEST
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(db='PZ_MTEST')
        >>> valid_aids = ibs.get_valid_aids()
        >>> daid_list = valid_aids[1:60]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)

    Example1:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(db='PZ_Master0')
        >>> valid_aids = ibs.get_valid_aids()
        >>> daid_list = valid_aids[1:60]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)

    Example2:
        >>> # DISABLE_DOCTEST
        >>> # Test background reindex
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> import time
        >>> ibs = ibeis.opendb(db='PZ_MTEST')
        >>> valid_aids = ibs.get_valid_aids()
        >>> # Remove all cached nnindexers
        >>> ibs.delete_flann_cachedir()
        >>> # This request should build a new nnindexer
        >>> daid_list = valid_aids[1:30]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one subindexer')
        >>> # The next request should trigger a background process
        >>> # and build two subindexer
        >>> daid_list = valid_aids[1:60]
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> # Do some work in the foreground to ensure that it doesnt block
        >>> # the background job
        >>> print('[FG] sleeping or doing bit compute')
        >>> # Takes about 15 seconds
        >>> with ut.Timer():
        ...     ut.enumerate_primes(int(9E4))
        >>> #time.sleep(10)
        >>> print('[FG] done sleeping')
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 2, 'two subindexer')
        >>> # And this shoud build just one subindexer
        >>> daid_list = valid_aids[1:60]
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one big subindexer')

    """
    min_reindex_thresh = qreq_.qparams.min_reindex_thresh
    max_subindexers = qreq_.qparams.max_subindexers

    daid_list = qreq_.get_internal_daids()
    print('[mindex] make MultiNeighborIndex over %d annots' % (len(daid_list),))
    print('[mindex] index_method=%r' % index_method)

    # Split annotations into groups accorindg to index_method
    ibs = qreq_.ibs
    if index_method == 'name':
        # each group are annotations of the same name
        num_indexers = 8
        aids_list, overflow_aids, num_bins = group_daids_for_indexing_by_name(ibs, daid_list, num_indexers, verbose)
    elif index_method == 'multi':
        neighbor_index_cache.check_background_process()
        # Use greedy set cover to get a list of nnindxers that are already built
        tup = neighbor_index_cache.group_daids_by_cached_nnindexer(
            qreq_, daid_list, min_reindex_thresh)
        uncovered_aids, covered_aids_list = tup
        # If the number of bins gets too big do a reindex
        # in the background
        num_subindexers = len(covered_aids_list) + (len(uncovered_aids) > 1)
        if num_subindexers > max_subindexers:
            print('need to reindex something')
            if USE_FORGROUND_REINDEX:
                aids_list = [sorted(ut.flatten(covered_aids_list))]
                #ut.embed()
            else:
                neighbor_index_cache.request_background_nnindexer(qreq_, daid_list)
                aids_list = covered_aids_list
        else:
            aids_list = covered_aids_list
        if len(uncovered_aids) > 0:
            aids_list.append(uncovered_aids)
        num_bins = len(aids_list)
    else:
        raise AssertionError('unknown index_method=%r' % (index_method,))

    # Build a neighbor indexer for each
    nn_indexer_list = []
    #extra_indexes = []
    for tx, aids in enumerate(aids_list):
        print('[mindex] building forest %d/%d with %d aids' %
                (tx + 1, num_bins, len(aids)))
        if len(aids) > 0:
            # Dont bother shallow copying qreq_ here.
            # just passing aids is enough
            nnindexer = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, aids)
            nn_indexer_list.append(nnindexer)
    #if len(unknown_aids) > 0:
    #    print('[mindex] building unknown forest')
    #    unknown_vecs_list = ibs.get_annot_vecs(overflow_aids, config2_=qreq_.get_internal_data_config2())
    #    unknown_index = NeighborIndex(overflow_aids, unknown_vecs_list)
    #    extra_indexes.append(unknown_index)
    ##print('[mindex] building normalizer forest')  # TODO
    #mxer.nn_indexer_list = nn_indexer_list
    #mxer.extra_indexes = extra_indexes
    #mxer.overflow_index = overflow_index
    #mxer.unknown_index = unknown_index
    mxer = MultiNeighborIndex(nn_indexer_list, min_reindex_thresh, max_subindexers)
    return mxer
Beispiel #4
0
def request_ibeis_mindexer(qreq_, index_method='multi', verbose=True):
    """

    CommandLine:
        python -m ibeis.algo.hots.multi_index --test-request_ibeis_mindexer:2

    Example0:
        >>> # SLOW_DOCTEST
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(db='PZ_MTEST')
        >>> valid_aids = ibs.get_valid_aids()
        >>> daid_list = valid_aids[1:60]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)

    Example1:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(db='PZ_Master0')
        >>> valid_aids = ibs.get_valid_aids()
        >>> daid_list = valid_aids[1:60]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)

    Example2:
        >>> # DISABLE_DOCTEST
        >>> # Test background reindex
        >>> from ibeis.algo.hots.multi_index import *  # NOQA
        >>> import ibeis
        >>> import time
        >>> ibs = ibeis.opendb(db='PZ_MTEST')
        >>> valid_aids = ibs.get_valid_aids()
        >>> # Remove all cached nnindexers
        >>> ibs.delete_flann_cachedir()
        >>> # This request should build a new nnindexer
        >>> daid_list = valid_aids[1:30]
        >>> cfgdict = dict(fg_on=False)
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one subindexer')
        >>> # The next request should trigger a background process
        >>> # and build two subindexer
        >>> daid_list = valid_aids[1:60]
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> # Do some work in the foreground to ensure that it doesnt block
        >>> # the background job
        >>> print('[FG] sleeping or doing bit compute')
        >>> # Takes about 15 seconds
        >>> with ut.Timer():
        ...     ut.enumerate_primes(int(9E4))
        >>> #time.sleep(10)
        >>> print('[FG] done sleeping')
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 2, 'two subindexer')
        >>> # And this shoud build just one subindexer
        >>> daid_list = valid_aids[1:60]
        >>> qreq_ = ibs.new_query_request(daid_list, daid_list, cfgdict=cfgdict)
        >>> index_method = 'multi'
        >>> mxer = request_ibeis_mindexer(qreq_, index_method)
        >>> ut.assert_eq(len(mxer.nn_indexer_list), 1, 'one big subindexer')

    """
    min_reindex_thresh = qreq_.qparams.min_reindex_thresh
    max_subindexers = qreq_.qparams.max_subindexers

    daid_list = qreq_.get_internal_daids()
    print('[mindex] make MultiNeighborIndex over %d annots' % (len(daid_list),))
    print('[mindex] index_method=%r' % index_method)

    # Split annotations into groups accorindg to index_method
    ibs = qreq_.ibs
    if index_method == 'name':
        # each group are annotations of the same name
        num_indexers = 8
        aids_list, overflow_aids, num_bins = group_daids_for_indexing_by_name(ibs, daid_list, num_indexers, verbose)
    elif index_method == 'multi':
        neighbor_index_cache.check_background_process()
        # Use greedy set cover to get a list of nnindxers that are already built
        tup = neighbor_index_cache.group_daids_by_cached_nnindexer(
            qreq_, daid_list, min_reindex_thresh)
        uncovered_aids, covered_aids_list = tup
        # If the number of bins gets too big do a reindex
        # in the background
        num_subindexers = len(covered_aids_list) + (len(uncovered_aids) > 1)
        if num_subindexers > max_subindexers:
            print('need to reindex something')
            if USE_FORGROUND_REINDEX:
                aids_list = [sorted(ut.flatten(covered_aids_list))]
                #ut.embed()
            else:
                neighbor_index_cache.request_background_nnindexer(qreq_, daid_list)
                aids_list = covered_aids_list
        else:
            aids_list = covered_aids_list
        if len(uncovered_aids) > 0:
            aids_list.append(uncovered_aids)
        num_bins = len(aids_list)
    else:
        raise AssertionError('unknown index_method=%r' % (index_method,))

    # Build a neighbor indexer for each
    nn_indexer_list = []
    #extra_indexes = []
    for tx, aids in enumerate(aids_list):
        print('[mindex] building forest %d/%d with %d aids' %
                (tx + 1, num_bins, len(aids)))
        if len(aids) > 0:
            # Dont bother shallow copying qreq_ here.
            # just passing aids is enough
            nnindexer = neighbor_index_cache.request_memcached_ibeis_nnindexer(qreq_, aids)
            nn_indexer_list.append(nnindexer)
    #if len(unknown_aids) > 0:
    #    print('[mindex] building unknown forest')
    #    unknown_vecs_list = ibs.get_annot_vecs(overflow_aids, config2_=qreq_.get_internal_data_config2())
    #    unknown_index = NeighborIndex(overflow_aids, unknown_vecs_list)
    #    extra_indexes.append(unknown_index)
    ##print('[mindex] building normalizer forest')  # TODO
    #mxer.nn_indexer_list = nn_indexer_list
    #mxer.extra_indexes = extra_indexes
    #mxer.overflow_index = overflow_index
    #mxer.unknown_index = unknown_index
    mxer = MultiNeighborIndex(nn_indexer_list, min_reindex_thresh, max_subindexers)
    return mxer