Пример #1
0
def load_query_results(hs, qcx_list, force_load=False):
    query_cfg = hs.prefs.query_cfg
    # Build query big cache uid
    query_uid = query_cfg.get_uid()
    hs_uid = hs.get_db_name()
    qcxs_uid = helpers.hashstr(tuple(qcx_list))
    qres_uid = hs_uid + query_uid + qcxs_uid
    cache_dir = join(hs.dirs.cache_dir, 'query_results_bigcache')
    print('\n===============')
    print('\n[rr2] Load Query Results')
    print('[rr2] load_query_results(): %r' % qres_uid)
    io_kwargs = dict(dpath=cache_dir,
                     fname='query_results',
                     uid=qres_uid,
                     ext='.cPkl')
    # Return cache if available
    if not hs.args.nocache_query and (not force_load):
        qcx2_res = io.smart_load(**io_kwargs)
        if qcx2_res is not None:
            print('[rr2] load_query_results(): cache hit')
            return qcx2_res
        print('[rr2] load_query_results(): cache miss')
    else:
        print('[rr2] load_query_results(): cache off')
    # Individually load / compute queries
    qcx2_res = [hs.query(qcx) for qcx in qcx_list]
    # Save to the cache
    print('[rr2] Saving query_results to bigcache: %r' % qres_uid)
    helpers.ensuredir(cache_dir)
    io.smart_save(qcx2_res, **io_kwargs)
    return qcx2_res
Пример #2
0
def load_query_results(hs, qcx_list, force_load=False):
    query_cfg = hs.prefs.query_cfg
    # Build query big cache uid
    query_uid = query_cfg.get_uid()
    hs_uid    = hs.get_db_name()
    qcxs_uid  = helpers.hashstr(tuple(qcx_list))
    qres_uid  = hs_uid + query_uid + qcxs_uid
    cache_dir = join(hs.dirs.cache_dir, 'query_results_bigcache')
    print('\n===============')
    print('\n[rr2] Load Query Results')
    print('[rr2] load_query_results(): %r' % qres_uid)
    io_kwargs = dict(dpath=cache_dir, fname='query_results', uid=qres_uid, ext='.cPkl')
    # Return cache if available
    if not hs.args.nocache_query and (not force_load):
        qcx2_res = io.smart_load(**io_kwargs)
        if qcx2_res is not None:
            print('[rr2] load_query_results(): cache hit')
            return qcx2_res
        print('[rr2] load_query_results(): cache miss')
    else:
        print('[rr2] load_query_results(): cache off')
    # Individually load / compute queries
    qcx2_res = [hs.query(qcx) for qcx in qcx_list]
    # Save to the cache
    print('[rr2] Saving query_results to bigcache: %r' % qres_uid)
    helpers.ensuredir(cache_dir)
    io.smart_save(qcx2_res, **io_kwargs)
    return qcx2_res
Пример #3
0
def cache_test_results(qx2_bestranks, hs, qreq, qcxs, dcxs):
    test_uid = qreq.get_query_uid(hs, qcxs)
    cache_dir = join(hs.dirs.cache_dir, 'experiment_harness_results')
    util.ensuredir(cache_dir)
    io_kwargs = {
        'dpath': cache_dir,
        'fname': 'test_results',
        'uid': test_uid,
        'ext': '.cPkl'
    }
    io.smart_save(qx2_bestranks, **io_kwargs)
Пример #4
0
def precompute_akmeans(data,
                       num_clusters,
                       max_iters=100,
                       flann_params=None,
                       cache_dir=None,
                       force_recomp=False,
                       same_data=True,
                       uid=''):
    'precompute aproximate kmeans'
    if flann_params is None:
        flann_params = {}
    print('[algos] pre_akmeans()')
    if same_data:
        data_uid = helpers.hashstr_arr(data, 'dID')
        uid += data_uid
    clusters_fname = 'akmeans_clusters'
    datax2cl_fname = 'akmeans_datax2cl'
    try:
        if not force_recomp:
            clusters = io.smart_load(cache_dir,
                                     clusters_fname,
                                     uid,
                                     '.npy',
                                     can_fail=False)
            datax2_clusterx = io.smart_load(cache_dir,
                                            datax2cl_fname,
                                            uid,
                                            '.npy',
                                            can_fail=False)
        else:
            raise Exception('forcing')
        # Hack to refine akmeans with a few more iterations
        if '--refine' in sys.argv or '--refine-exit' in sys.argv:
            max_iters_override = helpers.get_arg('--refine', type_=int)
            print('Overriding max_iters=%r' % max_iters_override)
            if not max_iters_override is None:
                max_iters = max_iters_override
            datax2_clusterx_old = datax2_clusterx
            print('[algos] refining:')
            print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
            print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
            (datax2_clusterx,
             clusters) = __akmeans_iterate(data, clusters, datax2_clusterx_old,
                                           max_iters, flann_params, 0, 10)
            io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
            io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
                          '.npy')
            if '--refine-exit' in sys.argv:
                print('exiting after refine')
                sys.exit(1)
        print('[algos] pre_akmeans(): ... loaded akmeans.')
    except Exception as ex:
        print('[algos] pre_akmeans(): ... could not load akmeans.')
        errstr = helpers.indent(repr(ex), '[algos]    ')
        print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
        print('[algos] pre_akmeans(): printing debug_smart_load')
        print('---- <DEBUG SMART LOAD>---')
        io.debug_smart_load(cache_dir, clusters_fname)
        io.debug_smart_load(cache_dir, datax2cl_fname)
        print('----</DEBUG SMART LOAD>---')
        #print('[algos] Press Ctrl+C to stop k-means early (and save)')
        #signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
        print('[algos] computing:')
        print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
        print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
        print('[algos] pre_akmeans(): calling akmeans')
        (datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
                                              flann_params)
        print('[algos] pre_akmeans(): finished running akmeans')
        io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
        io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
        #print('[algos] Removing Ctrl+C signal handler')
        #signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
    print('[algos] pre_akmeans(): return')
    return (datax2_clusterx, clusters)
Пример #5
0
def get_test_results(hs, qcx_list, qdat, cfgx=0, nCfg=1,
                     force_load=False):
    dcxs = hs.get_indexed_sample()
    query_uid = qdat.get_uid()
    print('[harn] get_test_results(): %r' % query_uid)
    hs_uid    = hs.get_db_name()
    qcxs_uid  = helpers.hashstr_arr(qcx_list)
    test_uid  = hs_uid + query_uid + qcxs_uid
    cache_dir = join(hs.dirs.cache_dir, 'experiment_harness_results')
    io_kwargs = dict(dpath=cache_dir, fname='test_results', uid=test_uid, ext='.cPkl')
    # High level caching
    qx2_bestranks = []
    #nChips = hs.get_num_chip()
    #nNames = len(hs.tables.nx2_name) - 2
    nQuery = len(qcx_list)
    #NMultiNames =
    nPrevQ = nQuery * cfgx
    qx2_reslist = []
    if not hs.args.nocache_query and (not force_load):
        test_results = io.smart_load(**io_kwargs)
        if test_results is None:
            pass
        elif len(test_results) != 1:
            print('recaching test_results')
        elif not test_results is None:
            return test_results, [[{0: None}]] * nQuery
    for qx, qcx in enumerate(qcx_list):
        print(textwrap.dedent('''
        [harn]----------------
        [harn] TEST %d/%d
        [harn]----------------''' % (qx + nPrevQ + 1, nQuery * nCfg)))
        gt_cxs = hs.get_other_indexed_cxs(qcx)
        #title = 'q' + hs.cidstr(qcx) + ' - ' + notes
        #print('[harn] title=%r' % (title,))
        #print('[harn] gt_' + hs.cidstr(gt_cxs))
        res_list = mc3.execute_query_safe(hs, qdat, [qcx], dcxs)
        bestranks = []
        algos = []
        qx2_reslist += [res_list]
        assert len(res_list) == 1
        for qcx2_res in res_list:
            assert len(qcx2_res) == 1
            res = qcx2_res[qcx]
            algos += [res.title]
            gt_ranks = res.get_gt_ranks(gt_cxs)
            #print('[harn] cx_ranks(/%4r) = %r' % (nChips, gt_ranks))
            #print('[harn] cx_ranks(/%4r) = %r' % (NMultiNames, gt_ranks))
            #print('ns_ranks(/%4r) = %r' % (nNames, gt_ranks))
            if len(gt_ranks) == 0:
                _bestrank = -1
            else:
                _bestrank = min(gt_ranks)
            bestranks += [_bestrank]
        # record metadata
        qx2_bestranks += [bestranks]
    mat_vals = np.array(qx2_bestranks)
    test_results = (mat_vals,)
    # High level caching
    helpers.ensuredir(cache_dir)
    io.smart_save(test_results, **io_kwargs)
    return test_results, qx2_reslist
Пример #6
0
def precompute_akmeans(data, num_clusters, max_iters=100,
                       flann_params=None,  cache_dir=None,
                       force_recomp=False, same_data=True, uid=''):
    'precompute aproximate kmeans'
    if flann_params is None:
        flann_params = {}
    print('[algos] pre_akmeans()')
    if same_data:
        data_uid = helpers.hashstr_arr(data, 'dID')
        uid += data_uid
    clusters_fname = 'akmeans_clusters'
    datax2cl_fname = 'akmeans_datax2cl'
    try:
        if not force_recomp:
            clusters = io.smart_load(cache_dir, clusters_fname, uid, '.npy',
                                     can_fail=False)
            datax2_clusterx = io.smart_load(cache_dir, datax2cl_fname, uid,
                                            '.npy', can_fail=False)
        else:
            raise Exception('forcing')
        # Hack to refine akmeans with a few more iterations
        if '--refine' in sys.argv or '--refine-exit' in sys.argv:
            max_iters_override = helpers.get_arg('--refine', type_=int)
            print('Overriding max_iters=%r' % max_iters_override)
            if not max_iters_override is None:
                max_iters = max_iters_override
            datax2_clusterx_old = datax2_clusterx
            print('[algos] refining:')
            print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
            print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
            (datax2_clusterx, clusters) = __akmeans_iterate(
                data, clusters, datax2_clusterx_old, max_iters, flann_params,
                0, 10)
            io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
            io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
                          '.npy')
            if '--refine-exit' in sys.argv:
                print('exiting after refine')
                sys.exit(1)
        print('[algos] pre_akmeans(): ... loaded akmeans.')
    except Exception as ex:
        print('[algos] pre_akmeans(): ... could not load akmeans.')
        errstr = helpers.indent(repr(ex), '[algos]    ')
        print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
        print('[algos] pre_akmeans(): printing debug_smart_load')
        print('---- <DEBUG SMART LOAD>---')
        io.debug_smart_load(cache_dir, clusters_fname)
        io.debug_smart_load(cache_dir, datax2cl_fname)
        print('----</DEBUG SMART LOAD>---')
        #print('[algos] Press Ctrl+C to stop k-means early (and save)')
        #signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
        print('[algos] computing:')
        print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
        print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
        print('[algos] pre_akmeans(): calling akmeans')
        (datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
                                              flann_params)
        print('[algos] pre_akmeans(): finished running akmeans')
        io.smart_save(clusters,        cache_dir, clusters_fname, uid, '.npy')
        io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
        #print('[algos] Removing Ctrl+C signal handler')
        #signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
    print('[algos] pre_akmeans(): return')
    return (datax2_clusterx, clusters)
Пример #7
0
def bigcache_feat_save(cache_dir, uid, ext, kpts_list, desc_list):
    print('[fc2] Caching desc_list and kpts_list')
    io.smart_save(kpts_list, cache_dir, 'kpts_list', uid, ext)
    io.smart_save(desc_list, cache_dir, 'desc_list', uid, ext)