def bigcache_feat_load(cache_dir, uid, ext):
#io.debug_smart_load(cache_dir, fname='*', uid=uid, ext='.*')
kpts_list = io.smart_load(cache_dir, 'kpts_list', uid, ext, can_fail=True)
desc_list = io.smart_load(cache_dir, 'desc_list', uid, ext, can_fail=True)
if desc_list is None or kpts_list is None:
return None
desc_list = desc_list.tolist()
kpts_list = kpts_list.tolist()
print('[fc2] Loaded kpts_list and desc_list from big cache')
return kpts_list, desc_list
def load_query_results(hs, qcx_list, force_load=False):
query_cfg = hs.prefs.query_cfg
# Build query big cache uid
query_uid = query_cfg.get_uid()
hs_uid = hs.get_db_name()
qcxs_uid = helpers.hashstr(tuple(qcx_list))
qres_uid = hs_uid + query_uid + qcxs_uid
cache_dir = join(hs.dirs.cache_dir, 'query_results_bigcache')
print('\n===============')
print('\n[rr2] Load Query Results')
print('[rr2] load_query_results(): %r' % qres_uid)
io_kwargs = dict(dpath=cache_dir, fname='query_results', uid=qres_uid, ext='.cPkl')
# Return cache if available
if not hs.args.nocache_query and (not force_load):
qcx2_res = io.smart_load(**io_kwargs)
if qcx2_res is not None:
print('[rr2] load_query_results(): cache hit')
return qcx2_res
print('[rr2] load_query_results(): cache miss')
else:
print('[rr2] load_query_results(): cache off')
# Individually load / compute queries
qcx2_res = [hs.query(qcx) for qcx in qcx_list]
# Save to the cache
print('[rr2] Saving query_results to bigcache: %r' % qres_uid)
helpers.ensuredir(cache_dir)
io.smart_save(qcx2_res, **io_kwargs)
return qcx2_res
def load_query_results(hs, qcx_list, force_load=False):
query_cfg = hs.prefs.query_cfg
# Build query big cache uid
query_uid = query_cfg.get_uid()
hs_uid = hs.get_db_name()
qcxs_uid = helpers.hashstr(tuple(qcx_list))
qres_uid = hs_uid + query_uid + qcxs_uid
cache_dir = join(hs.dirs.cache_dir, 'query_results_bigcache')
print('\n===============')
print('\n[rr2] Load Query Results')
print('[rr2] load_query_results(): %r' % qres_uid)
io_kwargs = dict(dpath=cache_dir,
fname='query_results',
uid=qres_uid,
ext='.cPkl')
# Return cache if available
if not hs.args.nocache_query and (not force_load):
qcx2_res = io.smart_load(**io_kwargs)
if qcx2_res is not None:
print('[rr2] load_query_results(): cache hit')
return qcx2_res
print('[rr2] load_query_results(): cache miss')
else:
print('[rr2] load_query_results(): cache off')
# Individually load / compute queries
qcx2_res = [hs.query(qcx) for qcx in qcx_list]
# Save to the cache
print('[rr2] Saving query_results to bigcache: %r' % qres_uid)
helpers.ensuredir(cache_dir)
io.smart_save(qcx2_res, **io_kwargs)
return qcx2_res
def load_cached_test_results(hs, qreq, qcxs, dcxs, nocache_testres,
test_results_verbosity):
test_uid = qreq.get_query_uid(hs, qcxs)
cache_dir = join(hs.dirs.cache_dir, 'experiment_harness_results')
io_kwargs = {
'dpath': cache_dir,
'fname': 'test_results',
'uid': test_uid,
'ext': '.cPkl'
}
if test_results_verbosity == 2:
print('[harn] test_uid = %r' % test_uid)
# High level caching
if not params.args.nocache_query and (not nocache_testres):
qx2_bestranks = io.smart_load(**io_kwargs)
if qx2_bestranks is None:
print('[harn] qx2_bestranks cache returned None!')
elif len(qx2_bestranks) != len(qcxs):
print('[harn] Re-Caching qx2_bestranks')
else:
return qx2_bestranks
def precompute_akmeans(data,
num_clusters,
max_iters=100,
flann_params=None,
cache_dir=None,
force_recomp=False,
same_data=True,
uid=''):
'precompute aproximate kmeans'
if flann_params is None:
flann_params = {}
print('[algos] pre_akmeans()')
if same_data:
data_uid = helpers.hashstr_arr(data, 'dID')
uid += data_uid
clusters_fname = 'akmeans_clusters'
datax2cl_fname = 'akmeans_datax2cl'
try:
if not force_recomp:
clusters = io.smart_load(cache_dir,
clusters_fname,
uid,
'.npy',
can_fail=False)
datax2_clusterx = io.smart_load(cache_dir,
datax2cl_fname,
uid,
'.npy',
can_fail=False)
else:
raise Exception('forcing')
# Hack to refine akmeans with a few more iterations
if '--refine' in sys.argv or '--refine-exit' in sys.argv:
max_iters_override = helpers.get_arg('--refine', type_=int)
print('Overriding max_iters=%r' % max_iters_override)
if not max_iters_override is None:
max_iters = max_iters_override
datax2_clusterx_old = datax2_clusterx
print('[algos] refining:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
(datax2_clusterx,
clusters) = __akmeans_iterate(data, clusters, datax2_clusterx_old,
max_iters, flann_params, 0, 10)
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
'.npy')
if '--refine-exit' in sys.argv:
print('exiting after refine')
sys.exit(1)
print('[algos] pre_akmeans(): ... loaded akmeans.')
except Exception as ex:
print('[algos] pre_akmeans(): ... could not load akmeans.')
errstr = helpers.indent(repr(ex), '[algos] ')
print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
print('[algos] pre_akmeans(): printing debug_smart_load')
print('---- <DEBUG SMART LOAD>---')
io.debug_smart_load(cache_dir, clusters_fname)
io.debug_smart_load(cache_dir, datax2cl_fname)
print('----</DEBUG SMART LOAD>---')
#print('[algos] Press Ctrl+C to stop k-means early (and save)')
#signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
print('[algos] computing:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
print('[algos] pre_akmeans(): calling akmeans')
(datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
flann_params)
print('[algos] pre_akmeans(): finished running akmeans')
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
#print('[algos] Removing Ctrl+C signal handler')
#signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
print('[algos] pre_akmeans(): return')
return (datax2_clusterx, clusters)
def get_test_results(hs, qcx_list, qdat, cfgx=0, nCfg=1,
force_load=False):
dcxs = hs.get_indexed_sample()
query_uid = qdat.get_uid()
print('[harn] get_test_results(): %r' % query_uid)
hs_uid = hs.get_db_name()
qcxs_uid = helpers.hashstr_arr(qcx_list)
test_uid = hs_uid + query_uid + qcxs_uid
cache_dir = join(hs.dirs.cache_dir, 'experiment_harness_results')
io_kwargs = dict(dpath=cache_dir, fname='test_results', uid=test_uid, ext='.cPkl')
# High level caching
qx2_bestranks = []
#nChips = hs.get_num_chip()
#nNames = len(hs.tables.nx2_name) - 2
nQuery = len(qcx_list)
#NMultiNames =
nPrevQ = nQuery * cfgx
qx2_reslist = []
if not hs.args.nocache_query and (not force_load):
test_results = io.smart_load(**io_kwargs)
if test_results is None:
pass
elif len(test_results) != 1:
print('recaching test_results')
elif not test_results is None:
return test_results, [[{0: None}]] * nQuery
for qx, qcx in enumerate(qcx_list):
print(textwrap.dedent('''
[harn]----------------
[harn] TEST %d/%d
[harn]----------------''' % (qx + nPrevQ + 1, nQuery * nCfg)))
gt_cxs = hs.get_other_indexed_cxs(qcx)
#title = 'q' + hs.cidstr(qcx) + ' - ' + notes
#print('[harn] title=%r' % (title,))
#print('[harn] gt_' + hs.cidstr(gt_cxs))
res_list = mc3.execute_query_safe(hs, qdat, [qcx], dcxs)
bestranks = []
algos = []
qx2_reslist += [res_list]
assert len(res_list) == 1
for qcx2_res in res_list:
assert len(qcx2_res) == 1
res = qcx2_res[qcx]
algos += [res.title]
gt_ranks = res.get_gt_ranks(gt_cxs)
#print('[harn] cx_ranks(/%4r) = %r' % (nChips, gt_ranks))
#print('[harn] cx_ranks(/%4r) = %r' % (NMultiNames, gt_ranks))
#print('ns_ranks(/%4r) = %r' % (nNames, gt_ranks))
if len(gt_ranks) == 0:
_bestrank = -1
else:
_bestrank = min(gt_ranks)
bestranks += [_bestrank]
# record metadata
qx2_bestranks += [bestranks]
mat_vals = np.array(qx2_bestranks)
test_results = (mat_vals,)
# High level caching
helpers.ensuredir(cache_dir)
io.smart_save(test_results, **io_kwargs)
return test_results, qx2_reslist
def precompute_akmeans(data, num_clusters, max_iters=100,
flann_params=None, cache_dir=None,
force_recomp=False, same_data=True, uid=''):
'precompute aproximate kmeans'
if flann_params is None:
flann_params = {}
print('[algos] pre_akmeans()')
if same_data:
data_uid = helpers.hashstr_arr(data, 'dID')
uid += data_uid
clusters_fname = 'akmeans_clusters'
datax2cl_fname = 'akmeans_datax2cl'
try:
if not force_recomp:
clusters = io.smart_load(cache_dir, clusters_fname, uid, '.npy',
can_fail=False)
datax2_clusterx = io.smart_load(cache_dir, datax2cl_fname, uid,
'.npy', can_fail=False)
else:
raise Exception('forcing')
# Hack to refine akmeans with a few more iterations
if '--refine' in sys.argv or '--refine-exit' in sys.argv:
max_iters_override = helpers.get_arg('--refine', type_=int)
print('Overriding max_iters=%r' % max_iters_override)
if not max_iters_override is None:
max_iters = max_iters_override
datax2_clusterx_old = datax2_clusterx
print('[algos] refining:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
(datax2_clusterx, clusters) = __akmeans_iterate(
data, clusters, datax2_clusterx_old, max_iters, flann_params,
0, 10)
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
'.npy')
if '--refine-exit' in sys.argv:
print('exiting after refine')
sys.exit(1)
print('[algos] pre_akmeans(): ... loaded akmeans.')
except Exception as ex:
print('[algos] pre_akmeans(): ... could not load akmeans.')
errstr = helpers.indent(repr(ex), '[algos] ')
print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
print('[algos] pre_akmeans(): printing debug_smart_load')
print('---- <DEBUG SMART LOAD>---')
io.debug_smart_load(cache_dir, clusters_fname)
io.debug_smart_load(cache_dir, datax2cl_fname)
print('----</DEBUG SMART LOAD>---')
#print('[algos] Press Ctrl+C to stop k-means early (and save)')
#signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
print('[algos] computing:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
print('[algos] pre_akmeans(): calling akmeans')
(datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
flann_params)
print('[algos] pre_akmeans(): finished running akmeans')
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
#print('[algos] Removing Ctrl+C signal handler')
#signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
print('[algos] pre_akmeans(): return')
return (datax2_clusterx, clusters)