def show_descriptors_match_distances(orgres2_distance,
fnum=1,
db_name='',
**kwargs):
disttype_list = orgres2_distance.itervalues().next().keys()
orgtype_list = orgres2_distance.keys()
(nRow, nCol) = len(orgtype_list), len(disttype_list)
nColors = nRow * nCol
color_list = df2.distinct_colors(nColors)
df2.figure(fnum=fnum, docla=True, doclf=True)
pnum_ = lambda px: (nRow, nCol, px + 1)
plot_type = helpers.get_arg('--plot-type', default='plot')
# Remember min and max val for each distance type (l1, emd...)
distkey2_min = {distkey: np.uint64(-1) for distkey in disttype_list}
distkey2_max = {distkey: 0 for distkey in disttype_list}
def _distplot(dists, color, label, distkey, plot_type=plot_type):
data = sorted(dists)
ax = df2.gca()
min_ = distkey2_min[distkey]
max_ = distkey2_max[distkey]
if plot_type == 'plot':
df2.plot(data, color=color, label=label)
#xticks = np.linspace(np.min(data), np.max(data), 3)
#yticks = np.linspace(0, len(data), 5)
#ax.set_xticks(xticks)
#ax.set_yticks(yticks)
ax.set_ylim(min_, max_)
ax.set_xlim(0, len(dists))
ax.set_ylabel('distance')
ax.set_xlabel('matches indexes (sorted by distance)')
df2.legend(loc='lower right')
if plot_type == 'pdf':
df2.plot_pdf(data, color=color, label=label)
ax.set_ylabel('pr')
ax.set_xlabel('distance')
ax.set_xlim(min_, max_)
df2.legend(loc='upper right')
df2.dark_background(ax)
df2.small_xticks(ax)
df2.small_yticks(ax)
px = 0
for orgkey in orgtype_list:
for distkey in disttype_list:
dists = orgres2_distance[orgkey][distkey]
if len(dists) == 0:
continue
min_ = dists.min()
max_ = dists.max()
distkey2_min[distkey] = min(distkey2_min[distkey], min_)
distkey2_max[distkey] = max(distkey2_max[distkey], max_)
for orgkey in orgtype_list:
for distkey in disttype_list:
print(((orgkey, distkey)))
dists = orgres2_distance[orgkey][distkey]
df2.figure(fnum=fnum, pnum=pnum_(px))
color = color_list[px]
title = distkey + ' ' + orgkey
label = 'P(%s | %s)' % (distkey, orgkey)
_distplot(dists, color, label, distkey, **kwargs)
#ax = df2.gca()
#ax.set_title(title)
px += 1
subtitle = 'the matching distances between sift descriptors'
title = '(sift) matching distances'
if db_name != '':
title = db_name + ' ' + title
df2.set_figtitle(title, subtitle)
df2.adjust_subplots_safe()
def precompute_akmeans(data,
num_clusters,
max_iters=100,
flann_params=None,
cache_dir=None,
force_recomp=False,
same_data=True,
uid=''):
'precompute aproximate kmeans'
if flann_params is None:
flann_params = {}
print('[algos] pre_akmeans()')
if same_data:
data_uid = helpers.hashstr_arr(data, 'dID')
uid += data_uid
clusters_fname = 'akmeans_clusters'
datax2cl_fname = 'akmeans_datax2cl'
try:
if not force_recomp:
clusters = io.smart_load(cache_dir,
clusters_fname,
uid,
'.npy',
can_fail=False)
datax2_clusterx = io.smart_load(cache_dir,
datax2cl_fname,
uid,
'.npy',
can_fail=False)
else:
raise Exception('forcing')
# Hack to refine akmeans with a few more iterations
if '--refine' in sys.argv or '--refine-exit' in sys.argv:
max_iters_override = helpers.get_arg('--refine', type_=int)
print('Overriding max_iters=%r' % max_iters_override)
if not max_iters_override is None:
max_iters = max_iters_override
datax2_clusterx_old = datax2_clusterx
print('[algos] refining:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
(datax2_clusterx,
clusters) = __akmeans_iterate(data, clusters, datax2_clusterx_old,
max_iters, flann_params, 0, 10)
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
'.npy')
if '--refine-exit' in sys.argv:
print('exiting after refine')
sys.exit(1)
print('[algos] pre_akmeans(): ... loaded akmeans.')
except Exception as ex:
print('[algos] pre_akmeans(): ... could not load akmeans.')
errstr = helpers.indent(repr(ex), '[algos] ')
print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
print('[algos] pre_akmeans(): printing debug_smart_load')
print('---- <DEBUG SMART LOAD>---')
io.debug_smart_load(cache_dir, clusters_fname)
io.debug_smart_load(cache_dir, datax2cl_fname)
print('----</DEBUG SMART LOAD>---')
#print('[algos] Press Ctrl+C to stop k-means early (and save)')
#signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
print('[algos] computing:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
print('[algos] pre_akmeans(): calling akmeans')
(datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
flann_params)
print('[algos] pre_akmeans(): finished running akmeans')
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
#print('[algos] Removing Ctrl+C signal handler')
#signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
print('[algos] pre_akmeans(): return')
return (datax2_clusterx, clusters)
#import re
import warnings
# Scientific
import numpy as np
# Hotspotter
import draw_func2 as df2
import extract_patch
from hscom import fileio as io
from hscom import helpers
#from interaction import interact_keypoints, interact_chipres, interact_chip # NOQA
FNUMS = dict(image=1, chip=2, res=3, inspect=4, special=5, name=6)
IN_IMAGE_OVERRIDE = helpers.get_arg('--in-image-override',
type_=bool,
default=None)
SHOW_QUERY_OVERRIDE = helpers.get_arg('--show-query-override',
type_=bool,
default=None)
NO_LABEL_OVERRIDE = helpers.get_arg('--no-label-override',
type_=bool,
default=None)
@profile
def draw():
df2.adjust_subplots_safe()
df2.draw()
#import re
import warnings
# Scientific
import numpy as np
# Hotspotter
import draw_func2 as df2
import extract_patch
from hscom import fileio as io
from hscom import helpers
#from interaction import interact_keypoints, interact_chipres, interact_chip # NOQA
FNUMS = dict(image=1, chip=2, res=3, inspect=4, special=5, name=6)
IN_IMAGE_OVERRIDE = helpers.get_arg('--in-image-override', type_=bool, default=None)
SHOW_QUERY_OVERRIDE = helpers.get_arg('--show-query-override', type_=bool, default=None)
NO_LABEL_OVERRIDE = helpers.get_arg('--no-label-override', type_=bool, default=None)
@profile
def draw():
df2.adjust_subplots_safe()
df2.draw()
def register_FNUMS(FNUMS_):
global FNUMS
FNUMS = FNUMS_
def show_descriptors_match_distances(orgres2_distance, fnum=1, db_name='', **kwargs):
disttype_list = orgres2_distance.itervalues().next().keys()
orgtype_list = orgres2_distance.keys()
(nRow, nCol) = len(orgtype_list), len(disttype_list)
nColors = nRow * nCol
color_list = df2.distinct_colors(nColors)
df2.figure(fnum=fnum, docla=True, doclf=True)
pnum_ = lambda px: (nRow, nCol, px + 1)
plot_type = helpers.get_arg('--plot-type', default='plot')
# Remember min and max val for each distance type (l1, emd...)
distkey2_min = {distkey: np.uint64(-1) for distkey in disttype_list}
distkey2_max = {distkey: 0 for distkey in disttype_list}
def _distplot(dists, color, label, distkey, plot_type=plot_type):
data = sorted(dists)
ax = df2.gca()
min_ = distkey2_min[distkey]
max_ = distkey2_max[distkey]
if plot_type == 'plot':
df2.plot(data, color=color, label=label)
#xticks = np.linspace(np.min(data), np.max(data), 3)
#yticks = np.linspace(0, len(data), 5)
#ax.set_xticks(xticks)
#ax.set_yticks(yticks)
ax.set_ylim(min_, max_)
ax.set_xlim(0, len(dists))
ax.set_ylabel('distance')
ax.set_xlabel('matches indexes (sorted by distance)')
df2.legend(loc='lower right')
if plot_type == 'pdf':
df2.plot_pdf(data, color=color, label=label)
ax.set_ylabel('pr')
ax.set_xlabel('distance')
ax.set_xlim(min_, max_)
df2.legend(loc='upper right')
df2.dark_background(ax)
df2.small_xticks(ax)
df2.small_yticks(ax)
px = 0
for orgkey in orgtype_list:
for distkey in disttype_list:
dists = orgres2_distance[orgkey][distkey]
if len(dists) == 0:
continue
min_ = dists.min()
max_ = dists.max()
distkey2_min[distkey] = min(distkey2_min[distkey], min_)
distkey2_max[distkey] = max(distkey2_max[distkey], max_)
for orgkey in orgtype_list:
for distkey in disttype_list:
print(((orgkey, distkey)))
dists = orgres2_distance[orgkey][distkey]
df2.figure(fnum=fnum, pnum=pnum_(px))
color = color_list[px]
title = distkey + ' ' + orgkey
label = 'P(%s | %s)' % (distkey, orgkey)
_distplot(dists, color, label, distkey, **kwargs)
#ax = df2.gca()
#ax.set_title(title)
px += 1
subtitle = 'the matching distances between sift descriptors'
title = '(sift) matching distances'
if db_name != '':
title = db_name + ' ' + title
df2.set_figtitle(title, subtitle)
df2.adjust_subplots_safe()
from hscom import helpers
from hscom import helpers as util
from hsviz import viz
import multiprocessing
import numpy as np # NOQA
if __name__ == '__main__':
multiprocessing.freeze_support()
# Debugging vars
chip_cfg = None
#l')=103.7900s
cx_list = None
kwargs = {}
# --- LOAD TABLES --- #
args = argparse2.parse_arguments(defaultdb='NAUTS')
hs = api.HotSpotter(args)
hs.load_tables()
hs.update_samples()
# --- LOAD CHIPS --- #
force_compute = helpers.get_flag('--force', default=False)
cc2.load_chips(hs, force_compute=force_compute)
cx = helpers.get_arg('--cx', type_=int)
if not cx is None:
#tau = np.pi * 2
#hs.change_theta(cx, tau / 8)
viz.show_chip(hs, cx, draw_kpts=False, fnum=1)
viz.show_image(hs, hs.cx2_gx(cx), fnum=2)
else:
print('usage: feature_compute.py --cx [cx]')
exec(viz.df2.present())
# valid_cxs = np.where(hs.tables.cx2_cid > 0)[0]
if not np.iterable(cx_list):
valid_cxs = [cx_list]
cx_list = np.array(cx_list) # HACK
hs.load_chips(cx_list=cx_list)
hs.load_features(cx_list=cx_list)
#%%
# =============================================================================
# Detail plot function
# =============================================================================
(print, print_, print_on, print_off, rrr, profile, printDBG) = \
__common__.init(__name__, '[viz]', DEBUG=False)
NO_LABEL_OVERRIDE = helpers.get_arg('--no-label-override', type_=bool, default=None)
# COLORS
ORANGE = np.array((255, 127, 0, 255)) / 255.0
RED = np.array((255, 0, 0, 255)) / 255.0
GREEN = np.array(( 0, 255, 0, 255)) / 255.0
BLUE = np.array(( 0, 0, 255, 255)) / 255.0
YELLOW = np.array((255, 255, 0, 255)) / 255.0
BLACK = np.array(( 0, 0, 0, 255)) / 255.0
WHITE = np.array((255, 255, 255, 255)) / 255.0
GRAY = np.array((127, 127, 127, 255)) / 255.0
DEEP_PINK = np.array((255, 20, 147, 255)) / 255.0
PINK = np.array((255, 100, 100, 255)) / 255.0
FALSE_RED = np.array((255, 51, 0, 255)) / 255.0
TRUE_GREEN = np.array(( 0, 255, 0, 255)) / 255.0
#!/usr/env python
from __future__ import division, print_function
from hotspotter import HotSpotterAPI as api
from hotspotter import feature_compute2 as fc2
from hscom import helpers
from hscom import helpers as util
from hsviz import viz
from hscom import argparse2
import multiprocessing
if __name__ == '__main__':
multiprocessing.freeze_support()
print('[fc2] __main__ = feature_compute2.py')
# Read Args
cx = helpers.get_arg('--cx', type_=int)
delete_features = helpers.get_flag('--delete-features', default=False)
nRandKpts = helpers.get_arg('--nRandKpts', type_=int)
# Debugging vars
feat_cfg = None
cx_list = None
kwargs = {}
# --- LOAD TABLES --- #
args = argparse2.parse_arguments(db='NAUTS')
hs = api.HotSpotter(args)
hs.load_tables()
# --- LOAD CHIPS --- #
hs.update_samples()
hs.load_chips()
# Delete features if needed
if delete_features:
fc2.clear_feature_cache(hs)
def test_configurations(hs, qcx_list, test_cfg_name_list, fnum=1):
if __QUIET__:
mc3.print_off()
from hotspotter import HotSpotterAPI as api
api.print_off()
# Test Each configuration
if not __QUIET__:
print(
textwrap.dedent("""
[harn]================
[harn] experiment_harness.test_configurations()""").strip())
hs.update_samples()
# Grab list of algorithm configurations to test
cfg_list = get_cfg_list(hs, test_cfg_name_list)
if not __QUIET__:
print('[harn] Testing %d different parameters' % len(cfg_list))
print('[harn] %d different chips' % len(qcx_list))
# Preallocate test result aggregation structures
sel_cols = params.args.sel_cols # FIXME
sel_rows = params.args.sel_rows # FIXME
sel_cols = [] if sel_cols is None else sel_cols
sel_rows = [] if sel_rows is None else sel_rows
nCfg = len(cfg_list)
nQuery = len(qcx_list)
#rc2_res = np.empty((nQuery, nCfg), dtype=list) # row/col -> result
mat_list = []
qreq = ds.QueryRequest()
# TODO Add to argparse2
nocache_testres = util.get_flag('--nocache-testres', False)
test_results_verbosity = 2 - (2 * __QUIET__)
test_cfg_verbosity = 2
dbname = hs.get_db_name()
testnameid = dbname + ' ' + str(test_cfg_name_list)
msg = textwrap.dedent('''
---------------------
[harn] TEST_CFG %d/%d: ''' + testnameid + '''
---------------------''')
mark_progress = util.simple_progres_func(test_cfg_verbosity, msg, '+')
nomemory = params.args.nomemory
# Run each test configuration
# Query Config / Col Loop
dcxs = hs.get_indexed_sample()
for cfgx, query_cfg in enumerate(cfg_list):
if not __QUIET__:
mark_progress(cfgx + 1, nCfg)
# Set data to the current config
qreq = mc3.prep_query_request(qreq=qreq,
qcxs=qcx_list,
dcxs=dcxs,
query_cfg=query_cfg)
# Run the test / read cache
with util.Indenter2('[%s cfg %d/%d]' % (dbname, cfgx + 1, nCfg)):
qx2_bestranks = get_test_results2(hs, qcx_list, qreq, cfgx, nCfg,
nocache_testres,
test_results_verbosity)
if not nomemory:
mat_list.append(qx2_bestranks)
# Store the results
if not __QUIET__:
print('[harn] Finished testing parameters')
if nomemory:
print('ran tests in memory savings mode. exiting')
return
#--------------------
# Print Best Results
rank_mat = np.hstack(
mat_list) # concatenate each query rank across configs
# Label the rank matrix:
_colxs = np.arange(nCfg)
lbld_mat = util.debug_vstack([_colxs, rank_mat])
_rowxs = np.arange(nQuery + 1).reshape(nQuery + 1, 1) - 1
lbld_mat = np.hstack([_rowxs, lbld_mat])
#------------
# Build row labels
qx2_lbl = []
for qx in xrange(nQuery):
qcx = qcx_list[qx]
label = 'qx=%d) q%s ' % (qx, hs.cidstr(qcx, notes=True))
qx2_lbl.append(label)
qx2_lbl = np.array(qx2_lbl)
#------------
# Build col labels
cfgx2_lbl = []
for cfgx in xrange(nCfg):
test_uid = cfg_list[cfgx].get_uid()
test_uid = cfg_list[cfgx].get_uid()
cfg_label = 'cfgx=(%3d) %s' % (cfgx, test_uid)
cfgx2_lbl.append(cfg_label)
cfgx2_lbl = np.array(cfgx2_lbl)
#------------
indent = util.indent
@ArgGaurdFalse
def print_rowlbl():
print('=====================')
print('[harn] Row/Query Labels: %s' % testnameid)
print('=====================')
print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
print('--- /Row/Query Labels ---')
print_rowlbl()
#------------
@ArgGaurdFalse
def print_collbl():
print('')
print('=====================')
print('[harn] Col/Config Labels: %s' % testnameid)
print('=====================')
print('[harn] configs:\n%s' % '\n'.join(cfgx2_lbl))
print('--- /Col/Config Labels ---')
print_collbl()
#------------
# Build Colscore
qx2_min_rank = []
qx2_argmin_rank = []
new_hard_qx_list = []
new_qcid_list = []
new_hardtup_list = []
for qx in xrange(nQuery):
ranks = rank_mat[qx]
min_rank = ranks.min()
bestCFG_X = np.where(ranks == min_rank)[0]
qx2_min_rank.append(min_rank)
qx2_argmin_rank.append(bestCFG_X)
# Mark examples as hard
if ranks.max() > 0:
new_hard_qx_list += [qx]
for qx in new_hard_qx_list:
# New list is in cid format instead of cx format
# because you should be copying and pasting it
notes = ' ranks = ' + str(rank_mat[qx])
qcx = qcx_list[qx]
qcid = hs.tables.cx2_cid[qcx]
new_hardtup_list += [(qcid, notes)]
new_qcid_list += [qcid]
@ArgGaurdFalse
def print_rowscore():
print('')
print('=======================')
print('[harn] Scores per Query: %s' % testnameid)
print('=======================')
for qx in xrange(nQuery):
bestCFG_X = qx2_argmin_rank[qx]
min_rank = qx2_min_rank[qx]
minimizing_cfg_str = indent('\n'.join(cfgx2_lbl[bestCFG_X]),
' ')
#minimizing_cfg_str = str(bestCFG_X)
print('-------')
print(qx2_lbl[qx])
print(' best_rank = %d ' % min_rank)
if len(cfgx2_lbl) != 1:
print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
print_rowscore()
#------------
@ArgGaurdFalse
def print_hardcase():
print('===')
print('--- hard new_hardtup_list (w.r.t these configs): %s' %
testnameid)
print('\n'.join(map(repr, new_hardtup_list)))
print('There are %d hard cases ' % len(new_hardtup_list))
print(sorted([x[0] for x in new_hardtup_list]))
print('--- /Print Hardcase ---')
print_hardcase()
@ArgGaurdFalse
def echo_hardcase():
print('====')
print('--- hardcase commandline: %s' % testnameid)
hardcids_str = ' '.join(map(str, [' ', '--qcid'] + new_qcid_list))
print(hardcids_str)
print('--- /Echo Hardcase ---')
echo_hardcase()
#------------
# Build Colscore
X_list = [1, 5]
# Build a dictionary mapping X (as in #ranks < X) to a list of cfg scores
nLessX_dict = {int(X): np.zeros(nCfg) for X in iter(X_list)}
for cfgx in xrange(nCfg):
ranks = rank_mat[:, cfgx]
for X in iter(X_list):
#nLessX_ = sum(np.bitwise_and(ranks < X, ranks >= 0))
nLessX_ = sum(np.logical_and(ranks < X, ranks >= 0))
nLessX_dict[int(X)][cfgx] = nLessX_
@ArgGaurdFalse
def print_colscore():
print('')
print('==================')
print('[harn] Scores per Config: %s' % testnameid)
print('==================')
for cfgx in xrange(nCfg):
print('[score] %s' % (cfgx2_lbl[cfgx]))
for X in iter(X_list):
nLessX_ = nLessX_dict[int(X)][cfgx]
print(' ' + rankscore_str(X, nLessX_, nQuery))
print('--- /Scores per Config ---')
print_colscore()
#------------
@ArgGaurdFalse
def print_latexsum():
print('')
print('==========================')
print('[harn] LaTeX: %s' % testnameid)
print('==========================')
# Create configuration latex table
criteria_lbls = ['#ranks < %d' % X for X in X_list]
db_name = hs.get_db_name(True)
cfg_score_title = db_name + ' rank scores'
cfgscores = np.array([nLessX_dict[int(X)] for X in X_list]).T
replace_rowlbl = [(' *cfgx *', ' ')]
tabular_kwargs = dict(title=cfg_score_title,
out_of=nQuery,
bold_best=True,
replace_rowlbl=replace_rowlbl,
flip=True)
tabular_str = latex_formater.make_score_tabular(
cfgx2_lbl, criteria_lbls, cfgscores, **tabular_kwargs)
#latex_formater.render(tabular_str)
print(tabular_str)
print('--- /LaTeX ---')
print_latexsum()
#------------
best_rankscore_summary = []
to_intersect_list = []
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in nLessX_dict.iteritems():
max_LessX = cfgx2_nLessX.max()
bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
best_rankscore += rankscore_str(X, max_LessX, nQuery)
best_rankscore_summary += [best_rankscore]
to_intersect_list += [cfgx2_lbl[bestCFG_X]]
intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
for ix in xrange(1, len(to_intersect_list)):
intersected = np.intersect1d(intersected, to_intersect_list[ix])
@ArgGaurdFalse
def print_bestcfg():
print('')
print('==========================')
print('[harn] Best Configurations: %s' % testnameid)
print('==========================')
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in nLessX_dict.iteritems():
max_LessX = cfgx2_nLessX.max()
bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
best_rankscore += rankscore_str(X, max_LessX, nQuery)
uid_list = cfgx2_lbl[bestCFG_X]
#best_rankcfg = ''.join(map(wrap_uid, uid_list))
best_rankcfg = format_uid_list(uid_list)
#indent('\n'.join(uid_list), ' ')
print(best_rankscore)
print(best_rankcfg)
print('[cfg*] %d cfg(s) are the best of %d total cfgs' %
(len(intersected), nCfg))
print(format_uid_list(intersected))
print('--- /Best Configurations ---')
print_bestcfg()
#------------
@ArgGaurdFalse
def print_rankmat():
print('')
print('-------------')
print('RankMat: %s' % testnameid)
print(' nRows=%r, nCols=%r' % lbld_mat.shape)
print(' labled rank matrix: rows=queries, cols=cfgs:')
#np.set_printoptions(threshold=5000, linewidth=5000, precision=5)
with util.NpPrintOpts(threshold=5000, linewidth=5000, precision=5):
print(lbld_mat)
print('[harn]-------------')
print_rankmat()
#------------
sumstrs = []
sumstrs.append('')
sumstrs.append('||===========================')
sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
sumstrs.append('||---------------------------')
sumstrs.append(util.joins('\n|| ', best_rankscore_summary))
sumstrs.append('||===========================')
print('\n' + '\n'.join(sumstrs) + '\n')
#print('--- /SUMMARY ---')
# Draw results
if not __QUIET__:
print('remember to inspect with --sel-rows (-r) and --sel-cols (-c) ')
if len(sel_rows) > 0 and len(sel_cols) == 0:
sel_cols = range(len(cfg_list))
if len(sel_cols) > 0 and len(sel_rows) == 0:
sel_rows = range(len(qcx_list))
if params.args.view_all:
sel_rows = range(len(qcx_list))
sel_cols = range(len(cfg_list))
sel_cols = list(sel_cols)
sel_rows = list(sel_rows)
total = len(sel_cols) * len(sel_rows)
rciter = itertools.product(sel_rows, sel_cols)
prev_cfg = None
skip_to = util.get_arg('--skip-to', default=None)
dev_mode = util.get_arg('--devmode', default=False)
skip_list = []
if dev_mode:
hs.prefs.display_cfg.N = 3
df2.FONTS.axtitle = df2.FONTS.smaller
df2.FONTS.xlabel = df2.FONTS.smaller
df2.FONTS.figtitle = df2.FONTS.smaller
df2.SAFE_POS['top'] = .8
df2.SAFE_POS['bottom'] = .01
for count, (r, c) in enumerate(rciter):
if skip_to is not None:
if count < skip_to:
continue
if count in skip_list:
continue
# Get row and column index
qcx = qcx_list[r]
query_cfg = cfg_list[c]
print('\n\n___________________________________')
print(' --- VIEW %d / %d --- ' % (count + 1, total))
print('--------------------------------------')
print('viewing (r, c) = (%r, %r)' % (r, c))
# Load / Execute the query
qreq = mc3.prep_query_request(qreq=qreq,
qcxs=[qcx],
dcxs=dcxs,
query_cfg=query_cfg)
qcx2_res = mc3.process_query_request(hs, qreq, safe=True)
res = qcx2_res[qcx]
# Print Query UID
print(res.uid)
# Draw Result
#res.show_top(hs, fnum=fnum)
if prev_cfg != query_cfg:
# This is way too aggro. Needs to be a bit lazier
hs.refresh_features()
prev_cfg = query_cfg
fnum = count
title_uid = res.uid
title_uid = title_uid.replace('_FEAT', '\n_FEAT')
res.show_analysis(hs,
fnum=fnum,
aug='\n' + title_uid,
annote=1,
show_name=False,
show_gname=False,
time_appart=False)
df2.adjust_subplots_safe()
if params.args.save_figures:
from hsviz import allres_viz
allres_viz.dump(hs, 'analysis', quality=True, overwrite=False)
if not __QUIET__:
print('[harn] EXIT EXPERIMENT HARNESS')
def precompute_akmeans(data, num_clusters, max_iters=100,
flann_params=None, cache_dir=None,
force_recomp=False, same_data=True, uid=''):
'precompute aproximate kmeans'
if flann_params is None:
flann_params = {}
print('[algos] pre_akmeans()')
if same_data:
data_uid = helpers.hashstr_arr(data, 'dID')
uid += data_uid
clusters_fname = 'akmeans_clusters'
datax2cl_fname = 'akmeans_datax2cl'
try:
if not force_recomp:
clusters = io.smart_load(cache_dir, clusters_fname, uid, '.npy',
can_fail=False)
datax2_clusterx = io.smart_load(cache_dir, datax2cl_fname, uid,
'.npy', can_fail=False)
else:
raise Exception('forcing')
# Hack to refine akmeans with a few more iterations
if '--refine' in sys.argv or '--refine-exit' in sys.argv:
max_iters_override = helpers.get_arg('--refine', type_=int)
print('Overriding max_iters=%r' % max_iters_override)
if not max_iters_override is None:
max_iters = max_iters_override
datax2_clusterx_old = datax2_clusterx
print('[algos] refining:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
(datax2_clusterx, clusters) = __akmeans_iterate(
data, clusters, datax2_clusterx_old, max_iters, flann_params,
0, 10)
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid,
'.npy')
if '--refine-exit' in sys.argv:
print('exiting after refine')
sys.exit(1)
print('[algos] pre_akmeans(): ... loaded akmeans.')
except Exception as ex:
print('[algos] pre_akmeans(): ... could not load akmeans.')
errstr = helpers.indent(repr(ex), '[algos] ')
print('[algos] pre_akmeans(): ... caught ex:\n %s ' % errstr)
print('[algos] pre_akmeans(): printing debug_smart_load')
print('---- <DEBUG SMART LOAD>---')
io.debug_smart_load(cache_dir, clusters_fname)
io.debug_smart_load(cache_dir, datax2cl_fname)
print('----</DEBUG SMART LOAD>---')
#print('[algos] Press Ctrl+C to stop k-means early (and save)')
#signal.signal(signal.SIGINT, force_quit_akmeans) # set ctrl+c behavior
print('[algos] computing:')
print('[algos] ' + '_'.join([clusters_fname, uid]) + '.npy')
print('[algos] ' + '_'.join([datax2cl_fname, uid]) + '.npy')
print('[algos] pre_akmeans(): calling akmeans')
(datax2_clusterx, clusters) = akmeans(data, num_clusters, max_iters,
flann_params)
print('[algos] pre_akmeans(): finished running akmeans')
io.smart_save(clusters, cache_dir, clusters_fname, uid, '.npy')
io.smart_save(datax2_clusterx, cache_dir, datax2cl_fname, uid, '.npy')
#print('[algos] Removing Ctrl+C signal handler')
#signal.signal(signal.SIGINT, signal.SIG_DFL) # reset ctrl+c behavior
print('[algos] pre_akmeans(): return')
return (datax2_clusterx, clusters)
