def dev_reload():
print('===========================')
print('[dev] performing dev_reload')
print('---------------------------')
from hotspotter import algos
from hotspotter import chip_compute2 as cc2
from hotspotter import feature_compute2 as fc2
from hsviz import interaction
#from scripts
from hscom import fileio as io
from hotspotter import nn_filters
rrr()
io.rrr()
ds.rrr()
mf.rrr()
nn_filters.rrr()
mc3.rrr()
viz.rrr()
interaction.rrr()
#scripts.rrr()
vr2.rrr()
helpers.rrr()
cc2.rrr()
fc2.rrr()
algos.rrr()
df2.rrr()
print('---------------------------')
print('df2 reset')
df2.reset()
print('---------------------------')
print('[dev] finished dev_reload()')
print('===========================')
def dev_reload():
print('===========================')
print('[dev] performing dev_reload')
print('---------------------------')
from hotspotter import algos
from hotspotter import chip_compute2 as cc2
from hotspotter import feature_compute2 as fc2
from hsviz import interaction
#from scripts
from hscom import fileio as io
from hotspotter import nn_filters
rrr()
io.rrr()
ds.rrr()
mf.rrr()
nn_filters.rrr()
mc3.rrr()
viz.rrr()
interaction.rrr()
#scripts.rrr()
vr2.rrr()
helpers.rrr()
cc2.rrr()
fc2.rrr()
algos.rrr()
df2.rrr()
print('---------------------------')
print('df2 reset')
df2.reset()
print('---------------------------')
print('[dev] finished dev_reload()')
print('===========================')
def precompute_queries(back):
# Batch -> Precompute Queries
# TODO:
#http://stackoverflow.com/questions/15637768/
# pyqt-how-to-capture-output-of-pythons-interpreter-
# and-display-it-in-qedittext
#prevBlock = back.front.blockSignals(True)
import matching_functions as mf
import DataStructures as ds
import match_chips3 as mc3
import sys
back.precompute_feats()
valid_cx = back.hs.get_valid_cxs()
if back.hs.args.quiet:
mc3.print_off()
ds.print_off()
mf.print_off()
fmtstr = helpers.progress_str(len(valid_cx), '[back*] Query qcx=%r: ')
for count, qcx in enumerate(valid_cx):
sys.stdout.write(fmtstr % (qcx, count))
back.hs.query(qcx, dochecks=False)
if count % 100 == 0:
sys.stdout.write('\n ...')
sys.stdout.write('\n ...')
mc3.print_on()
ds.print_on()
mf.print_on()
print('')
def precompute_queries(back):
# Batch -> Precompute Queries
# TODO:
#http://stackoverflow.com/questions/15637768/
# pyqt-how-to-capture-output-of-pythons-interpreter-
# and-display-it-in-qedittext
#prevBlock = back.front.blockSignals(True)
from hotspotter import matching_functions as mf
from hotspotter import DataStructures as ds
from hotspotter import match_chips3 as mc3
import sys
import pdb
pdb.set_trace()
back.precompute_feats()
valid_cx = back.hs.get_valid_cxs()
if back.hs.args.quiet:
mc3.print_off()
ds.print_off()
mf.print_off()
fmtstr = helpers.progress_str(len(valid_cx), '[back*] Query qcx=%r: ')
for count, qcx in enumerate(valid_cx):
sys.stdout.write(fmtstr % (qcx, count))
back.hs.query(qcx, dochecks=False)
if count % 100 == 0:
sys.stdout.write('\n ...')
sys.stdout.write('\n ...')
mc3.print_on()
ds.print_on()
mf.print_on()
print('')
def all_printoff():
from hotspotter import fileio as io
from hotspotter import HotSpotterAPI as api
ds.print_off()
mf.print_off()
io.print_off()
api.print_off()
mc3.print_off()
vr2.print_off()
def all_printoff():
from hotspotter import fileio as io
from hotspotter import HotSpotterAPI as api
ds.print_off()
mf.print_off()
io.print_off()
api.print_off()
mc3.print_off()
vr2.print_off()
def _dev_reload():
# TODO Integrate this better
print('===========================')
print('[dev] performing dev_reload')
print('---------------------------')
from hotspotter import DataStructures as ds
from hotspotter import algos
from hotspotter import load_data2 as ld2
from hotspotter import chip_compute2 as cc2
from hotspotter import feature_compute2 as fc2
from hotspotter import match_chips3 as mc3
from hotspotter import matching_functions as mf
from hotspotter import nn_filters
from hotspotter import report_results2 as rr2
from hotspotter import voting_rules2 as vr2
# Common
from hscom import fileio as io # NOQA
from hscom import helpers # NOQA
# Viz
from hsviz import draw_func2 as df2 # NOQA
from hsviz import interact # NOQA
from hsviz import viz
# GUI
from hsgui import guitools # NOQA
from hsgui import guifront # NOQA
from hsgui import guiback # NOQA
# Self
rrr()
# com
helpers.rrr()
io.rrr()
# hotspotter
ld2.rrr()
ds.rrr()
mf.rrr()
nn_filters.rrr()
mc3.rrr()
vr2.rrr()
cc2.rrr()
rr2.rrr()
fc2.rrr()
algos.rrr()
# gui
guitools.rrr()
guifront.rrr()
guiback.rrr()
# viz
viz.rrr()
interact.rrr()
df2.rrr()
print('---------------------------')
print('df2 reset')
df2.reset()
print('---------------------------')
print('[dev] finished dev_reload()')
print('===========================')
def _dev_reload():
# TODO Integrate this better
print('===========================')
print('[dev] performing dev_reload')
print('---------------------------')
from hotspotter import DataStructures as ds
from hotspotter import algos
from hotspotter import load_data2 as ld2
from hotspotter import chip_compute2 as cc2
from hotspotter import feature_compute2 as fc2
from hotspotter import match_chips3 as mc3
from hotspotter import matching_functions as mf
from hotspotter import nn_filters
from hotspotter import report_results2 as rr2
from hotspotter import voting_rules2 as vr2
# Common
from hscom import fileio as io # NOQA
from hscom import helpers # NOQA
# Viz
from hsviz import draw_func2 as df2 # NOQA
from hsviz import interact # NOQA
from hsviz import viz
# GUI
from hsgui import guitools # NOQA
from hsgui import guifront # NOQA
from hsgui import guiback # NOQA
# Self
rrr()
# com
helpers.rrr()
io.rrr()
# hotspotter
ld2.rrr()
ds.rrr()
mf.rrr()
nn_filters.rrr()
mc3.rrr()
vr2.rrr()
cc2.rrr()
rr2.rrr()
fc2.rrr()
algos.rrr()
# gui
guitools.rrr()
guifront.rrr()
guiback.rrr()
# viz
viz.rrr()
interact.rrr()
df2.rrr()
print('---------------------------')
print('df2 reset')
df2.reset()
print('---------------------------')
print('[dev] finished dev_reload()')
print('===========================')
def investigate_vsone_groundtruth(hs, qcx_list, fnum=1):
print('--------------------------------------')
print('[dev] investigate_vsone_groundtruth')
query_cfg = ds.get_vsone_cfg(sv_on=True, ratio_thresh=1.5)
for qcx in qcx_list:
res = hs.query_groundtruth(hs, qcx, query_cfg)
#print(query_cfg)
#print(res)
#res.show_query(hs, fnum=fnum)
fnum += 1
res.show_topN(hs, fnum=fnum, query_cfg=query_cfg)
fnum += 1
return fnum
def investigate_vsone_groundtruth(hs, qcx_list, fnum=1):
print('--------------------------------------')
print('[dev] investigate_vsone_groundtruth')
query_cfg = ds.get_vsone_cfg(sv_on=True, ratio_thresh=1.5)
for qcx in qcx_list:
res = hs.query_groundtruth(hs, qcx, query_cfg)
#print(query_cfg)
#print(res)
#res.show_query(hs, fnum=fnum)
fnum += 1
res.show_topN(hs, fnum=fnum, query_cfg=query_cfg)
fnum += 1
return fnum
def vary_vsmany_cfg(hs, qcx_list, vary_dicts, fnum, **kwargs):
vary_cfg = helpers.dict_union(*vary_dicts)
query_cfg = ds.get_vsmany_cfg(hs, **kwargs)
return vary_query_cfg(hs, qcx_list, query_cfg, vary_cfg, fnum)
def vary_vsmany_cfg(hs, qcx_list, vary_dicts, fnum, **kwargs):
vary_cfg = helpers.dict_union(*vary_dicts)
query_cfg = ds.get_vsmany_cfg(hs, **kwargs)
return vary_query_cfg(hs, qcx_list, query_cfg, vary_cfg, fnum)
def test_configurations(hs, qcx_list, test_cfg_name_list, fnum=1):
if __QUIET__:
mc3.print_off()
from hotspotter import HotSpotterAPI as api
api.print_off()
# Test Each configuration
if not __QUIET__:
print(
textwrap.dedent("""
[harn]================
[harn] experiment_harness.test_configurations()""").strip())
hs.update_samples()
# Grab list of algorithm configurations to test
cfg_list = get_cfg_list(hs, test_cfg_name_list)
if not __QUIET__:
print('[harn] Testing %d different parameters' % len(cfg_list))
print('[harn] %d different chips' % len(qcx_list))
# Preallocate test result aggregation structures
sel_cols = params.args.sel_cols # FIXME
sel_rows = params.args.sel_rows # FIXME
sel_cols = [] if sel_cols is None else sel_cols
sel_rows = [] if sel_rows is None else sel_rows
nCfg = len(cfg_list)
nQuery = len(qcx_list)
#rc2_res = np.empty((nQuery, nCfg), dtype=list) # row/col -> result
mat_list = []
qreq = ds.QueryRequest()
# TODO Add to argparse2
nocache_testres = util.get_flag('--nocache-testres', False)
test_results_verbosity = 2 - (2 * __QUIET__)
test_cfg_verbosity = 2
dbname = hs.get_db_name()
testnameid = dbname + ' ' + str(test_cfg_name_list)
msg = textwrap.dedent('''
---------------------
[harn] TEST_CFG %d/%d: ''' + testnameid + '''
---------------------''')
mark_progress = util.simple_progres_func(test_cfg_verbosity, msg, '+')
nomemory = params.args.nomemory
# Run each test configuration
# Query Config / Col Loop
dcxs = hs.get_indexed_sample()
for cfgx, query_cfg in enumerate(cfg_list):
if not __QUIET__:
mark_progress(cfgx + 1, nCfg)
# Set data to the current config
qreq = mc3.prep_query_request(qreq=qreq,
qcxs=qcx_list,
dcxs=dcxs,
query_cfg=query_cfg)
# Run the test / read cache
with util.Indenter2('[%s cfg %d/%d]' % (dbname, cfgx + 1, nCfg)):
qx2_bestranks = get_test_results2(hs, qcx_list, qreq, cfgx, nCfg,
nocache_testres,
test_results_verbosity)
if not nomemory:
mat_list.append(qx2_bestranks)
# Store the results
if not __QUIET__:
print('[harn] Finished testing parameters')
if nomemory:
print('ran tests in memory savings mode. exiting')
return
#--------------------
# Print Best Results
rank_mat = np.hstack(
mat_list) # concatenate each query rank across configs
# Label the rank matrix:
_colxs = np.arange(nCfg)
lbld_mat = util.debug_vstack([_colxs, rank_mat])
_rowxs = np.arange(nQuery + 1).reshape(nQuery + 1, 1) - 1
lbld_mat = np.hstack([_rowxs, lbld_mat])
#------------
# Build row labels
qx2_lbl = []
for qx in xrange(nQuery):
qcx = qcx_list[qx]
label = 'qx=%d) q%s ' % (qx, hs.cidstr(qcx, notes=True))
qx2_lbl.append(label)
qx2_lbl = np.array(qx2_lbl)
#------------
# Build col labels
cfgx2_lbl = []
for cfgx in xrange(nCfg):
test_uid = cfg_list[cfgx].get_uid()
test_uid = cfg_list[cfgx].get_uid()
cfg_label = 'cfgx=(%3d) %s' % (cfgx, test_uid)
cfgx2_lbl.append(cfg_label)
cfgx2_lbl = np.array(cfgx2_lbl)
#------------
indent = util.indent
@ArgGaurdFalse
def print_rowlbl():
print('=====================')
print('[harn] Row/Query Labels: %s' % testnameid)
print('=====================')
print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
print('--- /Row/Query Labels ---')
print_rowlbl()
#------------
@ArgGaurdFalse
def print_collbl():
print('')
print('=====================')
print('[harn] Col/Config Labels: %s' % testnameid)
print('=====================')
print('[harn] configs:\n%s' % '\n'.join(cfgx2_lbl))
print('--- /Col/Config Labels ---')
print_collbl()
#------------
# Build Colscore
qx2_min_rank = []
qx2_argmin_rank = []
new_hard_qx_list = []
new_qcid_list = []
new_hardtup_list = []
for qx in xrange(nQuery):
ranks = rank_mat[qx]
min_rank = ranks.min()
bestCFG_X = np.where(ranks == min_rank)[0]
qx2_min_rank.append(min_rank)
qx2_argmin_rank.append(bestCFG_X)
# Mark examples as hard
if ranks.max() > 0:
new_hard_qx_list += [qx]
for qx in new_hard_qx_list:
# New list is in cid format instead of cx format
# because you should be copying and pasting it
notes = ' ranks = ' + str(rank_mat[qx])
qcx = qcx_list[qx]
qcid = hs.tables.cx2_cid[qcx]
new_hardtup_list += [(qcid, notes)]
new_qcid_list += [qcid]
@ArgGaurdFalse
def print_rowscore():
print('')
print('=======================')
print('[harn] Scores per Query: %s' % testnameid)
print('=======================')
for qx in xrange(nQuery):
bestCFG_X = qx2_argmin_rank[qx]
min_rank = qx2_min_rank[qx]
minimizing_cfg_str = indent('\n'.join(cfgx2_lbl[bestCFG_X]),
' ')
#minimizing_cfg_str = str(bestCFG_X)
print('-------')
print(qx2_lbl[qx])
print(' best_rank = %d ' % min_rank)
if len(cfgx2_lbl) != 1:
print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
print_rowscore()
#------------
@ArgGaurdFalse
def print_hardcase():
print('===')
print('--- hard new_hardtup_list (w.r.t these configs): %s' %
testnameid)
print('\n'.join(map(repr, new_hardtup_list)))
print('There are %d hard cases ' % len(new_hardtup_list))
print(sorted([x[0] for x in new_hardtup_list]))
print('--- /Print Hardcase ---')
print_hardcase()
@ArgGaurdFalse
def echo_hardcase():
print('====')
print('--- hardcase commandline: %s' % testnameid)
hardcids_str = ' '.join(map(str, [' ', '--qcid'] + new_qcid_list))
print(hardcids_str)
print('--- /Echo Hardcase ---')
echo_hardcase()
#------------
# Build Colscore
X_list = [1, 5]
# Build a dictionary mapping X (as in #ranks < X) to a list of cfg scores
nLessX_dict = {int(X): np.zeros(nCfg) for X in iter(X_list)}
for cfgx in xrange(nCfg):
ranks = rank_mat[:, cfgx]
for X in iter(X_list):
#nLessX_ = sum(np.bitwise_and(ranks < X, ranks >= 0))
nLessX_ = sum(np.logical_and(ranks < X, ranks >= 0))
nLessX_dict[int(X)][cfgx] = nLessX_
@ArgGaurdFalse
def print_colscore():
print('')
print('==================')
print('[harn] Scores per Config: %s' % testnameid)
print('==================')
for cfgx in xrange(nCfg):
print('[score] %s' % (cfgx2_lbl[cfgx]))
for X in iter(X_list):
nLessX_ = nLessX_dict[int(X)][cfgx]
print(' ' + rankscore_str(X, nLessX_, nQuery))
print('--- /Scores per Config ---')
print_colscore()
#------------
@ArgGaurdFalse
def print_latexsum():
print('')
print('==========================')
print('[harn] LaTeX: %s' % testnameid)
print('==========================')
# Create configuration latex table
criteria_lbls = ['#ranks < %d' % X for X in X_list]
db_name = hs.get_db_name(True)
cfg_score_title = db_name + ' rank scores'
cfgscores = np.array([nLessX_dict[int(X)] for X in X_list]).T
replace_rowlbl = [(' *cfgx *', ' ')]
tabular_kwargs = dict(title=cfg_score_title,
out_of=nQuery,
bold_best=True,
replace_rowlbl=replace_rowlbl,
flip=True)
tabular_str = latex_formater.make_score_tabular(
cfgx2_lbl, criteria_lbls, cfgscores, **tabular_kwargs)
#latex_formater.render(tabular_str)
print(tabular_str)
print('--- /LaTeX ---')
print_latexsum()
#------------
best_rankscore_summary = []
to_intersect_list = []
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in nLessX_dict.iteritems():
max_LessX = cfgx2_nLessX.max()
bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
best_rankscore += rankscore_str(X, max_LessX, nQuery)
best_rankscore_summary += [best_rankscore]
to_intersect_list += [cfgx2_lbl[bestCFG_X]]
intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
for ix in xrange(1, len(to_intersect_list)):
intersected = np.intersect1d(intersected, to_intersect_list[ix])
@ArgGaurdFalse
def print_bestcfg():
print('')
print('==========================')
print('[harn] Best Configurations: %s' % testnameid)
print('==========================')
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in nLessX_dict.iteritems():
max_LessX = cfgx2_nLessX.max()
bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
best_rankscore += rankscore_str(X, max_LessX, nQuery)
uid_list = cfgx2_lbl[bestCFG_X]
#best_rankcfg = ''.join(map(wrap_uid, uid_list))
best_rankcfg = format_uid_list(uid_list)
#indent('\n'.join(uid_list), ' ')
print(best_rankscore)
print(best_rankcfg)
print('[cfg*] %d cfg(s) are the best of %d total cfgs' %
(len(intersected), nCfg))
print(format_uid_list(intersected))
print('--- /Best Configurations ---')
print_bestcfg()
#------------
@ArgGaurdFalse
def print_rankmat():
print('')
print('-------------')
print('RankMat: %s' % testnameid)
print(' nRows=%r, nCols=%r' % lbld_mat.shape)
print(' labled rank matrix: rows=queries, cols=cfgs:')
#np.set_printoptions(threshold=5000, linewidth=5000, precision=5)
with util.NpPrintOpts(threshold=5000, linewidth=5000, precision=5):
print(lbld_mat)
print('[harn]-------------')
print_rankmat()
#------------
sumstrs = []
sumstrs.append('')
sumstrs.append('||===========================')
sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
sumstrs.append('||---------------------------')
sumstrs.append(util.joins('\n|| ', best_rankscore_summary))
sumstrs.append('||===========================')
print('\n' + '\n'.join(sumstrs) + '\n')
#print('--- /SUMMARY ---')
# Draw results
if not __QUIET__:
print('remember to inspect with --sel-rows (-r) and --sel-cols (-c) ')
if len(sel_rows) > 0 and len(sel_cols) == 0:
sel_cols = range(len(cfg_list))
if len(sel_cols) > 0 and len(sel_rows) == 0:
sel_rows = range(len(qcx_list))
if params.args.view_all:
sel_rows = range(len(qcx_list))
sel_cols = range(len(cfg_list))
sel_cols = list(sel_cols)
sel_rows = list(sel_rows)
total = len(sel_cols) * len(sel_rows)
rciter = itertools.product(sel_rows, sel_cols)
prev_cfg = None
skip_to = util.get_arg('--skip-to', default=None)
dev_mode = util.get_arg('--devmode', default=False)
skip_list = []
if dev_mode:
hs.prefs.display_cfg.N = 3
df2.FONTS.axtitle = df2.FONTS.smaller
df2.FONTS.xlabel = df2.FONTS.smaller
df2.FONTS.figtitle = df2.FONTS.smaller
df2.SAFE_POS['top'] = .8
df2.SAFE_POS['bottom'] = .01
for count, (r, c) in enumerate(rciter):
if skip_to is not None:
if count < skip_to:
continue
if count in skip_list:
continue
# Get row and column index
qcx = qcx_list[r]
query_cfg = cfg_list[c]
print('\n\n___________________________________')
print(' --- VIEW %d / %d --- ' % (count + 1, total))
print('--------------------------------------')
print('viewing (r, c) = (%r, %r)' % (r, c))
# Load / Execute the query
qreq = mc3.prep_query_request(qreq=qreq,
qcxs=[qcx],
dcxs=dcxs,
query_cfg=query_cfg)
qcx2_res = mc3.process_query_request(hs, qreq, safe=True)
res = qcx2_res[qcx]
# Print Query UID
print(res.uid)
# Draw Result
#res.show_top(hs, fnum=fnum)
if prev_cfg != query_cfg:
# This is way too aggro. Needs to be a bit lazier
hs.refresh_features()
prev_cfg = query_cfg
fnum = count
title_uid = res.uid
title_uid = title_uid.replace('_FEAT', '\n_FEAT')
res.show_analysis(hs,
fnum=fnum,
aug='\n' + title_uid,
annote=1,
show_name=False,
show_gname=False,
time_appart=False)
df2.adjust_subplots_safe()
if params.args.save_figures:
from hsviz import allres_viz
allres_viz.dump(hs, 'analysis', quality=True, overwrite=False)
if not __QUIET__:
print('[harn] EXIT EXPERIMENT HARNESS')