def vary_two_cfg(hs, qcx, cx, query_cfg, vary_cfg, fnum=1):
if len(vary_cfg) > 2:
raise Exception('can only vary at most two cfgeters')
print('[dev] vary_two_cfg: q' + hs.vs_str(qcx, cx))
cfg_keys = vary_cfg.keys()
cfg_vals = vary_cfg.values()
cfg1_name = cfg_keys[0]
cfg2_name = cfg_keys[1]
cfg1_steps = cfg_vals[0]
cfg2_steps = cfg_vals[1]
nRows = len(cfg1_steps)
nCols = len(cfg2_steps)
print('[dev] Varying configs: nRows=%r, nCols=%r' % (nRows, nCols))
print('[dev] %r = %r ' % (cfg1_name, cfg1_steps))
print('[dev] %r = %r ' % (cfg2_name, cfg2_steps))
ylabel_args = dict(rotation='horizontal',
verticalalignment='bottom',
horizontalalignment='right',
fontproperties=df2.FONTS.medbold)
xlabel_args = dict(fontproperties=df2.FONTS.medbold)
#ax = df2.gca()
# Vary cfg1
#df2..gcf().clf()
print_lock_ = helpers.ModulePrintLock(mc3, df2)
assign_alg = query_cfg.agg_cfg.query_type
vert = not hs.args.horiz
plt_match_args = dict(fnum=fnum, show_gname=False, showTF=False, vert=vert)
for rowx, cfg1_value in enumerate(cfg1_steps):
query_cfg.update_cfg(**{cfg1_name: cfg1_value})
y_title = cfg1_name + '=' + helpers.format(cfg1_value, 3)
# Vary cfg2
for colx, cfg2_value in enumerate(cfg2_steps):
query_cfg.update_cfg(**{cfg2_name: cfg2_value})
pnum = (nRows, nCols, rowx * nCols + colx + 1)
# HACK
#print(pnum)
#print(query_cfg)
# query only the chips of interest (groundtruth) when doing vsone
if assign_alg == 'vsone':
res = hs.query_groundtruth(qcx, query_cfg)
# query the entire database in vsmany (just as fast as vgroundtruth)
elif assign_alg == 'vsmany':
res = hs.query(qcx, query_cfg)
res.plot_single_match(hs, cx, pnum=pnum, **plt_match_args)
x_title = cfg2_name + '=' + helpers.format(cfg2_value, 3) # helpers.commas(cfg2_value, 3)
ax = df2.gca()
if rowx == len(cfg1_steps) - 1:
ax.set_xlabel(x_title, **xlabel_args)
if colx == 0:
ax.set_ylabel(y_title, **ylabel_args)
del print_lock_
vary_title = '%s vary %s and %s' % (assign_alg, cfg1_name, cfg2_name)
figtitle = '%s %s %s' % (vary_title, hs.vs_str(qcx, cx), str(hs.cx2_property(qcx, 'Notes')))
subtitle = mc3.simplify_test_uid(query_cfg.get_uid())
df2.set_figtitle(figtitle, subtitle)
df2.adjust_subplots_xylabels()
fnum += 1
viz.save_if_requested(hs, vary_title)
return fnum
def print_test_results(test_results):
print('[harn] ---')
(col_lbls, row_lbls, mat_vals, test_uid, nLeX) = test_results
test_uid = mc3.simplify_test_uid(test_uid)
print('[harn] test_uid=%r' % test_uid)
#print('[harn] row_lbls=\n%s' % str(row_lbls))
#print('[harn] col_lbls=\n%s' % str('\n '.join(col_lbls)))
print('[harn] lowest_gt_ranks(NN,FILT,SV)=\n%s' % str(mat_vals))
def vary_two_cfg(hs, qcx, cx, query_cfg, vary_cfg, fnum=1):
if len(vary_cfg) > 2:
raise Exception('can only vary at most two cfgeters')
print('[dev] vary_two_cfg: q' + hs.vs_str(qcx, cx))
cfg_keys = vary_cfg.keys()
cfg_vals = vary_cfg.values()
cfg1_name = cfg_keys[0]
cfg2_name = cfg_keys[1]
cfg1_steps = cfg_vals[0]
cfg2_steps = cfg_vals[1]
nRows = len(cfg1_steps)
nCols = len(cfg2_steps)
print('[dev] Varying configs: nRows=%r, nCols=%r' % (nRows, nCols))
print('[dev] %r = %r ' % (cfg1_name, cfg1_steps))
print('[dev] %r = %r ' % (cfg2_name, cfg2_steps))
ylabel_args = dict(rotation='horizontal',
verticalalignment='bottom',
horizontalalignment='right',
fontproperties=df2.FONTS.medbold)
xlabel_args = dict(fontproperties=df2.FONTS.medbold)
#ax = df2.gca()
# Vary cfg1
#df2..gcf().clf()
print_lock_ = helpers.ModulePrintLock(mc3, df2)
assign_alg = query_cfg.agg_cfg.query_type
vert = not hs.args.horiz
plt_match_args = dict(fnum=fnum, show_gname=False, showTF=False, vert=vert)
for rowx, cfg1_value in enumerate(cfg1_steps):
query_cfg.update_cfg(**{cfg1_name: cfg1_value})
y_title = cfg1_name + '=' + helpers.format(cfg1_value, 3)
# Vary cfg2
for colx, cfg2_value in enumerate(cfg2_steps):
query_cfg.update_cfg(**{cfg2_name: cfg2_value})
pnum = (nRows, nCols, rowx * nCols + colx + 1)
# HACK
#print(pnum)
#print(query_cfg)
# query only the chips of interest (groundtruth) when doing vsone
if assign_alg == 'vsone':
res = hs.query_groundtruth(qcx, query_cfg)
# query the entire database in vsmany (just as fast as vgroundtruth)
elif assign_alg == 'vsmany':
res = hs.query(qcx, query_cfg)
res.plot_single_match(hs, cx, pnum=pnum, **plt_match_args)
x_title = cfg2_name + '=' + helpers.format(
cfg2_value, 3) # helpers.commas(cfg2_value, 3)
ax = df2.gca()
if rowx == len(cfg1_steps) - 1:
ax.set_xlabel(x_title, **xlabel_args)
if colx == 0:
ax.set_ylabel(y_title, **ylabel_args)
del print_lock_
vary_title = '%s vary %s and %s' % (assign_alg, cfg1_name, cfg2_name)
figtitle = '%s %s %s' % (vary_title, hs.vs_str(
qcx, cx), str(hs.cx2_property(qcx, 'Notes')))
subtitle = mc3.simplify_test_uid(query_cfg.get_uid())
df2.set_figtitle(figtitle, subtitle)
df2.adjust_subplots_xylabels()
fnum += 1
viz.save_if_requested(hs, vary_title)
return fnum
def test_configurations(hs, qcx_list, test_cfg_name_list, fnum=1):
print('\n*********************\n')
print('[harn]================')
print('[harn]test_scoring(hs)')
varied_params_list = get_varried_params_list(test_cfg_name_list)
#vary_dicts = vary_dicts[0]
# query_cxs, other_cxs, notes
cfg_list = [Config.QueryConfig(hs, **_dict) for _dict in varied_params_list]
qdat = ds.QueryData()
# Preallocate test result aggregation structures
print('')
print('[harn] Testing %d different parameters' % len(cfg_list))
print('[harn] %d different chips' % len(qcx_list))
nCfg = len(cfg_list)
nQuery = len(qcx_list)
rc2_res = np.empty((nQuery, nCfg), dtype=list)
mat_list = []
c = hs.get_arg('cols', [])
for cfgx, query_cfg in enumerate(cfg_list):
print(textwrap.dedent('''
[harn]---------------')
[harn] TEST_CFG %d/%d'
[harn]---------------''' % (cfgx + 1, nCfg)))
qdat.set_cfg(query_cfg)
force_load = cfgx in c
(mat_vals, ), qx2_reslist = get_test_results(hs, qcx_list, qdat,
cfgx, nCfg, force_load)
mat_list.append(mat_vals)
for qx, reslist in enumerate(qx2_reslist):
assert len(reslist) == 1
qcx2_res = reslist[0]
assert len(qcx2_res) == 1
res = qcx2_res.values()[0]
rc2_res[qx, cfgx] = res
# Keep the best results
print('[harn] Finished testing parameters')
print('')
print('---------------------------------')
#--------------------
# Print Best Results
rank_mat = np.hstack(mat_list)
# Label the rank matrix:
_colxs = np.arange(nCfg)
lbld_mat = np.vstack([_colxs, rank_mat])
_rowxs = np.arange(nQuery + 1).reshape(nQuery + 1, 1) - 1
lbld_mat = np.hstack([_rowxs, lbld_mat])
# Build row labels
qx2_lbl = []
for qx in xrange(nQuery):
qcx = qcx_list[qx]
label = 'qx %d) q%s ' % (qx, hs.cidstr(qcx, notes=True))
qx2_lbl.append(label)
qx2_lbl = np.array(qx2_lbl)
# Build col labels
cfgx2_lbl = []
for cfgx in xrange(nCfg):
test_uid = mc3.simplify_test_uid(cfg_list[cfgx].get_uid())
test_uid = mc3.simplify_test_uid(cfg_list[cfgx].get_uid())
cfg_label = 'cfgx %3d) %s' % (cfgx, test_uid)
cfgx2_lbl.append(cfg_label)
cfgx2_lbl = np.array(cfgx2_lbl)
#------------
indent = helpers.indent
print('')
print('[harn]-------------')
print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
#------------
print('')
print('[harn]-------------')
print('[harn] configs:\n%s' % '\n'.join(cfgx2_lbl))
#------------
PRINT_ROW_SCORES = helpers.get_flag('--printrow', False)
if PRINT_ROW_SCORES:
print('')
print('[harn]-------------')
print('[harn] Scores per query')
print('[harn]-------------')
qx2_min_rank = []
qx2_argmin_rank = []
indent = helpers.indent
new_hard_qx_list = []
for qx in xrange(nQuery):
ranks = rank_mat[qx]
min_rank = ranks.min()
bestCFG_X = np.where(ranks == min_rank)[0]
qx2_min_rank.append(min_rank)
qx2_argmin_rank.append(bestCFG_X)
print('[row_score] %3d) %s' % (qx, qx2_lbl[qx]))
print('[row_score] best_rank = %d ' % min_rank)
print('[row_score] minimizing_configs = %s ' %
indent('\n'.join(cfgx2_lbl[bestCFG_X]), ' '))
if ranks.max() > 0:
new_hard_qx_list += [qx]
print('--- hard qcx_list (w.r.t these configs) ---')
new_hard_qcx_list = []
for qx in new_hard_qx_list:
# New list is in cid format instead of cx format
# because you should be copying and pasting it
notes = ' ranks = ' + str(rank_mat[qx])
qcx = qcx_list[qx]
qcid = hs.tables.cx2_cid[qcx]
new_hard_qcx_list += [(qcid, notes)]
print('\n'.join(map(repr, new_hard_qcx_list)))
else:
print('~~~~~~~~ --printrow')
#------------
def rankscore_str(thresh, nLess, total):
#helper to print rank scores of configs
percent = 100 * nLess / total
return '#ranks < %d = %d/%d = (%.1f%%) (err=%d)' % (thresh, nLess, total, percent, (total - nLess))
print('')
print('[harn]-------------')
print('[harn] Scores per config')
print('[harn]-------------')
X_list = [1, 5]
# Build a dictionary mapping X (as in #ranks < X) to a list of cfg scores
nLessX_dict = {int(X): np.zeros(nCfg) for X in iter(X_list)}
for cfgx in xrange(nCfg):
ranks = rank_mat[:, cfgx]
print('[col_score] %d) %s' % (cfgx, cfgx2_lbl[cfgx]))
for X in iter(X_list):
#nLessX_ = sum(np.bitwise_and(ranks < X, ranks >= 0))
nLessX_ = sum(np.logical_and(ranks < X, ranks >= 0))
print('[col_score] ' + rankscore_str(X, nLessX_, nQuery))
nLessX_dict[int(X)][cfgx] = nLessX_
LATEX_SUMMARY = True
if LATEX_SUMMARY:
print('--- LaTeX ---')
# Create configuration latex table
criteria_lbls = ['#ranks < %d' % X for X in X_list]
db_name = hs.get_db_name(True)
cfg_score_title = db_name + ' rank scores'
cfgscores = np.array([nLessX_dict[int(X)] for X in X_list]).T
from hscom import latex_formater as latex
replace_rowlbl = [(' *cfgx *', ' ')]
tabular_kwargs = dict(title=cfg_score_title, out_of=nQuery,
bold_best=True, replace_rowlbl=replace_rowlbl,
flip=True)
tabular_str = latex.make_score_tabular(cfgx2_lbl, criteria_lbls,
cfgscores, **tabular_kwargs)
print(tabular_str)
print('--- /LaTeX ---')
#------------
print('')
print('[harn]---------------')
print('[harn] Best configurations')
print('[harn]---------------')
best_rankscore_summary = []
for X, cfgx2_nLessX in nLessX_dict.iteritems():
max_LessX = cfgx2_nLessX.max()
bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
best_rankscore = '[best_cfg] %d config(s) scored ' % len(bestCFG_X)
best_rankscore += rankscore_str(X, max_LessX, nQuery)
best_rankcfg = indent('\n'.join(cfgx2_lbl[bestCFG_X]), ' ')
print(best_rankscore)
print(best_rankcfg)
best_rankscore_summary += [best_rankscore]
#------------
PRINT_MAT = True and (not '--noprintmat' in sys.argv)
if PRINT_MAT:
print('')
print('[harn]-------------')
print('[harn] labled rank matrix: rows=queries, cols=cfgs:\n%s' % lbld_mat)
print('[harn]-------------')
#------------
print('[col_score] --- summary ---')
print('\n'.join(best_rankscore_summary))
# Draw results
rciter = itertools.product(hs.get_arg('rows', []),
hs.get_arg('cols', []))
for r, c in rciter:
#print('viewing (r,c)=(%r,%r)' % (r,c))
res = rc2_res[r, c]
#res.printme()
res.show_topN(hs, fnum=fnum)
fnum += 1
print('--remember you have -r and -c available to you')
