def print_confusion_stats():
"""
CommandLine:
python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
"""
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMatStats: %s' % testnameid)
print('column_lbls = %r' % (column_lbls,))
#cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
#cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)
tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)
tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(tp_rawscores, 'tp_rawscores', cfgx2_lbl)
fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(fp_rawscores, 'fp_rawscores', cfgx2_lbl)
tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(tn_rawscores, 'tn_rawscores', cfgx2_lbl)
fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(fn_rawscores, 'fn_rawscores', cfgx2_lbl)
#print(tp_rawscores_str)
#print(fp_rawscores_str)
#print(tn_rawscores_str)
#print(fn_rawscores_str)
print(tp_rawscore_statstr)
print(fp_rawscore_statstr)
print(tn_rawscore_statstr)
print(fn_rawscore_statstr)
def build_commandline(e=e, **kwargs):
# Equivalent command line version of this func
import ibeis.dev
valid_e_flags = ut.flatten(
[[tup[1]] if len(tup) == 2 else [tup[1]] + tup[2]
for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS])
if e in valid_e_flags:
epref = '-e'
else:
# hack to use tf
epref = '--tf'
if dbdir is not None:
db_flag = '--dbdir'
db_value = dbdir
else:
db_flag = '--db'
db_value = db
command_parts = [
'ibeis',
epref,
e,
db_flag,
db_value,
'-a',
' '.join(a).replace('(', '\(').replace(')', '\)'),
'-t',
' '.join(t),
]
if qaid_override is not None:
command_parts.extend(
['--qaid=' + ','.join(map(str, qaid_override))])
if daid_override is not None:
command_parts.extend(
['--daid-override=' + ','.join(map(str, daid_override))])
if 'disttype' in kwargs:
command_parts.extend(
['--disttype=', ','.join(map(str, kwargs['disttype']))])
# hack parse out important args that were on command line
if 'f' in kwargs:
command_parts.extend(['-f', ' '.join(kwargs['f'])])
if 'test_cfgx_slice' in kwargs:
# very hacky, much more than checking for f
slice_ = kwargs['test_cfgx_slice']
slice_attrs = [
getattr(slice_, attr, '')
for attr in ['start', 'stop', 'step']
]
slice_attrs = ut.replace_nones(slice_attrs, '')
slicestr = ':'.join(map(str, slice_attrs))
command_parts.extend(['--test_cfgx_slice', slicestr])
command_parts.extend(['--show'])
command_line_str = ' '.join(command_parts)
# Warning, not always equivalent
print('Equivalent Command Line:')
print(command_line_str)
return command_line_str
def _make_lnbnn_scores(infr, edges):
edge_to_data = infr._get_cm_edge_data(edges)
edges = list(edge_to_data.keys())
edge_scores = list(ut.take_column(edge_to_data.values(), 'score'))
edge_scores = ut.replace_nones(edge_scores, np.nan)
edge_scores = np.array(edge_scores)
# take the inf-norm
normscores = edge_scores / vt.safe_max(edge_scores, nans=False)
return normscores
def print_confusion_stats():
"""
CommandLine:
python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
"""
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMatStats: %s' % testnameid)
print('column_lbls = %r' % (column_lbls, ))
#cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
#cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)
tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)
tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(
tp_rawscores, 'tp_rawscores', cfgx2_lbl)
fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(
fp_rawscores, 'fp_rawscores', cfgx2_lbl)
tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(
tn_rawscores, 'tn_rawscores', cfgx2_lbl)
fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(
fn_rawscores, 'fn_rawscores', cfgx2_lbl)
#print(tp_rawscores_str)
#print(fp_rawscores_str)
#print(tn_rawscores_str)
#print(fn_rawscores_str)
print(tp_rawscore_statstr)
print(fp_rawscore_statstr)
print(tn_rawscore_statstr)
print(fn_rawscore_statstr)
def apply_match_scores(infr):
"""
Applies precomputed matching scores to edges that already exist in the
graph. Typically you should run infr.apply_match_edges() before running
this.
CommandLine:
python -m wbia.algo.graph.core apply_match_scores --show
Example:
>>> # xdoctest: +REQUIRES(--slow)
>>> # ENABLE_DOCTEST
>>> from wbia.algo.graph.core import * # NOQA
>>> infr = testdata_infr('PZ_MTEST')
>>> infr.exec_matching()
>>> infr.apply_match_edges()
>>> infr.apply_match_scores()
>>> infr.get_edge_attrs('score')
"""
if infr.cm_list is None:
infr.print('apply_match_scores - no scores to apply!')
return
infr.print('apply_match_scores', 1)
edges = list(infr.graph.edges())
edge_to_data = infr._get_cm_edge_data(edges)
# Remove existing attrs
ut.nx_delete_edge_attr(infr.graph, 'score')
ut.nx_delete_edge_attr(infr.graph, 'rank')
ut.nx_delete_edge_attr(infr.graph, 'normscore')
edges = list(edge_to_data.keys())
edge_scores = list(ut.take_column(edge_to_data.values(), 'score'))
edge_scores = ut.replace_nones(edge_scores, np.nan)
edge_scores = np.array(edge_scores)
edge_ranks = np.array(ut.take_column(edge_to_data.values(), 'rank'))
# take the inf-norm
normscores = edge_scores / vt.safe_max(edge_scores, nans=False)
# Add new attrs
infr.set_edge_attrs('score', ut.dzip(edges, edge_scores))
infr.set_edge_attrs('rank', ut.dzip(edges, edge_ranks))
# Hack away zero probabilites
# probs = np.vstack([p_nomatch, p_match, p_notcomp]).T + 1e-9
# probs = vt.normalize(probs, axis=1, ord=1, out=probs)
# entropy = -(np.log2(probs) * probs).sum(axis=1)
infr.set_edge_attrs('normscore', dict(zip(edges, normscores)))
def get_annotedge_viewdist(ibs, edges):
edges = np.array(edges)
unique_annots = ibs.annots(np.unique(edges)).view()
annots1 = unique_annots.view(edges.T[0])
annots2 = unique_annots.view(edges.T[1])
view_ints1 = annots1.viewpoint_int
view_ints2 = annots2.viewpoint_int
DIST = ibs.const.VIEW.DIST
view_dists = [
DIST[tup] if tup in DIST else DIST[tup[::-1]]
for tup in zip(view_ints1, view_ints2)
]
view_dists = np.array(ut.replace_nones(view_dists, np.nan))
return view_dists
def build_commandline(e=e, **kwargs):
# Equivalent command line version of this func
import ibeis.dev
valid_e_flags = ut.flatten([[tup[1]] if len(tup) == 2 else [tup[1]] + tup[2] for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS])
if e in valid_e_flags:
epref = '-e'
else:
# hack to use tf
epref = '--tf'
command_parts = ['ibeis',
epref, e,
'--db', db,
'-a', ' '.join(a).replace('(', '\(').replace(')', '\)'),
'-t', ' '.join(t),
]
if qaid_override is not None:
command_parts.extend(['--qaid=' + ','.join(map(str, qaid_override))])
if daid_override is not None:
command_parts.extend(['--daid-override=' + ','.join(map(str, daid_override))])
if 'disttypes' in kwargs:
command_parts.extend(['--disttypes=', ','.join(map(str, kwargs['disttypes']))])
# hack parse out important args that were on command line
if 'f' in kwargs:
command_parts.extend(['-f', ' '.join(kwargs['f'])])
if 'test_cfgx_slice' in kwargs:
# very hacky, much more than checking for f
slice_ = kwargs['test_cfgx_slice']
slice_attrs = [getattr(slice_, attr, '')
for attr in ['start', 'stop', 'step']]
slice_attrs = ut.replace_nones(slice_attrs, '')
slicestr = ':'.join(map(str, slice_attrs))
command_parts.extend(['--test_cfgx_slice', slicestr])
command_parts.extend(['--show'])
command_line_str = ' '.join(command_parts)
# Warning, not always equivalent
print('Equivalent Command Line:')
print(command_line_str)
return command_line_str
def split_analysis(ibs):
"""
CommandLine:
python -m ibeis.other.dbinfo split_analysis --show
python -m ibeis split_analysis --show
python -m ibeis split_analysis --show --good
Ignore:
# mount
sshfs -o idmap=user lev:/ ~/lev
# unmount
fusermount -u ~/lev
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> dbdir = '/media/danger/GGR/GGR-IBEIS'
>>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
>>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
>>> import guitool_ibeis as gt
>>> gt.ensure_qtapp()
>>> win = split_analysis(ibs)
>>> ut.quit_if_noshow()
>>> import plottool_ibeis as pt
>>> gt.qtapp_loop(qwin=win)
>>> #ut.show_if_requested()
"""
#nid_list = ibs.get_valid_nids(filter_empty=True)
import datetime
day1 = datetime.date(2016, 1, 30)
day2 = datetime.date(2016, 1, 31)
filter_kw = {
'multiple': None,
#'view': ['right'],
#'minqual': 'good',
'is_known': True,
'min_pername': 1,
}
aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)),
})
)
aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
})
)
all_aids = aids1 + aids2
all_annots = ibs.annots(all_aids)
print('%d annots on day 1' % (len(aids1)) )
print('%d annots on day 2' % (len(aids2)) )
print('%d annots overall' % (len(all_annots)) )
print('%d names overall' % (len(ut.unique(all_annots.nids))) )
nid_list, annots_list = all_annots.group(all_annots.nids)
REVIEWED_EDGES = True
if REVIEWED_EDGES:
aids_list = [annots.aids for annots in annots_list]
#aid_pairs = [annots.get_am_aidpairs() for annots in annots_list] # Slower
aid_pairs = ibs.get_unflat_am_aidpairs(aids_list) # Faster
else:
# ALL EDGES
aid_pairs = [annots.get_aidpairs() for annots in annots_list]
speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs)
import vtool_ibeis as vt
max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list])
nan_idx = np.where(np.isnan(max_speeds))[0]
inf_idx = np.where(np.isinf(max_speeds))[0]
bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx])))
ok_idx = ut.index_complement(bad_idx, len(max_speeds))
print('#nan_idx = %r' % (len(nan_idx),))
print('#inf_idx = %r' % (len(inf_idx),))
print('#ok_idx = %r' % (len(ok_idx),))
ok_speeds = max_speeds[ok_idx]
ok_nids = ut.take(nid_list, ok_idx)
ok_annots = ut.take(annots_list, ok_idx)
sortx = np.argsort(ok_speeds)[::-1]
sorted_speeds = np.array(ut.take(ok_speeds, sortx))
sorted_annots = np.array(ut.take(ok_annots, sortx))
sorted_nids = np.array(ut.take(ok_nids, sortx)) # NOQA
sorted_speeds = np.clip(sorted_speeds, 0, 100)
#idx = vt.find_elbow_point(sorted_speeds)
#EXCESSIVE_SPEED = sorted_speeds[idx]
# http://www.infoplease.com/ipa/A0004737.html
# http://www.speedofanimals.com/animals/zebra
#ZEBRA_SPEED_MAX = 64 # km/h
#ZEBRA_SPEED_RUN = 50 # km/h
ZEBRA_SPEED_SLOW_RUN = 20 # km/h
#ZEBRA_SPEED_FAST_WALK = 10 # km/h
#ZEBRA_SPEED_WALK = 7 # km/h
MAX_SPEED = ZEBRA_SPEED_SLOW_RUN
#MAX_SPEED = ZEBRA_SPEED_WALK
#MAX_SPEED = EXCESSIVE_SPEED
flags = sorted_speeds > MAX_SPEED
flagged_ok_annots = ut.compress(sorted_annots, flags)
inf_annots = ut.take(annots_list, inf_idx)
flagged_annots = inf_annots + flagged_ok_annots
print('MAX_SPEED = %r km/h' % (MAX_SPEED,))
print('%d annots with infinite speed' % (len(inf_annots),))
print('%d annots with large speed' % (len(flagged_ok_annots),))
print('Marking all pairs of annots above the threshold as non-matching')
from ibeis.algo.graph import graph_iden
import networkx as nx
progkw = dict(freq=1, bs=True, est_window=len(flagged_annots))
bad_edges_list = []
good_edges_list = []
for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw):
edge_to_speeds = annots.get_speeds()
bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
good_edges = [edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED]
bad_edges_list.append(bad_edges)
good_edges_list.append(good_edges)
all_bad_edges = ut.flatten(bad_edges_list)
good_edges_list = ut.flatten(good_edges_list)
print('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)),))
print('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)),))
if 1:
from ibeis.viz import viz_graph2
import guitool_ibeis as gt
gt.ensure_qtapp()
if ut.get_argflag('--good'):
print('Looking at GOOD (no speed problems) edges')
aid_pairs = good_edges_list
else:
print('Looking at BAD (speed problems) edges')
aid_pairs = all_bad_edges
aids = sorted(list(set(ut.flatten(aid_pairs))))
infr = graph_iden.AnnotInference(ibs, aids, verbose=False)
infr.initialize_graph()
# Use random scores to randomize sort order
rng = np.random.RandomState(0)
scores = (-rng.rand(len(aid_pairs)) * 10).tolist()
infr.graph.add_edges_from(aid_pairs)
if True:
edge_sample_size = 250
pop_nids = ut.unique(ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs))))
sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size]
sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0))
sample_size = len(ut.unique(sorted_nids))
am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(*sorted_pairs))
flags = ut.not_list(ut.flag_None_items(am_rowids))
#am_rowids = ut.compress(am_rowids, flags)
positive_tags = ['SplitCase', 'Photobomb']
flags_list = [ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0)
for tag in positive_tags]
print('edge_case_hist: ' + ut.repr3(
['%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags)]))
is_positive = ut.or_lists(*flags_list)
num_positive = sum(ut.lmap(any, ut.group_items(is_positive, sorted_nids).values()))
pop = len(pop_nids)
print('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'),))
print('--- Sampling wrt edges ---')
print('edge_sample_size = %r' % (edge_sample_size,))
print('edge_population_size = %r' % (len(aid_pairs),))
print('num_positive_edges = %r' % (sum(is_positive)))
print('--- Sampling wrt names ---')
print('name_population_size = %r' % (pop,))
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=.95)
nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores)))
win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
init_mode=None)
win.populate_edge_model()
win.show()
return win
# Make review interface for only bad edges
infr_list = []
iter_ = list(zip(flagged_annots, bad_edges_list))
for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw):
aids = annots.aids
nids = [1] * len(aids)
infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False)
infr.initialize_graph()
infr.reset_feedback()
infr_list.append(infr)
# Check which ones are user defined as incorrect
#num_positive = 0
#for infr in infr_list:
# flag = np.any(infr.get_feedback_probs()[0] == 0)
# num_positive += flag
#print('num_positive = %r' % (num_positive,))
#pop = len(infr_list)
#print('pop = %r' % (pop,))
iter_ = list(zip(infr_list, bad_edges_list))
for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw):
flipped_edges = []
for aid1, aid2 in bad_edges:
if infr.graph.has_edge(aid1, aid2):
flipped_edges.append((aid1, aid2))
infr.add_feedback((aid1, aid2), NEGTV)
nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig')
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges})
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges})
#for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw):
# annots = ibs.annots(infr.aids)
# edge_to_speeds = annots.get_speeds()
# bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
def inference_stats(infr_list_):
relabel_stats = []
for infr in infr_list_:
num_ccs, num_inconsistent = infr.relabel_using_reviews()
state_hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, 'decision').values())
if POSTV not in state_hist:
state_hist[POSTV] = 0
hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, '_speed_split').values())
subgraphs = infr.positive_connected_compoments()
subgraph_sizes = [len(g) for g in subgraphs]
info = ut.odict([
('num_nonmatch_edges', state_hist[NEGTV]),
('num_match_edges', state_hist[POSTV]),
('frac_nonmatch_edges', state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV])),
('num_inconsistent', num_inconsistent),
('num_ccs', num_ccs),
('edges_flipped', hist.get('flip', 0)),
('edges_unchanged', hist.get('orig', 0)),
('bad_unreviewed_edges', hist.get('new', 0)),
('orig_size', len(infr.graph)),
('new_sizes', subgraph_sizes),
])
relabel_stats.append(info)
return relabel_stats
relabel_stats = inference_stats(infr_list)
print('\nAll Split Info:')
lines = []
for key in relabel_stats[0].keys():
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent'))
can_split_flags = num_incon_list == 0
print('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags)))
splittable_infrs = ut.compress(infr_list, can_split_flags)
relabel_stats = inference_stats(splittable_infrs)
print('\nTrival Split Info:')
lines = []
for key in relabel_stats[0].keys():
if key in ['num_inconsistent']:
continue
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (
key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges'))
num_nonmatch_edges = np.array(ut.take_column(relabel_stats, 'num_nonmatch_edges'))
flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3)
reasonable_infr = ut.compress(splittable_infrs, flags1)
new_sizes_list = ut.take_column(relabel_stats, 'new_sizes')
flags2 = [len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > .3
for sizes in new_sizes_list]
reasonable_infr = ut.compress(splittable_infrs, flags2)
print('#reasonable_infr = %r' % (len(reasonable_infr),))
for infr in ut.InteractiveIter(reasonable_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
rest = ~np.logical_or(flags1, flags2)
nonreasonable_infr = ut.compress(splittable_infrs, rest)
rng = np.random.RandomState(0)
random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng)
random_infr = ut.take(nonreasonable_infr, random_idx)
for infr in ut.InteractiveIter(random_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
#import scipy.stats as st
#conf_interval = .95
#st.norm.cdf(conf_interval)
# view-source:http://www.surveysystem.com/sscalc.htm
#zval = 1.96 # 95 percent confidence
#zValC = 3.8416 #
#zValC = 6.6564
#import statsmodels.stats.api as sms
#es = sms.proportion_effectsize(0.5, 0.75)
#sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1)
pop = 279
num_positive = 3
sample_size = 15
conf_level = .95
#conf_level = .99
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=.95)
pop = 279
#err_frac = .05 # 5%
err_frac = .10 # 10%
conf_level = .95
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
pop = 675
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.1)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.68, prior=.2)
vt.calc_sample_from_error_bars(.10, pop, conf_level=.68)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.68)
def _update_rows(model, rebuild_structure=True):
"""
Uses the current ider and col_sort_index to create
row_indices
"""
if VERBOSE:
print('[APIItemModel] +-----------')
print('[APIItemModel] _update_rows')
# this is not slow
#with ut.Timer('update_rows'):
#printDBG('UPDATE ROWS!')
#print('UPDATE ROWS!')
#print('num_rows=%r' % len(model.col_level_list))
#print('UPDATE model(%s) rows' % model.name)
#print('[api_model] UPDATE ROWS: %r' % (model.name,))
#print(ut.get_caller_name(range(4, 12)))
if len(model.col_level_list) == 0:
return
#old_root = model.root_node # NOQA
if rebuild_structure:
#with ut.Timer('[%s] _update_rows: %r' %
# ('cyth' if _atn.CYTHONIZED else 'pyth',
# model.name,), newline=False):
model.root_node = _atn.build_internal_structure(model)
#print('-----')
#def lazy_update_rows():
# with ut.Timer('lazy updater: %r' % (model.name,)):
# printDBG('[model] calling lazy updater: %r' % (model.name,))
# REMOVING LAZY FUNCTION BECAUSE IT MIGHT HAVE CAUSED PROBLEMS
#with ut.Timer('[%s] _update_rows2: %r' %
# ('cyth' if _atn.CYTHONIZED else 'pyth',
# model.name,), newline=False):
if VERBOSE:
print('[APIItemModel] lazy_update_rows')
model.level_index_list = []
sort_index = 0 if model.col_sort_index is None else model.col_sort_index
#print('[item_model] sort_index=%r' % (sort_index,))
children = model.root_node.get_children() # THIS IS THE LINE THAT TAKES FOREVER
id_list = [child.get_id() for child in children]
#print('ids_ generated')
nodes = []
if len(id_list) != 0:
if VERBOSE:
print('[APIItemModel] lazy_update_rows len(id_list) = %r' % (len(id_list)))
# start sort
if model.col_sort_index is not None:
type_ = model.col_type_list[sort_index]
getter = model.col_getter_list[sort_index]
values = getter(id_list)
if type_ == 'PIXMAP':
# TODO: find a better sorting metric for pixmaps
values = ut.get_list_column(values, 0)
#print('values got')
else:
type_ = int
values = id_list
reverse = model.col_sort_reverse
#ut.embed()
# <NUMPY MULTIARRAY SORT>
#with ut.embed_on_exception_context:
if type_ is float:
values = np.array(ut.replace_nones(values, np.nan))
#values = np.array(values)
values[np.isnan(values)] = -np.inf # Force nan to be the smallest number
import vtool as vt
sortx = vt.argsort_records([values, id_list], reverse=reverse)
#sorting_records = np.rec.fromarrays([values, id_list])
#sort_stride = (-reverse * 2) + 1
#sortx = sorting_records.argsort()[::sort_stride]
# </NUMPY MULTIARRAY SORT>
nodes = ut.take(children, sortx)
#sorted_pairs = sorted(zip(values, id_list, children), reverse=reverse)
#nodes = [child for (value, id_, child) in sorted_pairs]
level = model.col_level_list[sort_index]
#print("row_indices sorted")
if level == 0:
model.root_node.set_children(nodes)
# end sort
if ut.USE_ASSERT:
assert nodes is not None, 'no indices'
model.level_index_list = nodes
#if VERBOSE:
# print('[APIItemModel] lazy_update_rows emmiting _rows_updated')
# EMIT THE NUMERR OF ROWS AND THE NAME OF FOR THE VIEW TO DISPLAY
model._rows_updated.emit(model.name, len(model.level_index_list))
# lazy method didn't work. Eagerly evaluate
#lazy_update_rows()
# HACK TO MAKE SURE TREE NODES DONT DELETE THEMSELVES
#if VERBOSE:
# print('[APIItemModel] build_scope_hack_list')
# SCOPE HACK SEEMS TO HAVE NOT HALPED
#model.scope_hack_list = []
#_atn.build_scope_hack_list(model.root_node, model.scope_hack_list)
#model.lazy_updater = lazy_update_rows
#print("Rows updated")
if VERBOSE:
print('[APIItemModel] finished _update_rows')
print('[APIItemModel] L__________')
def get_query_result_info(qreq_):
"""
Helper function.
Runs queries of a specific configuration returns the best rank of each query
Args:
qaids (list) : query annotation ids
daids (list) : database annotation ids
Returns:
qx2_bestranks
CommandLine:
python -m ibeis.expt.harness --test-get_query_result_info
python -m ibeis.expt.harness --test-get_query_result_info:0
python -m ibeis.expt.harness --test-get_query_result_info:1
python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1
python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --cmd
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> qreq_ = ibeis.main_helpers.testdata_qreq_(a=['default:qindex=0:3,dindex=0:5'])
>>> #ibs = ibeis.opendb('PZ_MTEST')
>>> #qaids = ibs.get_valid_aids()[0:3]
>>> #daids = ibs.get_valid_aids()[0:5]
>>> #qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
>>> cfgres_info = get_query_result_info(qreq_)
>>> print(ut.dict_str(cfgres_info))
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb('PZ_MTEST')
>>> #cfgdict = dict(codename='vsone')
>>> # ibs.cfg.query_cfg.codename = 'vsone'
>>> qaids = ibs.get_valid_aids()[0:3]
>>> daids = ibs.get_valid_aids()[0:5]
>>> qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
>>> cfgres_info = get_query_result_info(qreq_)
>>> print(ut.dict_str(cfgres_info))
Ignore:
ibeis -e rank_cdf --db humpbacks -a default:has_any=hasnotch,mingt=2 -t default:proot=BC_DTW --show --nocache-big
ibeis -e rank_cdf --db humpbacks -a default:is_known=True,mingt=2 -t default:pipeline_root=BC_DTW
--show --debug-depc
ibeis -e rank_cdf --db humpbacks -a default:is_known=True -t default:pipeline_root=BC_DTW --qaid=1,9,15,16,18 --daid-override=1,9,15,16,18,21,22 --show --debug-depc
--clear-all-depcache
"""
try:
ibs = qreq_.ibs
except AttributeError:
ibs = qreq_.depc.controller
import vtool as vt
cm_list = qreq_.execute()
#qreq_.ibs.query_chips(qreq_=qreq_, use_bigcache=False)
qx2_cm = cm_list
qaids = qreq_.qaids
#qaids2 = [cm.qaid for cm in cm_list]
qnids = ibs.get_annot_name_rowids(qaids)
import utool
with utool.embed_on_exception_context:
unique_dnids = np.unique(ibs.get_annot_name_rowids(qreq_.daids))
unique_qnids, groupxs = vt.group_indices(qnids)
cm_group_list = ut.apply_grouping(cm_list, groupxs)
qnid2_aggnamescores = {}
qnx2_nameres_info = []
#import utool
#utool.embed()
# Ranked list aggregation-ish
nameres_info_list = []
for qnid, cm_group in zip(unique_qnids, cm_group_list):
nid2_name_score_group = [
dict([(nid, cm.name_score_list[nidx])
for nid, nidx in cm.nid2_nidx.items()]) for cm in cm_group
]
aligned_name_scores = np.array([
ut.dict_take(nid2_name_score, unique_dnids.tolist(), -np.inf)
for nid2_name_score in nid2_name_score_group
]).T
name_score_list = np.nanmax(aligned_name_scores, axis=1)
qnid2_aggnamescores[qnid] = name_score_list
# sort
sortx = name_score_list.argsort()[::-1]
sorted_namescores = name_score_list[sortx]
sorted_dnids = unique_dnids[sortx]
## infer agg name results
is_positive = sorted_dnids == qnid
is_negative = np.logical_and(~is_positive, sorted_dnids > 0)
gt_name_rank = None if not np.any(is_positive) else np.where(
is_positive)[0][0]
gf_name_rank = None if not np.any(is_negative) else np.nonzero(
is_negative)[0][0]
gt_nid = sorted_dnids[gt_name_rank]
gf_nid = sorted_dnids[gf_name_rank]
gt_name_score = sorted_namescores[gt_name_rank]
gf_name_score = sorted_namescores[gf_name_rank]
qnx2_nameres_info = {}
qnx2_nameres_info['qnid'] = qnid
qnx2_nameres_info['gt_nid'] = gt_nid
qnx2_nameres_info['gf_nid'] = gf_nid
qnx2_nameres_info['gt_name_rank'] = gt_name_rank
qnx2_nameres_info['gf_name_rank'] = gf_name_rank
qnx2_nameres_info['gt_name_score'] = gt_name_score
qnx2_nameres_info['gf_name_score'] = gf_name_score
nameres_info_list.append(qnx2_nameres_info)
nameres_info = ut.dict_stack(nameres_info_list, 'qnx2_')
qaids = qreq_.qaids
daids = qreq_.daids
qx2_gtaids = ibs.get_annot_groundtruth(qaids, daid_list=daids)
# Get the groundtruth ranks and accuracy measures
qx2_qresinfo = [get_qres_name_result_info(ibs, cm, qreq_) for cm in qx2_cm]
cfgres_info = ut.dict_stack(qx2_qresinfo, 'qx2_')
#for key in qx2_qresinfo[0].keys():
# 'qx2_' + key
# ut.get_list_column(qx2_qresinfo, key)
if False:
qx2_avepercision = np.array([
cm.get_average_percision(ibs=ibs, gt_aids=gt_aids)
for (cm, gt_aids) in zip(qx2_cm, qx2_gtaids)
])
cfgres_info['qx2_avepercision'] = qx2_avepercision
# Compute mAP score # TODO: use mAP score
# (Actually map score doesn't make much sense if using name scoring
#mAP = qx2_avepercision[~np.isnan(qx2_avepercision)].mean() # NOQA
cfgres_info['qx2_bestranks'] = ut.replace_nones(
cfgres_info['qx2_bestranks'], -1)
cfgres_info.update(nameres_info)
return cfgres_info
def get_query_result_info(qreq_):
"""
Helper function.
Runs queries of a specific configuration returns the best rank of each query
Args:
qaids (list) : query annotation ids
daids (list) : database annotation ids
Returns:
qx2_bestranks
CommandLine:
python -m ibeis.expt.harness --test-get_query_result_info
python -m ibeis.expt.harness --test-get_query_result_info:0
python -m ibeis.expt.harness --test-get_query_result_info:1
python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1
python -m ibeis.expt.harness --test-get_query_result_info:0 --db lynx -a default:qsame_imageset=True,been_adjusted=True,excluderef=True -t default:K=1 --cmd
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> qreq_ = ibeis.main_helpers.testdata_qreq_(a=['default:qindex=0:3,dindex=0:5'])
>>> #ibs = ibeis.opendb('PZ_MTEST')
>>> #qaids = ibs.get_valid_aids()[0:3]
>>> #daids = ibs.get_valid_aids()[0:5]
>>> #qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
>>> cfgres_info = get_query_result_info(qreq_)
>>> print(ut.dict_str(cfgres_info))
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.expt.harness import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb('PZ_MTEST')
>>> #cfgdict = dict(codename='vsone')
>>> # ibs.cfg.query_cfg.codename = 'vsone'
>>> qaids = ibs.get_valid_aids()[0:3]
>>> daids = ibs.get_valid_aids()[0:5]
>>> qreq_ = ibs.new_query_request(qaids, daids, verbose=True, cfgdict={})
>>> cfgres_info = get_query_result_info(qreq_)
>>> print(ut.dict_str(cfgres_info))
Ignore:
ibeis -e rank_cdf --db humpbacks -a default:has_any=hasnotch,mingt=2 -t default:proot=BC_DTW --show --nocache-big
ibeis -e rank_cdf --db humpbacks -a default:is_known=True,mingt=2 -t default:pipeline_root=BC_DTW
--show --debug-depc
ibeis -e rank_cdf --db humpbacks -a default:is_known=True -t default:pipeline_root=BC_DTW --qaid=1,9,15,16,18 --daid-override=1,9,15,16,18,21,22 --show --debug-depc
--clear-all-depcache
"""
try:
ibs = qreq_.ibs
except AttributeError:
ibs = qreq_.depc.controller
import vtool as vt
cm_list = qreq_.execute()
#qreq_.ibs.query_chips(qreq_=qreq_, use_bigcache=False)
qx2_cm = cm_list
qaids = qreq_.qaids
#qaids2 = [cm.qaid for cm in cm_list]
qnids = ibs.get_annot_name_rowids(qaids)
import utool
with utool.embed_on_exception_context:
unique_dnids = np.unique(ibs.get_annot_name_rowids(qreq_.daids))
unique_qnids, groupxs = vt.group_indices(qnids)
cm_group_list = ut.apply_grouping(cm_list, groupxs)
qnid2_aggnamescores = {}
qnx2_nameres_info = []
#import utool
#utool.embed()
# Ranked list aggregation-ish
nameres_info_list = []
for qnid, cm_group in zip(unique_qnids, cm_group_list):
nid2_name_score_group = [
dict([(nid, cm.name_score_list[nidx]) for nid, nidx in cm.nid2_nidx.items()])
for cm in cm_group
]
aligned_name_scores = np.array([
ut.dict_take(nid2_name_score, unique_dnids.tolist(), -np.inf)
for nid2_name_score in nid2_name_score_group
]).T
name_score_list = np.nanmax(aligned_name_scores, axis=1)
qnid2_aggnamescores[qnid] = name_score_list
# sort
sortx = name_score_list.argsort()[::-1]
sorted_namescores = name_score_list[sortx]
sorted_dnids = unique_dnids[sortx]
## infer agg name results
is_positive = sorted_dnids == qnid
is_negative = np.logical_and(~is_positive, sorted_dnids > 0)
gt_name_rank = None if not np.any(is_positive) else np.where(is_positive)[0][0]
gf_name_rank = None if not np.any(is_negative) else np.nonzero(is_negative)[0][0]
gt_nid = sorted_dnids[gt_name_rank]
gf_nid = sorted_dnids[gf_name_rank]
gt_name_score = sorted_namescores[gt_name_rank]
gf_name_score = sorted_namescores[gf_name_rank]
qnx2_nameres_info = {}
qnx2_nameres_info['qnid'] = qnid
qnx2_nameres_info['gt_nid'] = gt_nid
qnx2_nameres_info['gf_nid'] = gf_nid
qnx2_nameres_info['gt_name_rank'] = gt_name_rank
qnx2_nameres_info['gf_name_rank'] = gf_name_rank
qnx2_nameres_info['gt_name_score'] = gt_name_score
qnx2_nameres_info['gf_name_score'] = gf_name_score
nameres_info_list.append(qnx2_nameres_info)
nameres_info = ut.dict_stack(nameres_info_list, 'qnx2_')
qaids = qreq_.qaids
daids = qreq_.daids
qx2_gtaids = ibs.get_annot_groundtruth(qaids, daid_list=daids)
# Get the groundtruth ranks and accuracy measures
qx2_qresinfo = [get_qres_name_result_info(ibs, cm, qreq_) for cm in qx2_cm]
cfgres_info = ut.dict_stack(qx2_qresinfo, 'qx2_')
#for key in qx2_qresinfo[0].keys():
# 'qx2_' + key
# ut.get_list_column(qx2_qresinfo, key)
if False:
qx2_avepercision = np.array(
[cm.get_average_percision(ibs=ibs, gt_aids=gt_aids) for
(cm, gt_aids) in zip(qx2_cm, qx2_gtaids)])
cfgres_info['qx2_avepercision'] = qx2_avepercision
# Compute mAP score # TODO: use mAP score
# (Actually map score doesn't make much sense if using name scoring
#mAP = qx2_avepercision[~np.isnan(qx2_avepercision)].mean() # NOQA
cfgres_info['qx2_bestranks'] = ut.replace_nones(cfgres_info['qx2_bestranks'] , -1)
cfgres_info.update(nameres_info)
return cfgres_info
def _update_rows(model, rebuild_structure=True):
"""
Uses the current ider and col_sort_index to create
row_indices
"""
# with ut.Timer('[gt] update_rows (%s)' % (model.name,)):
if True:
# flag = model.blockSignals(True)
if VERBOSE_MODEL:
logger.info('[APIItemModel] +-----------')
logger.info('[APIItemModel] _update_rows')
# this is not slow
# logger.info('UPDATE ROWS!')
if len(model.col_level_list) == 0:
return
# old_root = model.root_node # NOQA
if rebuild_structure:
# logger.info('Rebuilging api_item_model internal structure')
model.beginResetModel(
) # I think this is preventing a segfault
model.root_node = _atn.build_internal_structure(model)
model.endResetModel()
if VERBOSE_MODEL:
logger.info('[APIItemModel] lazy_update_rows')
model.level_index_list = []
sort_index = 0 if model.col_sort_index is None else model.col_sort_index
children = (model.root_node.get_children()
) # THIS IS THE LINE THAT TAKES FOREVER
id_list = [child.get_id() for child in children]
# logger.info('ids_ generated')
nodes = []
if len(id_list) != 0:
if VERBOSE_MODEL:
logger.info(
'[APIItemModel] lazy_update_rows len(id_list) = %r' %
(len(id_list)))
# start sort
if model.col_sort_index is not None:
type_ = model.col_type_list[sort_index]
getter = model.col_getter_list[sort_index]
values = getter(id_list)
if type_ == 'PIXMAP':
# TODO: find a better sorting metric for pixmaps
values = ut.get_list_column(values, 0)
else:
type_ = int
values = id_list
reverse = model.col_sort_reverse
# <NUMPY MULTIARRAY SORT>
if True:
if values is None:
logger.info('SORTING VALUES IS NONE. VERY WEIRD')
if type_ is float:
values = np.array(ut.replace_nones(values, np.nan))
# Force nan to be the smallest number
values[np.isnan(values)] = -np.inf
elif type_ is str:
values = ut.replace_nones(values, '')
import vtool as vt
sortx = vt.argsort_records([values, id_list],
reverse=reverse)
# </NUMPY MULTIARRAY SORT>
nodes = ut.take(children, sortx)
level = model.col_level_list[sort_index]
if level == 0:
model.root_node.set_children(nodes)
# end sort
if ut.USE_ASSERT:
assert nodes is not None, 'no indices'
model.level_index_list = nodes
# Book keeping for lazy loading rows
model.num_rows_total = len(model.level_index_list)
# model.num_cols_total = len(model.col_name_list)
model.num_cols_loaded = 0
if model.lazy_rows:
model.num_rows_loaded = 0
else:
model.num_rows_loaded = model.num_rows_total
# emit the numerr of rows and the name of for the view to display
# model.blockSignals(flag)
model._rows_updated.emit(model.name, model.num_rows_total)
if VERBOSE_MODEL:
logger.info('[APIItemModel] finished _update_rows')
logger.info('[APIItemModel] L__________')
def print_results(ibs, testres):
"""
Prints results from an experiment harness run.
Rows store different qaids (query annotation ids)
Cols store different configurations (algorithm parameters)
Args:
ibs (IBEISController): ibeis controller object
testres (test_result.TestResult):
CommandLine:
python dev.py -e print --db PZ_MTEST -a default:dpername=1,qpername=[1,2] -t default:fg_on=False
python dev.py -e print -t best --db seals2 --allgt --vz
python dev.py -e print --db PZ_MTEST --allgt -t custom --print-confusion-stats
python dev.py -e print --db PZ_MTEST --allgt --noqcache --index 0:10:2 -t custom:rrvsone_on=True --print-confusion-stats
python dev.py -e print --db PZ_MTEST --allgt --noqcache --qaid4 -t custom:rrvsone_on=True --print-confusion-stats
python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
python -m ibeis --tf print_results --db PZ_MTEST -a default -t default:lnbnn_on=True default:lnbnn_on=False,bar_l2_on=True default:lnbnn_on=False,normonly_on=True
CommandLine:
python -m ibeis.expt.experiment_printres --test-print_results
utprof.py -m ibeis.expt.experiment_printres --test-print_results
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_printres import * # NOQA
>>> from ibeis.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(
>>> 'PZ_MTEST', a='default:dpername=1,qpername=[1,2]', t='default:fg_on=False')
>>> result = print_results(ibs, testres)
>>> print(result)
"""
(cfg_list, cfgx2_cfgresinfo, testnameid, cfgx2_lbl, cfgx2_qreq_) = ut.dict_take(
testres.__dict__, ['cfg_list', 'cfgx2_cfgresinfo', 'testnameid', 'cfgx2_lbl', 'cfgx2_qreq_'])
# cfgx2_cfgresinfo is a list of dicts of lists
# Parse result info out of the lists
cfgx2_nextbestranks = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_next_bestranks')
cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
#cfgx2_aveprecs = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_avepercision')
cfgx2_scorediffs = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_scorediff')
#cfgx2_gt_raw_score = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
column_lbls = [ut.remove_chars(ut.remove_vowels(lbl), [' ', ','])
for lbl in cfgx2_lbl]
scorediffs_mat = np.array(ut.replace_nones(cfgx2_scorediffs, np.nan))
print(' --- PRINT RESULTS ---')
print(' use --rank-lt-list=1,5 to specify X_LIST')
if True:
# Num of ranks less than to score
X_LIST = testres.get_X_LIST()
#X_LIST = [1, 5]
#nConfig = len(cfg_list)
#nQuery = len(testres.qaids)
cfgx2_nQuery = list(map(len, testres.cfgx2_qaids))
#cfgx2_qx2_ranks = testres.get_infoprop_list('qx2_bestranks')
#--------------------
# A positive scorediff indicates the groundtruth was better than the
# groundfalse scores
istrue_list = [scorediff > 0 for scorediff in scorediffs_mat]
isfalse_list = [~istrue for istrue in istrue_list]
#------------
# Build Colscore
nLessX_dict = testres.get_nLessX_dict()
#------------
best_rankscore_summary = []
#to_intersect_list = []
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
max_nLessX = cfgx2_nLessX.max()
bestX_cfgx_list = np.where(cfgx2_nLessX == max_nLessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestX_cfgx_list)
# FIXME
best_rankscore += rankscore_str(X, max_nLessX, cfgx2_nQuery[bestX_cfgx_list[0]])
best_rankscore_summary += [best_rankscore]
#to_intersect_list.append(ut.take(cfgx2_lbl, max_nLessX))
#intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
#for ix in range(1, len(to_intersect_list)):
# intersected = np.intersect1d(intersected, to_intersect_list[ix])
#if False:
# #gt_raw_score_mat = np.vstack(cfgx2_gt_raw_score).T
# #rank_mat = testres.get_rank_mat()
# #------------
# # Build row lbls
# if False:
# qx2_lbl = np.array([
# 'qx=%d) q%s ' % (qx, ibsfuncs.aidstr(testres.qaids[qx], ibs=ibs, notes=True))
# for qx in range(nQuery)])
# #------------
# # Build Colscore and hard cases
# if False:
# qx2_min_rank = []
# qx2_argmin_rank = []
# new_hard_qaids = []
# new_hardtup_list = []
# for qx in range(nQuery):
# ranks = rank_mat[qx]
# valid_ranks = ranks[ranks >= 0]
# min_rank = ranks.min() if len(valid_ranks) > 0 else -3
# bestCFG_X = np.where(ranks == min_rank)[0]
# qx2_min_rank.append(min_rank)
# # Find the best rank over all configurations
# qx2_argmin_rank.append(bestCFG_X)
#@ut.memoize
#def get_new_hard_qx_list(testres):
# """ Mark any query as hard if it didnt get everything correct """
# rank_mat = testres.get_rank_mat()
# is_new_hard_list = rank_mat.max(axis=1) > 0
# new_hard_qx_list = np.where(is_new_hard_list)[0]
# return new_hard_qx_list
# new_hard_qx_list = testres.get_new_hard_qx_list()
# for qx in new_hard_qx_list:
# # New list is in aid format instead of cx format
# # because you should be copying and pasting it
# notes = ' ranks = ' + str(rank_mat[qx])
# qaid = testres.qaids[qx]
# name = ibs.get_annot_names(qaid)
# new_hardtup_list += [(qaid, name + " - " + notes)]
# new_hard_qaids += [qaid]
@ut.argv_flag_dec
def intersect_hack():
failed = testres.rank_mat > 0
colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
#failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
#failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)
qaids_expt = failed_col1_only_aids
gt_avl_aids1 = ut.flatten(gt_aids1)
gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
#jsontext = ut.to_json({
# 'qaids': list(qaids_expt),
# 'dinclude_aids1': list(gt_aids_expt1),
# 'dinclude_aids2': list(gt_aids_expt2),
#})
#annotation_configs.varysize_pzm
#from ibeis.expt import annotation_configs
acfg = testres.acfg_list[0]
import copy
acfg1 = copy.deepcopy(acfg)
acfg2 = copy.deepcopy(acfg)
acfg1['qcfg']['min_pername'] = None
acfg2['qcfg']['min_pername'] = None
acfg1['dcfg']['min_pername'] = None
acfg2['dcfg']['min_gt_per_name'] = None
acfg1['qcfg']['default_aids'] = qaids_expt
acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
acfg2['qcfg']['default_aids'] = qaids_expt
acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2
from ibeis.init import filter_annots
from ibeis.expt import experiment_helpers
annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)
acfg_name_list = dict( # NOQA
acfg_list=[acfg1, acfg2],
expanded_aids_list=[annots1, annots2],
)
test_cfg_name_list = ['candidacy_k']
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)
t1, t2 = testres_list # NOQA
#ut.embed()
#intersect_hack()
#@ut.argv_flag_dec
#def print_rowlbl():
# print('=====================')
# print('[harn] Row/Query Labels: %s' % testnameid)
# print('=====================')
# print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
#print_rowlbl()
#------------
@ut.argv_flag_dec
def print_collbl():
print('=====================')
print('[harn] Col/Config Labels: %s' % testnameid)
print('=====================')
enum_cfgx2_lbl = ['%2d) %s' % (count, cfglbl)
for count, cfglbl in enumerate(cfgx2_lbl)]
print('[harn] cfglbl:\n%s' % '\n'.join(enum_cfgx2_lbl))
print_collbl()
#------------
@ut.argv_flag_dec
def print_cfgstr():
print('=====================')
print('[harn] Config Strings: %s' % testnameid)
print('=====================')
cfgstr_list = [query_cfg.get_cfgstr() for query_cfg in cfg_list]
enum_cfgstr_list = ['%2d) %s' % (count, cfgstr)
for count, cfgstr in enumerate(cfgstr_list)]
print('\n[harn] cfgstr:\n%s' % '\n'.join(enum_cfgstr_list))
print_cfgstr()
#------------
#@ut.argv_flag_dec
#def print_rowscore():
# print('=======================')
# print('[harn] Scores per Query: %s' % testnameid)
# print('=======================')
# for qx in range(nQuery):
# bestCFG_X = qx2_argmin_rank[qx]
# min_rank = qx2_min_rank[qx]
# minimizing_cfg_str = ut.indentjoin(cfgx2_lbl[bestCFG_X], '\n * ')
# #minimizing_cfg_str = str(bestCFG_X)
# print('-------')
# print(qx2_lbl[qx])
# print(' best_rank = %d ' % min_rank)
# if len(cfgx2_lbl) != 1:
# print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
#print_rowscore()
#------------
#@ut.argv_flag_dec
#def print_row_ave_precision():
# print('=======================')
# print('[harn] Scores per Query: %s' % testnameid)
# print('=======================')
# for qx in range(nQuery):
# aveprecs = ', '.join(['%.2f' % (aveprecs[qx],) for aveprecs in cfgx2_aveprecs])
# print('-------')
# print(qx2_lbl[qx])
# print(' aveprecs = %s ' % aveprecs)
#print_row_ave_precision()
##------------
#@ut.argv_flag_dec
#def print_hardcase():
# print('--- hard new_hardtup_list (w.r.t these configs): %s' % testnameid)
# print('\n'.join(map(repr, new_hardtup_list)))
# print('There are %d hard cases ' % len(new_hardtup_list))
# aid_list = [aid_notes[0] for aid_notes in new_hardtup_list]
# name_list = ibs.get_annot_names(aid_list)
# name_set = set(name_list)
# print(sorted(aid_list))
# print('Names: %r' % (name_set,))
#print_hardcase()
#default=not ut.get_argflag('--allhard'))
#------------
#@ut.argv_flag_dec
#def echo_hardcase():
# print('--- hardcase commandline: %s' % testnameid)
# # Show index for current query where hardids reside
# #print('--index ' + (' '.join(map(str, new_hard_qx_list))))
# #print('--take new_hard_qx_list')
# #hardaids_str = ' '.join(map(str, [' ', '--qaid'] + new_hard_qaids))
# hardaids_str = ' '.join(map(str, [' ', '--set-aids-as-hard'] + new_hard_qaids))
# print(hardaids_str)
##echo_hardcase(default=not ut.get_argflag('--allhard'))
#echo_hardcase()
#@ut.argv_flag_dec
#def print_bestcfg():
# print('==========================')
# print('[harn] Best Configurations: %s' % testnameid)
# print('==========================')
# # print each configs scores less than X=thresh
# for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
# max_LessX = cfgx2_nLessX.max()
# bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
# best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
# best_rankscore += rankscore_str(X, max_LessX, nQuery)
# cfglbl_list = cfgx2_lbl[bestCFG_X]
# best_rankcfg = format_cfgstr_list(cfglbl_list)
# #indent('\n'.join(cfgstr_list), ' ')
# print(best_rankscore)
# print(best_rankcfg)
# print('[cfg*] %d cfg(s) are the best of %d total cfgs' % (len(intersected), nConfig))
# print(format_cfgstr_list(intersected))
#print_bestcfg()
#------------
#@ut.argv_flag_dec
#def print_gtscore():
# # Prints best ranks
# print('gtscore_mat: %s' % testnameid)
# print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
# header = (' labled rank matrix: rows=queries, cols=cfgs:')
# print('\n'.join(cfgx2_lbl))
# column_list = gt_raw_score_mat.T
# print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
# column_lbls=column_lbls, header=header,
# transpose=False,
# use_lbl_width=len(cfgx2_lbl) < 5))
#print_gtscore()
#------------
#@ut.argv_flag_dec
#def print_best_rankmat():
# # Prints best ranks
# print('-------------')
# print('RankMat: %s' % testnameid)
# print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
# header = (' labled rank matrix: rows=queries, cols=cfgs:')
# print('\n'.join(cfgx2_lbl))
# column_list = rank_mat.T
# print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
# column_lbls=column_lbls, header=header,
# transpose=False,
# use_lbl_width=len(cfgx2_lbl) < 5))
#print_best_rankmat()
#@ut.argv_flag_dec
#def print_diffmat():
# # score differences over configs
# print('-------------')
# print('Diffmat: %s' % testnameid)
# diff_matstr = get_diffmat_str(rank_mat, testres.qaids, nConfig)
# print(diff_matstr)
#print_diffmat()
#@ut.argv_flag_dec
#def print_rankhist_time():
# print('A rank histogram is a dictionary. '
# 'The keys denote the range of the ranks that the values fall in')
# # TODO: rectify this code with other hist code
# config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
# config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()
# _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
# for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
# #full_cfgstr = testres.cfgx2_qreq_[cfgx].get_full_cfgstr()
# #ut.print_dict(ut.dict_hist(ranks), 'rank histogram', sorted_=True)
# # find the qxs that belong to each bin
# aid_list1 = testres.qaids
# aid_list2 = qx2_gt_aid
# ibs.assert_valid_aids(aid_list1)
# ibs.assert_valid_aids(aid_list2)
# timedelta_list = ibs.get_annot_pair_timdelta(aid_list1, aid_list2)
# #timedelta_str_list = [ut.get_posix_timedelta_str2(delta)
# # for delta in timedelta_list]
# bin_edges = testres.get_rank_histogram_bin_edges()
# timedelta_groups = ut.dict_take(ut.group_items(timedelta_list, config_binxs), np.arange(len(bin_edges)), [])
# timedelta_stats = [ut.get_stats(deltas, use_nan=True, datacast=ut.get_posix_timedelta_str2) for deltas in timedelta_groups]
# print('Time statistics for each rank range:')
# print(ut.dict_str(dict(zip(bin_edges, timedelta_stats)), sorted_=True))
#print_rankhist_time()
#@ut.argv_flag_dec
#def print_rankhist():
# print('A rank histogram is a dictionary. '
# 'The keys denote the range of the ranks that the values fall in')
# # TODO: rectify this code with other hist code
# config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
# config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()
# _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
# for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
# print('Frequency of rank ranges:')
# ut.print_dict(agg_hist_dict, 'agg rank histogram', sorted_=True)
#print_rankhist()
#------------
# Print summary
#print(' --- SUMMARY ---')
#------------
#@ut.argv_flag_dec
#def print_colmap():
# print('==================')
# print('[harn] mAP per Config: %s (sorted by mAP)' % testnameid)
# print('==================')
# cfgx2_mAP = np.array([aveprec_list.mean() for aveprec_list in cfgx2_aveprecs])
# sortx = cfgx2_mAP.argsort()
# for cfgx in sortx:
# print('[mAP] cfgx=%r) mAP=%.3f -- %s' % (cfgx, cfgx2_mAP[cfgx], cfgx2_lbl[cfgx]))
# #print('L___ Scores per Config ___')
#print_colmap()
#------------
@ut.argv_flag_dec_true
def print_colscore():
print('==================')
print('[harn] Scores per Config: %s' % testnameid)
print('==================')
#for cfgx in range(nConfig):
# print('[score] %s' % (cfgx2_lbl[cfgx]))
# for X in X_LIST:
# nLessX_ = nLessX_dict[int(X)][cfgx]
# print(' ' + rankscore_str(X, nLessX_, nQuery))
print('\n[harn] ... sorted scores')
for X in X_LIST:
print('\n[harn] Sorted #ranks < %r scores' % (X))
sortx = np.array(nLessX_dict[int(X)]).argsort()
#frac_list = (nLessX_dict[int(X)] / cfgx2_nQuery)[:, None]
#print('cfgx2_nQuery = %r' % (cfgx2_nQuery,))
#print('frac_list = %r' % (frac_list,))
#print('Pairwise Difference: ' + str(ut.safe_pdist(frac_list, metric=ut.absdiff)))
for cfgx in sortx:
nLessX_ = nLessX_dict[int(X)][cfgx]
rankstr = rankscore_str(X, nLessX_, cfgx2_nQuery[cfgx], withlbl=False)
print('[score] %s --- %s' % (rankstr, cfgx2_lbl[cfgx]))
print_colscore()
#------------
ut.argv_flag_dec(print_latexsum)(ibs, testres)
@ut.argv_flag_dec
def print_next_rankmat():
# Prints nextbest ranks
print('-------------')
print('NextRankMat: %s' % testnameid)
header = (' top false rank matrix: rows=queries, cols=cfgs:')
print('\n'.join(cfgx2_lbl))
column_list = cfgx2_nextbestranks
print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
column_lbls=column_lbls, header=header,
transpose=False,
use_lbl_width=len(cfgx2_lbl) < 5))
print_next_rankmat()
#------------
@ut.argv_flag_dec
def print_scorediff_mat():
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMat: %s' % testnameid)
header = (' score difference between top true and top false: rows=queries, cols=cfgs:')
print('\n'.join(cfgx2_lbl))
column_list = cfgx2_scorediffs
column_type = [float] * len(column_list)
print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
column_lbls=column_lbls,
column_type=column_type,
header=header,
transpose=False,
use_lbl_width=len(cfgx2_lbl) < 5))
print_scorediff_mat(alias_flags=['--sdm'])
#------------
def jagged_stats_info(arr_, lbl, col_lbls):
arr = ut.recursive_replace(arr_, np.inf, np.nan)
# Treat infinite as nan
stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls)
sel_col_lbls = ut.take(col_lbls, sel_indices)
statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True)
stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str
return stat_str, sel_stat_str
@ut.argv_flag_dec
def print_confusion_stats():
"""
CommandLine:
python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
"""
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMatStats: %s' % testnameid)
print('column_lbls = %r' % (column_lbls,))
#cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
#cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)
tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)
tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(tp_rawscores, 'tp_rawscores', cfgx2_lbl)
fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(fp_rawscores, 'fp_rawscores', cfgx2_lbl)
tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(tn_rawscores, 'tn_rawscores', cfgx2_lbl)
fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(fn_rawscores, 'fn_rawscores', cfgx2_lbl)
#print(tp_rawscores_str)
#print(fp_rawscores_str)
#print(tn_rawscores_str)
#print(fn_rawscores_str)
print(tp_rawscore_statstr)
print(fp_rawscore_statstr)
print(tn_rawscore_statstr)
print(fn_rawscore_statstr)
print_confusion_stats(alias_flags=['--cs'])
ut.argv_flag_dec_true(testres.print_percent_identification_success)()
sumstrs = []
sumstrs.append('')
sumstrs.append('||===========================')
sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
sumstrs.append('||---------------------------')
sumstrs.append(ut.joins('\n|| ', best_rankscore_summary))
sumstrs.append('||===========================')
summary_str = '\n' + '\n'.join(sumstrs) + '\n'
#print(summary_str)
ut.colorprint(summary_str, 'blue')
print('To enable all printouts add --print-all to the commandline')
def print_results(ibs, testres):
"""
Prints results from an experiment harness run.
Rows store different qaids (query annotation ids)
Cols store different configurations (algorithm parameters)
Args:
ibs (IBEISController): ibeis controller object
testres (test_result.TestResult):
CommandLine:
python dev.py -e print --db PZ_MTEST -a default:dpername=1,qpername=[1,2] -t default:fg_on=False
python dev.py -e print -t best --db seals2 --allgt --vz
python dev.py -e print --db PZ_MTEST --allgt -t custom --print-confusion-stats
python dev.py -e print --db PZ_MTEST --allgt --noqcache --index 0:10:2 -t custom:rrvsone_on=True --print-confusion-stats
python dev.py -e print --db PZ_MTEST --allgt --noqcache --qaid4 -t custom:rrvsone_on=True --print-confusion-stats
python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
python -m ibeis --tf print_results -t default --db PZ_MTEST -a ctrl
python -m ibeis --tf print_results --db PZ_MTEST -a default -t default:lnbnn_on=True default:lnbnn_on=False,bar_l2_on=True default:lnbnn_on=False,normonly_on=True
CommandLine:
python -m ibeis.expt.experiment_printres --test-print_results
utprof.py -m ibeis.expt.experiment_printres --test-print_results
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.expt.experiment_printres import * # NOQA
>>> from ibeis.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts(
>>> 'PZ_MTEST', a='default:dpername=1,qpername=[1,2]', t='default:fg_on=False')
>>> result = print_results(ibs, testres)
>>> print(result)
"""
(cfg_list, cfgx2_cfgresinfo, testnameid, cfgx2_lbl,
cfgx2_qreq_) = ut.dict_take(testres.__dict__, [
'cfg_list', 'cfgx2_cfgresinfo', 'testnameid', 'cfgx2_lbl',
'cfgx2_qreq_'
])
# cfgx2_cfgresinfo is a list of dicts of lists
# Parse result info out of the lists
cfgx2_nextbestranks = ut.get_list_column(cfgx2_cfgresinfo,
'qx2_next_bestranks')
cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo,
'qx2_gt_raw_score')
cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo,
'qx2_gf_raw_score')
#cfgx2_aveprecs = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_avepercision')
cfgx2_scorediffs = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_scorediff')
#cfgx2_gt_raw_score = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
column_lbls = [
ut.remove_chars(ut.remove_vowels(lbl), [' ', ',']) for lbl in cfgx2_lbl
]
scorediffs_mat = np.array(ut.replace_nones(cfgx2_scorediffs, np.nan))
print(' --- PRINT RESULTS ---')
print(' use --rank-lt-list=1,5 to specify X_LIST')
if True:
# Num of ranks less than to score
X_LIST = testres.get_X_LIST()
#X_LIST = [1, 5]
#nConfig = len(cfg_list)
#nQuery = len(testres.qaids)
cfgx2_nQuery = list(map(len, testres.cfgx2_qaids))
#cfgx2_qx2_ranks = testres.get_infoprop_list('qx2_bestranks')
#--------------------
# A positive scorediff indicates the groundtruth was better than the
# groundfalse scores
istrue_list = [scorediff > 0 for scorediff in scorediffs_mat]
isfalse_list = [~istrue for istrue in istrue_list]
#------------
# Build Colscore
nLessX_dict = testres.get_nLessX_dict()
#------------
best_rankscore_summary = []
#to_intersect_list = []
# print each configs scores less than X=thresh
for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
max_nLessX = cfgx2_nLessX.max()
bestX_cfgx_list = np.where(cfgx2_nLessX == max_nLessX)[0]
best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestX_cfgx_list)
# FIXME
best_rankscore += rankscore_str(X, max_nLessX,
cfgx2_nQuery[bestX_cfgx_list[0]])
best_rankscore_summary += [best_rankscore]
#to_intersect_list.append(ut.take(cfgx2_lbl, max_nLessX))
#intersected = to_intersect_list[0] if len(to_intersect_list) > 0 else []
#for ix in range(1, len(to_intersect_list)):
# intersected = np.intersect1d(intersected, to_intersect_list[ix])
#if False:
# #gt_raw_score_mat = np.vstack(cfgx2_gt_raw_score).T
# #rank_mat = testres.get_rank_mat()
# #------------
# # Build row lbls
# if False:
# qx2_lbl = np.array([
# 'qx=%d) q%s ' % (qx, ibsfuncs.aidstr(testres.qaids[qx], ibs=ibs, notes=True))
# for qx in range(nQuery)])
# #------------
# # Build Colscore and hard cases
# if False:
# qx2_min_rank = []
# qx2_argmin_rank = []
# new_hard_qaids = []
# new_hardtup_list = []
# for qx in range(nQuery):
# ranks = rank_mat[qx]
# valid_ranks = ranks[ranks >= 0]
# min_rank = ranks.min() if len(valid_ranks) > 0 else -3
# bestCFG_X = np.where(ranks == min_rank)[0]
# qx2_min_rank.append(min_rank)
# # Find the best rank over all configurations
# qx2_argmin_rank.append(bestCFG_X)
#@ut.memoize
#def get_new_hard_qx_list(testres):
# """ Mark any query as hard if it didnt get everything correct """
# rank_mat = testres.get_rank_mat()
# is_new_hard_list = rank_mat.max(axis=1) > 0
# new_hard_qx_list = np.where(is_new_hard_list)[0]
# return new_hard_qx_list
# new_hard_qx_list = testres.get_new_hard_qx_list()
# for qx in new_hard_qx_list:
# # New list is in aid format instead of cx format
# # because you should be copying and pasting it
# notes = ' ranks = ' + str(rank_mat[qx])
# qaid = testres.qaids[qx]
# name = ibs.get_annot_names(qaid)
# new_hardtup_list += [(qaid, name + " - " + notes)]
# new_hard_qaids += [qaid]
@ut.argv_flag_dec
def intersect_hack():
failed = testres.rank_mat > 0
colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
#failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
#failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
gt_aids1 = ibs.get_annot_groundtruth(
failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
gt_aids2 = ibs.get_annot_groundtruth(
failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)
qaids_expt = failed_col1_only_aids
gt_avl_aids1 = ut.flatten(gt_aids1)
gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
#jsontext = ut.to_json({
# 'qaids': list(qaids_expt),
# 'dinclude_aids1': list(gt_aids_expt1),
# 'dinclude_aids2': list(gt_aids_expt2),
#})
#annotation_configs.varysize_pzm
#from ibeis.expt import annotation_configs
acfg = testres.acfg_list[0]
import copy
acfg1 = copy.deepcopy(acfg)
acfg2 = copy.deepcopy(acfg)
acfg1['qcfg']['min_pername'] = None
acfg2['qcfg']['min_pername'] = None
acfg1['dcfg']['min_pername'] = None
acfg2['dcfg']['min_gt_per_name'] = None
acfg1['qcfg']['default_aids'] = qaids_expt
acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
acfg2['qcfg']['default_aids'] = qaids_expt
acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2
from ibeis.init import filter_annots
from ibeis.expt import experiment_helpers
annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)
acfg_name_list = dict( # NOQA
acfg_list=[acfg1, acfg2],
expanded_aids_list=[annots1, annots2],
)
test_cfg_name_list = ['candidacy_k']
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(
test_cfg_name_list, ibs=ibs)
t1, t2 = testres_list # NOQA
#ut.embed()
#intersect_hack()
#@ut.argv_flag_dec
#def print_rowlbl():
# print('=====================')
# print('[harn] Row/Query Labels: %s' % testnameid)
# print('=====================')
# print('[harn] queries:\n%s' % '\n'.join(qx2_lbl))
#print_rowlbl()
#------------
@ut.argv_flag_dec
def print_collbl():
print('=====================')
print('[harn] Col/Config Labels: %s' % testnameid)
print('=====================')
enum_cfgx2_lbl = [
'%2d) %s' % (count, cfglbl)
for count, cfglbl in enumerate(cfgx2_lbl)
]
print('[harn] cfglbl:\n%s' % '\n'.join(enum_cfgx2_lbl))
print_collbl()
#------------
@ut.argv_flag_dec
def print_cfgstr():
print('=====================')
print('[harn] Config Strings: %s' % testnameid)
print('=====================')
cfgstr_list = [query_cfg.get_cfgstr() for query_cfg in cfg_list]
enum_cfgstr_list = [
'%2d) %s' % (count, cfgstr)
for count, cfgstr in enumerate(cfgstr_list)
]
print('\n[harn] cfgstr:\n%s' % '\n'.join(enum_cfgstr_list))
print_cfgstr()
#------------
#@ut.argv_flag_dec
#def print_rowscore():
# print('=======================')
# print('[harn] Scores per Query: %s' % testnameid)
# print('=======================')
# for qx in range(nQuery):
# bestCFG_X = qx2_argmin_rank[qx]
# min_rank = qx2_min_rank[qx]
# minimizing_cfg_str = ut.indentjoin(cfgx2_lbl[bestCFG_X], '\n * ')
# #minimizing_cfg_str = str(bestCFG_X)
# print('-------')
# print(qx2_lbl[qx])
# print(' best_rank = %d ' % min_rank)
# if len(cfgx2_lbl) != 1:
# print(' minimizing_cfg_x\'s = %s ' % minimizing_cfg_str)
#print_rowscore()
#------------
#@ut.argv_flag_dec
#def print_row_ave_precision():
# print('=======================')
# print('[harn] Scores per Query: %s' % testnameid)
# print('=======================')
# for qx in range(nQuery):
# aveprecs = ', '.join(['%.2f' % (aveprecs[qx],) for aveprecs in cfgx2_aveprecs])
# print('-------')
# print(qx2_lbl[qx])
# print(' aveprecs = %s ' % aveprecs)
#print_row_ave_precision()
##------------
#@ut.argv_flag_dec
#def print_hardcase():
# print('--- hard new_hardtup_list (w.r.t these configs): %s' % testnameid)
# print('\n'.join(map(repr, new_hardtup_list)))
# print('There are %d hard cases ' % len(new_hardtup_list))
# aid_list = [aid_notes[0] for aid_notes in new_hardtup_list]
# name_list = ibs.get_annot_names(aid_list)
# name_set = set(name_list)
# print(sorted(aid_list))
# print('Names: %r' % (name_set,))
#print_hardcase()
#default=not ut.get_argflag('--allhard'))
#------------
#@ut.argv_flag_dec
#def echo_hardcase():
# print('--- hardcase commandline: %s' % testnameid)
# # Show index for current query where hardids reside
# #print('--index ' + (' '.join(map(str, new_hard_qx_list))))
# #print('--take new_hard_qx_list')
# #hardaids_str = ' '.join(map(str, [' ', '--qaid'] + new_hard_qaids))
# hardaids_str = ' '.join(map(str, [' ', '--set-aids-as-hard'] + new_hard_qaids))
# print(hardaids_str)
##echo_hardcase(default=not ut.get_argflag('--allhard'))
#echo_hardcase()
#@ut.argv_flag_dec
#def print_bestcfg():
# print('==========================')
# print('[harn] Best Configurations: %s' % testnameid)
# print('==========================')
# # print each configs scores less than X=thresh
# for X, cfgx2_nLessX in six.iteritems(nLessX_dict):
# max_LessX = cfgx2_nLessX.max()
# bestCFG_X = np.where(cfgx2_nLessX == max_LessX)[0]
# best_rankscore = '[cfg*] %d cfg(s) scored ' % len(bestCFG_X)
# best_rankscore += rankscore_str(X, max_LessX, nQuery)
# cfglbl_list = cfgx2_lbl[bestCFG_X]
# best_rankcfg = format_cfgstr_list(cfglbl_list)
# #indent('\n'.join(cfgstr_list), ' ')
# print(best_rankscore)
# print(best_rankcfg)
# print('[cfg*] %d cfg(s) are the best of %d total cfgs' % (len(intersected), nConfig))
# print(format_cfgstr_list(intersected))
#print_bestcfg()
#------------
#@ut.argv_flag_dec
#def print_gtscore():
# # Prints best ranks
# print('gtscore_mat: %s' % testnameid)
# print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
# header = (' labled rank matrix: rows=queries, cols=cfgs:')
# print('\n'.join(cfgx2_lbl))
# column_list = gt_raw_score_mat.T
# print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
# column_lbls=column_lbls, header=header,
# transpose=False,
# use_lbl_width=len(cfgx2_lbl) < 5))
#print_gtscore()
#------------
#@ut.argv_flag_dec
#def print_best_rankmat():
# # Prints best ranks
# print('-------------')
# print('RankMat: %s' % testnameid)
# print(' nRows=%r, nCols=%r' % (nQuery, nConfig))
# header = (' labled rank matrix: rows=queries, cols=cfgs:')
# print('\n'.join(cfgx2_lbl))
# column_list = rank_mat.T
# print(ut.make_csv_table(column_list, row_lbls=testres.qaids,
# column_lbls=column_lbls, header=header,
# transpose=False,
# use_lbl_width=len(cfgx2_lbl) < 5))
#print_best_rankmat()
#@ut.argv_flag_dec
#def print_diffmat():
# # score differences over configs
# print('-------------')
# print('Diffmat: %s' % testnameid)
# diff_matstr = get_diffmat_str(rank_mat, testres.qaids, nConfig)
# print(diff_matstr)
#print_diffmat()
#@ut.argv_flag_dec
#def print_rankhist_time():
# print('A rank histogram is a dictionary. '
# 'The keys denote the range of the ranks that the values fall in')
# # TODO: rectify this code with other hist code
# config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
# config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()
# _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
# for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
# #full_cfgstr = testres.cfgx2_qreq_[cfgx].get_full_cfgstr()
# #ut.print_dict(ut.dict_hist(ranks), 'rank histogram', sorted_=True)
# # find the qxs that belong to each bin
# aid_list1 = testres.qaids
# aid_list2 = qx2_gt_aid
# ibs.assert_valid_aids(aid_list1)
# ibs.assert_valid_aids(aid_list2)
# timedelta_list = ibs.get_annot_pair_timdelta(aid_list1, aid_list2)
# #timedelta_str_list = [ut.get_posix_timedelta_str2(delta)
# # for delta in timedelta_list]
# bin_edges = testres.get_rank_histogram_bin_edges()
# timedelta_groups = ut.dict_take(ut.group_items(timedelta_list, config_binxs), np.arange(len(bin_edges)), [])
# timedelta_stats = [ut.get_stats(deltas, use_nan=True, datacast=ut.get_posix_timedelta_str2) for deltas in timedelta_groups]
# print('Time statistics for each rank range:')
# print(ut.dict_str(dict(zip(bin_edges, timedelta_stats)), sorted_=True))
#print_rankhist_time()
#@ut.argv_flag_dec
#def print_rankhist():
# print('A rank histogram is a dictionary. '
# 'The keys denote the range of the ranks that the values fall in')
# # TODO: rectify this code with other hist code
# config_gt_aids = ut.get_list_column(testres.cfgx2_cfgresinfo, 'qx2_gt_aid')
# config_rand_bin_qxs = testres.get_rank_histogram_qx_binxs()
# _iter = enumerate(zip(rank_mat.T, agg_hist_dict, config_gt_aids, config_rand_bin_qxs))
# for cfgx, (ranks, agg_hist_dict, qx2_gt_aid, config_binxs) in _iter:
# print('Frequency of rank ranges:')
# ut.print_dict(agg_hist_dict, 'agg rank histogram', sorted_=True)
#print_rankhist()
#------------
# Print summary
#print(' --- SUMMARY ---')
#------------
#@ut.argv_flag_dec
#def print_colmap():
# print('==================')
# print('[harn] mAP per Config: %s (sorted by mAP)' % testnameid)
# print('==================')
# cfgx2_mAP = np.array([aveprec_list.mean() for aveprec_list in cfgx2_aveprecs])
# sortx = cfgx2_mAP.argsort()
# for cfgx in sortx:
# print('[mAP] cfgx=%r) mAP=%.3f -- %s' % (cfgx, cfgx2_mAP[cfgx], cfgx2_lbl[cfgx]))
# #print('L___ Scores per Config ___')
#print_colmap()
#------------
@ut.argv_flag_dec_true
def print_colscore():
print('==================')
print('[harn] Scores per Config: %s' % testnameid)
print('==================')
#for cfgx in range(nConfig):
# print('[score] %s' % (cfgx2_lbl[cfgx]))
# for X in X_LIST:
# nLessX_ = nLessX_dict[int(X)][cfgx]
# print(' ' + rankscore_str(X, nLessX_, nQuery))
print('\n[harn] ... sorted scores')
for X in X_LIST:
print('\n[harn] Sorted #ranks < %r scores' % (X))
sortx = np.array(nLessX_dict[int(X)]).argsort()
#frac_list = (nLessX_dict[int(X)] / cfgx2_nQuery)[:, None]
#print('cfgx2_nQuery = %r' % (cfgx2_nQuery,))
#print('frac_list = %r' % (frac_list,))
#print('Pairwise Difference: ' + str(ut.safe_pdist(frac_list, metric=ut.absdiff)))
for cfgx in sortx:
nLessX_ = nLessX_dict[int(X)][cfgx]
rankstr = rankscore_str(X,
nLessX_,
cfgx2_nQuery[cfgx],
withlbl=False)
print('[score] %s --- %s' % (rankstr, cfgx2_lbl[cfgx]))
print_colscore()
#------------
ut.argv_flag_dec(print_latexsum)(ibs, testres)
@ut.argv_flag_dec
def print_next_rankmat():
# Prints nextbest ranks
print('-------------')
print('NextRankMat: %s' % testnameid)
header = (' top false rank matrix: rows=queries, cols=cfgs:')
print('\n'.join(cfgx2_lbl))
column_list = cfgx2_nextbestranks
print(
ut.make_csv_table(column_list,
row_lbls=testres.qaids,
column_lbls=column_lbls,
header=header,
transpose=False,
use_lbl_width=len(cfgx2_lbl) < 5))
print_next_rankmat()
#------------
@ut.argv_flag_dec
def print_scorediff_mat():
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMat: %s' % testnameid)
header = (
' score difference between top true and top false: rows=queries, cols=cfgs:'
)
print('\n'.join(cfgx2_lbl))
column_list = cfgx2_scorediffs
column_type = [float] * len(column_list)
print(
ut.make_csv_table(column_list,
row_lbls=testres.qaids,
column_lbls=column_lbls,
column_type=column_type,
header=header,
transpose=False,
use_lbl_width=len(cfgx2_lbl) < 5))
print_scorediff_mat(alias_flags=['--sdm'])
#------------
def jagged_stats_info(arr_, lbl, col_lbls):
arr = ut.recursive_replace(arr_, np.inf, np.nan)
# Treat infinite as nan
stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
sel_stat_dict, sel_indices = ut.find_interesting_stats(
stat_dict, col_lbls)
sel_col_lbls = ut.take(col_lbls, sel_indices)
statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True)
stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
sel_stat_str = 'sel_col_lbls = %s' % (
ut.list_str(sel_col_lbls), ) + '\n' + sel_stat_str
return stat_str, sel_stat_str
@ut.argv_flag_dec
def print_confusion_stats():
"""
CommandLine:
python dev.py --allgt --print-scorediff-mat-stats --print-confusion-stats -t rrvsone_grid
"""
# Prints nextbest ranks
print('-------------')
print('ScoreDiffMatStats: %s' % testnameid)
print('column_lbls = %r' % (column_lbls, ))
#cfgx2_gt_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gt_raw_score')
#cfgx2_gf_rawscores = ut.get_list_column(cfgx2_cfgresinfo, 'qx2_gf_raw_score')
gt_rawscores_mat = ut.replace_nones(cfgx2_gt_rawscores, np.nan)
gf_rawscores_mat = ut.replace_nones(cfgx2_gf_rawscores, np.nan)
tp_rawscores = vt.zipcompress(gt_rawscores_mat, istrue_list)
fp_rawscores = vt.zipcompress(gt_rawscores_mat, isfalse_list)
tn_rawscores = vt.zipcompress(gf_rawscores_mat, istrue_list)
fn_rawscores = vt.zipcompress(gf_rawscores_mat, isfalse_list)
tp_rawscores_str, tp_rawscore_statstr = jagged_stats_info(
tp_rawscores, 'tp_rawscores', cfgx2_lbl)
fp_rawscores_str, fp_rawscore_statstr = jagged_stats_info(
fp_rawscores, 'fp_rawscores', cfgx2_lbl)
tn_rawscores_str, tn_rawscore_statstr = jagged_stats_info(
tn_rawscores, 'tn_rawscores', cfgx2_lbl)
fn_rawscores_str, fn_rawscore_statstr = jagged_stats_info(
fn_rawscores, 'fn_rawscores', cfgx2_lbl)
#print(tp_rawscores_str)
#print(fp_rawscores_str)
#print(tn_rawscores_str)
#print(fn_rawscores_str)
print(tp_rawscore_statstr)
print(fp_rawscore_statstr)
print(tn_rawscore_statstr)
print(fn_rawscore_statstr)
print_confusion_stats(alias_flags=['--cs'])
ut.argv_flag_dec_true(testres.print_percent_identification_success)()
sumstrs = []
sumstrs.append('')
sumstrs.append('||===========================')
sumstrs.append('|| [cfg*] SUMMARY: %s' % testnameid)
sumstrs.append('||---------------------------')
sumstrs.append(ut.joins('\n|| ', best_rankscore_summary))
sumstrs.append('||===========================')
summary_str = '\n' + '\n'.join(sumstrs) + '\n'
#print(summary_str)
ut.colorprint(summary_str, 'blue')
print('To enable all printouts add --print-all to the commandline')