def test_pref():
guitool.ensure_qtapp()
import utool
root = utool.Pref(name='mypref')
root.pref1 = 5
root.pref2 = 'string'
root.pref3 = 'True'
root.pref4 = True
root.pref5 = utool.Pref(name='child')
root.pref5.pref6 = 3.5
root.createQWidget()
return root
def _init_gui(activate=True):
import guitool
if NOT_QUIET:
print('[main] _init_gui()')
guitool.ensure_qtapp()
#USE_OLD_BACKEND = '--old-backend' in sys.argv
#if USE_OLD_BACKEND:
from ibeis.gui import guiback
back = guiback.MainWindowBackend()
#else:
# from ibeis.gui import newgui
# back = newgui.IBEISGuiWidget()
if activate:
guitool.activate_qwindow(back.mainwin)
return back
def start_qt_interface(infr, loop=True):
import guitool as gt
from ibeis.viz.viz_graph2 import AnnotGraphWidget
from plottool import abstract_interaction
import plottool as pt
pt.qtensure()
gt.ensure_qtapp()
# win = AnnotGraphWidget(infr=infr, use_image=False, init_mode='review')
win = AnnotGraphWidget(infr=infr, use_image=False, init_mode=None)
abstract_interaction.register_interaction(win)
if loop:
gt.qtapp_loop(qwin=win, freq=10)
else:
win.show()
return win
def _init_gui(activate=True):
import guitool
if NOT_QUIET:
print('[main] _init_gui()')
guitool.ensure_qtapp()
#USE_OLD_BACKEND = '--old-backend' in sys.argv
#if USE_OLD_BACKEND:
from ibeis.gui import guiback
back = guiback.MainWindowBackend()
#else:
# from ibeis.gui import newgui
# back = newgui.IBEISGuiWidget()
if activate:
guitool.activate_qwindow(back.mainwin)
return back
def guiselect_workdir():
""" Prompts the user to specify a work directory """
import guitool
guitool.ensure_qtapp()
# Gui selection
work_dir = guitool.select_directory('Select a work directory')
# Make sure selection is ok
if not exists(work_dir):
try_again = guitool.user_option(
paremt=None,
msg='Directory %r does not exist.' % work_dir,
title='get work dir failed',
options=['Try Again'],
use_cache=False)
if try_again == 'Try Again':
return guiselect_workdir()
return work_dir
def guiselect_workdir():
""" Prompts the user to specify a work directory """
import guitool
guitool.ensure_qtapp()
# Gui selection
work_dir = guitool.select_directory('Select a work directory')
# Make sure selection is ok
if not exists(work_dir):
try_again = guitool.user_option(paremt=None,
msg='Directory %r does not exist.' %
work_dir,
title='get work dir failed',
options=['Try Again'],
use_cache=False)
if try_again == 'Try Again':
return guiselect_workdir()
return work_dir
def fix_splits_interaction(ibs):
"""
python -m ibeis fix_splits_interaction --show
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> dbdir = '/media/danger/GGR/GGR-IBEIS'
>>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
>>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
>>> import guitool as gt
>>> gt.ensure_qtapp()
>>> win = fix_splits_interaction(ibs)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> gt.qtapp_loop(qwin=win)
"""
split_props = {'splitcase', 'photobomb'}
all_annot_groups = ibs._annot_groups(ibs.group_annots_by_name(ibs.get_valid_aids())[0])
all_has_split = [len(split_props.intersection(ut.flatten(tags))) > 0 for tags in all_annot_groups.match_tags]
tosplit_annots = ut.compress(all_annot_groups.annots_list, all_has_split)
tosplit_annots = ut.take(tosplit_annots, ut.argsort(ut.lmap(len, tosplit_annots)))[::-1]
if ut.get_argflag('--reverse'):
tosplit_annots = tosplit_annots[::-1]
print('len(tosplit_annots) = %r' % (len(tosplit_annots),))
aids_list = [a.aids for a in tosplit_annots]
from ibeis.algo.graph import graph_iden
from ibeis.viz import viz_graph2
import guitool as gt
import plottool as pt
pt.qt4ensure()
gt.ensure_qtapp()
for aids in ut.InteractiveIter(aids_list):
infr = graph_iden.AnnotInference(ibs, aids)
infr.initialize_graph()
win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
init_mode='rereview')
win.populate_edge_model()
win.show()
return win
# eid = model.eid
gid = id_
print("SINGLE CLICKED ID: %r" % gid)
@slot_(QtCore.QModelIndex)
def on_doubleclick(ibswgt, qtindex):
printDBG('on_doubleclick')
model = qtindex.model()
id_ = model._get_row_id(qtindex)
if model.name == NAMES_TREE:
# eid = model.eid
gid = id_
#ibswgt.annotation_interact = interact_annotations2.ANNOTATION_Interaction2(ibswgt.ibs, gid)
print("DOUBLECLICKED ID: %r" % gid)
if __name__ == '__main__':
import ibeis
import sys
ibeis._preload(mpl=False, par=False)
guitool.ensure_qtapp()
dbdir = ibeis.sysres.get_args_dbdir(defaultdb='cache')
ibs = IBEISControl.IBEISController(dbdir=dbdir)
ibswgt = IBEISGuiWidget(ibs=ibs)
ibswgt.resize(900, 600)
if '--cmd' in sys.argv:
guitool.qtapp_loop(qwin=ibswgt, ipy=True)
exec(utool.ipython_execstr())
else:
guitool.qtapp_loop(qwin=ibswgt)
def edit_config(self, event):
import guitool
guitool.ensure_qtapp()
from guitool import PrefWidget2
self.widget = PrefWidget2.EditConfigWidget(config=self.config)
self.widget.show()
def test_inc_query(ibs_gt, num_initial=0):
"""
entry point for interactive query tests
test_interactive_incremental_queries
Args:
ibs (list) : IBEISController object
qaid_list (list) : list of annotation-ids to query
CommandLine:
python dev.py -t inc --db PZ_MTEST --qaid 1:30:3 --cmd
python dev.py --db PZ_MTEST --allgt --cmd
python dev.py --db PZ_MTEST --allgt -t inc
python dev.py --db PZ_MTEST --allgt -t inc
CommandLine:
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --interact-after 444440 --noqcache
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:1 --interact-after 444440 --noqcache
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:1
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:2
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3 --ninit 5000
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0
# Writes out test script
python -c "import utool as ut; ut.write_modscript_alias('Tinc.sh', 'ibeis.algo.hots.qt_inc_automatch')"
sh Tinc.sh --test-test_inc_query:0
sh Tinc.sh --test-test_inc_query:1
sh Tinc.sh --test-test_inc_query:2
sh Tinc.sh --test-test_inc_query:3
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3 --num-init 5000
sh Tinc.sh --test-test_inc_query:0 --ninit 10
sh Tinc.sh --test-test_inc_query:0 --ninit 10 --verbose-debug --verbose-helpful
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --ia 10
# Runs into a merge case
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --ia 30
Example0:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('testdb1')
>>> test_inc_query(ibs_gt)
Example1:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('PZ_MTEST')
>>> test_inc_query(ibs_gt)
Example2:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('GZ_ALL')
>>> test_inc_query(ibs_gt)
Example3:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('PZ_Master0')
>>> test_inc_query(ibs_gt)
"""
from ibeis import main_module
main_module._preload()
guitool.ensure_qtapp()
num_initial
self = IncQueryHarness()
num_initial = ut.get_argval(('--num-initial', '--num-init', '--ninit'), int, 0)
interactive_after = ut.get_argval(('--interactive-after', '--ia'), type_=int, default=None)
# Add information to an empty database from a groundtruth database
ibs, aid_list1, aid1_to_aid2 = ah.setup_incremental_test(ibs_gt)
if True or interactive_after is not None:
back = main_module._init_gui()
back.connect_ibeis_control(ibs)
else:
back = None
interactive = self.test_incremental_query(
ibs_gt, ibs, aid_list1, aid1_to_aid2,
interactive_after=interactive_after, num_initial=num_initial, back=back)
if back is None and interactive_after is None and interactive:
# need to startup backend
back = main_module._init_gui()
back.connect_ibeis_control(ibs)
if interactive or interactive_after is not None:
guitool.qtapp_loop()
def split_analysis(ibs):
"""
CommandLine:
python -m ibeis.other.dbinfo split_analysis --show
python -m ibeis split_analysis --show
python -m ibeis split_analysis --show --good
Ignore:
# mount
sshfs -o idmap=user lev:/ ~/lev
# unmount
fusermount -u ~/lev
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> dbdir = '/media/danger/GGR/GGR-IBEIS'
>>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
>>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
>>> import guitool as gt
>>> gt.ensure_qtapp()
>>> win = split_analysis(ibs)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> gt.qtapp_loop(qwin=win)
>>> #ut.show_if_requested()
"""
#nid_list = ibs.get_valid_nids(filter_empty=True)
import datetime
day1 = datetime.date(2016, 1, 30)
day2 = datetime.date(2016, 1, 31)
filter_kw = {
'multiple': None,
#'view': ['right'],
#'minqual': 'good',
'is_known': True,
'min_pername': 1,
}
aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)),
})
)
aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
})
)
all_aids = aids1 + aids2
all_annots = ibs.annots(all_aids)
print('%d annots on day 1' % (len(aids1)) )
print('%d annots on day 2' % (len(aids2)) )
print('%d annots overall' % (len(all_annots)) )
print('%d names overall' % (len(ut.unique(all_annots.nids))) )
nid_list, annots_list = all_annots.group(all_annots.nids)
REVIEWED_EDGES = True
if REVIEWED_EDGES:
aids_list = [annots.aids for annots in annots_list]
#aid_pairs = [annots.get_am_aidpairs() for annots in annots_list] # Slower
aid_pairs = ibs.get_unflat_am_aidpairs(aids_list) # Faster
else:
# ALL EDGES
aid_pairs = [annots.get_aidpairs() for annots in annots_list]
speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs)
import vtool as vt
max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list])
nan_idx = np.where(np.isnan(max_speeds))[0]
inf_idx = np.where(np.isinf(max_speeds))[0]
bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx])))
ok_idx = ut.index_complement(bad_idx, len(max_speeds))
print('#nan_idx = %r' % (len(nan_idx),))
print('#inf_idx = %r' % (len(inf_idx),))
print('#ok_idx = %r' % (len(ok_idx),))
ok_speeds = max_speeds[ok_idx]
ok_nids = ut.take(nid_list, ok_idx)
ok_annots = ut.take(annots_list, ok_idx)
sortx = np.argsort(ok_speeds)[::-1]
sorted_speeds = np.array(ut.take(ok_speeds, sortx))
sorted_annots = np.array(ut.take(ok_annots, sortx))
sorted_nids = np.array(ut.take(ok_nids, sortx)) # NOQA
sorted_speeds = np.clip(sorted_speeds, 0, 100)
#idx = vt.find_elbow_point(sorted_speeds)
#EXCESSIVE_SPEED = sorted_speeds[idx]
# http://www.infoplease.com/ipa/A0004737.html
# http://www.speedofanimals.com/animals/zebra
#ZEBRA_SPEED_MAX = 64 # km/h
#ZEBRA_SPEED_RUN = 50 # km/h
ZEBRA_SPEED_SLOW_RUN = 20 # km/h
#ZEBRA_SPEED_FAST_WALK = 10 # km/h
#ZEBRA_SPEED_WALK = 7 # km/h
MAX_SPEED = ZEBRA_SPEED_SLOW_RUN
#MAX_SPEED = ZEBRA_SPEED_WALK
#MAX_SPEED = EXCESSIVE_SPEED
flags = sorted_speeds > MAX_SPEED
flagged_ok_annots = ut.compress(sorted_annots, flags)
inf_annots = ut.take(annots_list, inf_idx)
flagged_annots = inf_annots + flagged_ok_annots
print('MAX_SPEED = %r km/h' % (MAX_SPEED,))
print('%d annots with infinite speed' % (len(inf_annots),))
print('%d annots with large speed' % (len(flagged_ok_annots),))
print('Marking all pairs of annots above the threshold as non-matching')
from ibeis.algo.graph import graph_iden
import networkx as nx
progkw = dict(freq=1, bs=True, est_window=len(flagged_annots))
bad_edges_list = []
good_edges_list = []
for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw):
edge_to_speeds = annots.get_speeds()
bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
good_edges = [edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED]
bad_edges_list.append(bad_edges)
good_edges_list.append(good_edges)
all_bad_edges = ut.flatten(bad_edges_list)
good_edges_list = ut.flatten(good_edges_list)
print('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)),))
print('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)),))
if 1:
from ibeis.viz import viz_graph2
import guitool as gt
gt.ensure_qtapp()
if ut.get_argflag('--good'):
print('Looking at GOOD (no speed problems) edges')
aid_pairs = good_edges_list
else:
print('Looking at BAD (speed problems) edges')
aid_pairs = all_bad_edges
aids = sorted(list(set(ut.flatten(aid_pairs))))
infr = graph_iden.AnnotInference(ibs, aids, verbose=False)
infr.initialize_graph()
# Use random scores to randomize sort order
rng = np.random.RandomState(0)
scores = (-rng.rand(len(aid_pairs)) * 10).tolist()
infr.graph.add_edges_from(aid_pairs)
if True:
edge_sample_size = 250
pop_nids = ut.unique(ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs))))
sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size]
sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0))
sample_size = len(ut.unique(sorted_nids))
am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(*sorted_pairs))
flags = ut.not_list(ut.flag_None_items(am_rowids))
#am_rowids = ut.compress(am_rowids, flags)
positive_tags = ['SplitCase', 'Photobomb']
flags_list = [ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0)
for tag in positive_tags]
print('edge_case_hist: ' + ut.repr3(
['%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags)]))
is_positive = ut.or_lists(*flags_list)
num_positive = sum(ut.lmap(any, ut.group_items(is_positive, sorted_nids).values()))
pop = len(pop_nids)
print('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'),))
print('--- Sampling wrt edges ---')
print('edge_sample_size = %r' % (edge_sample_size,))
print('edge_population_size = %r' % (len(aid_pairs),))
print('num_positive_edges = %r' % (sum(is_positive)))
print('--- Sampling wrt names ---')
print('name_population_size = %r' % (pop,))
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=.95)
nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores)))
win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
init_mode=None)
win.populate_edge_model()
win.show()
return win
# Make review interface for only bad edges
infr_list = []
iter_ = list(zip(flagged_annots, bad_edges_list))
for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw):
aids = annots.aids
nids = [1] * len(aids)
infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False)
infr.initialize_graph()
infr.reset_feedback()
infr_list.append(infr)
# Check which ones are user defined as incorrect
#num_positive = 0
#for infr in infr_list:
# flag = np.any(infr.get_feedback_probs()[0] == 0)
# num_positive += flag
#print('num_positive = %r' % (num_positive,))
#pop = len(infr_list)
#print('pop = %r' % (pop,))
iter_ = list(zip(infr_list, bad_edges_list))
for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw):
flipped_edges = []
for aid1, aid2 in bad_edges:
if infr.graph.has_edge(aid1, aid2):
flipped_edges.append((aid1, aid2))
infr.add_feedback((aid1, aid2), NEGTV)
nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig')
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges})
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges})
#for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw):
# annots = ibs.annots(infr.aids)
# edge_to_speeds = annots.get_speeds()
# bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
def inference_stats(infr_list_):
relabel_stats = []
for infr in infr_list_:
num_ccs, num_inconsistent = infr.relabel_using_reviews()
state_hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, 'decision').values())
if POSTV not in state_hist:
state_hist[POSTV] = 0
hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, '_speed_split').values())
subgraphs = infr.positive_connected_compoments()
subgraph_sizes = [len(g) for g in subgraphs]
info = ut.odict([
('num_nonmatch_edges', state_hist[NEGTV]),
('num_match_edges', state_hist[POSTV]),
('frac_nonmatch_edges', state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV])),
('num_inconsistent', num_inconsistent),
('num_ccs', num_ccs),
('edges_flipped', hist.get('flip', 0)),
('edges_unchanged', hist.get('orig', 0)),
('bad_unreviewed_edges', hist.get('new', 0)),
('orig_size', len(infr.graph)),
('new_sizes', subgraph_sizes),
])
relabel_stats.append(info)
return relabel_stats
relabel_stats = inference_stats(infr_list)
print('\nAll Split Info:')
lines = []
for key in relabel_stats[0].keys():
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent'))
can_split_flags = num_incon_list == 0
print('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags)))
splittable_infrs = ut.compress(infr_list, can_split_flags)
relabel_stats = inference_stats(splittable_infrs)
print('\nTrival Split Info:')
lines = []
for key in relabel_stats[0].keys():
if key in ['num_inconsistent']:
continue
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (
key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges'))
num_nonmatch_edges = np.array(ut.take_column(relabel_stats, 'num_nonmatch_edges'))
flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3)
reasonable_infr = ut.compress(splittable_infrs, flags1)
new_sizes_list = ut.take_column(relabel_stats, 'new_sizes')
flags2 = [len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > .3
for sizes in new_sizes_list]
reasonable_infr = ut.compress(splittable_infrs, flags2)
print('#reasonable_infr = %r' % (len(reasonable_infr),))
for infr in ut.InteractiveIter(reasonable_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
rest = ~np.logical_or(flags1, flags2)
nonreasonable_infr = ut.compress(splittable_infrs, rest)
rng = np.random.RandomState(0)
random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng)
random_infr = ut.take(nonreasonable_infr, random_idx)
for infr in ut.InteractiveIter(random_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
#import scipy.stats as st
#conf_interval = .95
#st.norm.cdf(conf_interval)
# view-source:http://www.surveysystem.com/sscalc.htm
#zval = 1.96 # 95 percent confidence
#zValC = 3.8416 #
#zValC = 6.6564
#import statsmodels.stats.api as sms
#es = sms.proportion_effectsize(0.5, 0.75)
#sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1)
pop = 279
num_positive = 3
sample_size = 15
conf_level = .95
#conf_level = .99
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=.95)
pop = 279
#err_frac = .05 # 5%
err_frac = .10 # 10%
conf_level = .95
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
pop = 675
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.1)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.68, prior=.2)
vt.calc_sample_from_error_bars(.10, pop, conf_level=.68)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.68)
def dans_splits(ibs):
"""
python -m ibeis dans_splits --show
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> dbdir = '/media/danger/GGR/GGR-IBEIS'
>>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
>>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
>>> import guitool as gt
>>> gt.ensure_qtapp()
>>> win = dans_splits(ibs)
>>> ut.quit_if_noshow()
>>> import plottool as pt
>>> gt.qtapp_loop(qwin=win)
"""
#pair = 9262, 932
dans_aids = [26548, 2190, 9418, 29965, 14738, 26600, 3039, 2742, 8249,
20154, 8572, 4504, 34941, 4040, 7436, 31866, 28291,
16009, 7378, 14453, 2590, 2738, 22442, 26483, 21640, 19003,
13630, 25395, 20015, 14948, 21429, 19740, 7908, 23583, 14301,
26912, 30613, 19719, 21887, 8838, 16184, 9181, 8649, 8276,
14678, 21950, 4925, 13766, 12673, 8417, 2018, 22434, 21149,
14884, 5596, 8276, 14650, 1355, 21725, 21889, 26376, 2867,
6906, 4890, 21524, 6690, 14738, 1823, 35525, 9045, 31723,
2406, 5298, 15627, 31933, 19535, 9137, 21002, 2448,
32454, 12615, 31755, 20015, 24573, 32001, 23637, 3192, 3197,
8702, 1240, 5596, 33473, 23874, 9558, 9245, 23570, 33075,
23721, 24012, 33405, 23791, 19498, 33149, 9558, 4971,
34183, 24853, 9321, 23691, 9723, 9236, 9723, 21078,
32300, 8700, 15334, 6050, 23277, 31164, 14103,
21231, 8007, 10388, 33387, 4319, 26880, 8007, 31164,
32300, 32140]
is_hyrbid = [7123, 7166, 7157, 7158, ] # NOQA
needs_mask = [26836, 29742] # NOQA
justfine = [19862] # NOQA
annots = ibs.annots(dans_aids)
unique_nids = ut.unique(annots.nids)
grouped_aids = ibs.get_name_aids(unique_nids)
annot_groups = ibs._annot_groups(grouped_aids)
split_props = {'splitcase', 'photobomb'}
needs_tag = [len(split_props.intersection(ut.flatten(tags))) == 0 for tags in annot_groups.match_tags]
num_needs_tag = sum(needs_tag)
num_had_split = len(needs_tag) - num_needs_tag
print('num_had_split = %r' % (num_had_split,))
print('num_needs_tag = %r' % (num_needs_tag,))
#all_annot_groups = ibs._annot_groups(ibs.group_annots_by_name(ibs.get_valid_aids())[0])
#all_has_split = [len(split_props.intersection(ut.flatten(tags))) > 0 for tags in all_annot_groups.match_tags]
#num_nondan = sum(all_has_split) - num_had_split
#print('num_nondan = %r' % (num_nondan,))
from ibeis.algo.graph import graph_iden
from ibeis.viz import viz_graph2
import guitool as gt
import plottool as pt
pt.qt4ensure()
gt.ensure_qtapp()
aids_list = ut.compress(grouped_aids, needs_tag)
aids_list = [a for a in aids_list if len(a) > 1]
print('len(aids_list) = %r' % (len(aids_list),))
for aids in aids_list:
infr = graph_iden.AnnotInference(ibs, aids)
infr.initialize_graph()
win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
init_mode='rereview')
win.populate_edge_model()
win.show()
return win
assert False
print("on double click")
printDBG('on_doubleclick')
model = qtindex.model()
id_ = model._get_row_id(qtindex)
print('dblclick (%r,%r)' % (qtindex.row(), qtindex.column()))
if model.name == THUMB_TABLE:
# imgsetid = model.imgsetid
gid = id_
if gid is None:
return
ibswgt.roi_interact = interact_annotations2.ANNOTATION_Interaction2(
ibswgt.ibs, gid)
print("DOUBLECLICKED ID: %r" % gid)
if __name__ == '__main__':
import ibeis
import sys
ibeis._preload(mpl=False, par=False)
guitool.ensure_qtapp()
dbdir = ibeis.sysres.get_args_dbdir(defaultdb='cache')
ibs = IBEISControl.IBEISController(dbdir=dbdir)
ibswgt = IBEISGuiWidget(ibs=ibs)
ibswgt.resize(900, 600)
if '--cmd' in sys.argv:
guitool.qtapp_loop(qwin=ibswgt, ipy=True)
exec(utool.ipython_execstr())
else:
guitool.qtapp_loop(qwin=ibswgt)
def ggr_random_name_splits():
"""
CommandLine:
python -m ibeis.viz.viz_graph2 ggr_random_name_splits --show
Ignore:
sshfs -o idmap=user lev:/ ~/lev
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.viz.viz_graph2 import * # NOQA
>>> ggr_random_name_splits()
"""
import guitool as gt
gt.ensure_qtapp()
#nid_list = ibs.get_valid_nids(filter_empty=True)
import ibeis
dbdir = '/media/danger/GGR/GGR-IBEIS'
dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath(
'~/lev/media/danger/GGR/GGR-IBEIS')
ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
import datetime
day1 = datetime.date(2016, 1, 30)
day2 = datetime.date(2016, 1, 31)
orig_filter_kw = {
'multiple': None,
#'view': ['right'],
#'minqual': 'good',
'is_known': True,
'min_pername': 2,
}
orig_aids = ibs.filter_annots_general(filter_kw=ut.dict_union(
orig_filter_kw, {
'min_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
}))
orig_all_annots = ibs.annots(orig_aids)
orig_unique_nids, orig_grouped_annots_ = orig_all_annots.group(
orig_all_annots.nids)
# Ensure we get everything
orig_grouped_annots = [
ibs.annots(aids_) for aids_ in ibs.get_name_aids(orig_unique_nids)
]
# pip install quantumrandom
if False:
import quantumrandom
data = quantumrandom.uint16()
seed = data.sum()
print('seed = %r' % (seed, ))
#import Crypto.Random
#from Crypto import Random
#quantumrandom.get_data()
#StrongRandom = Crypto.Random.random.StrongRandom
#aes.reseed(3340258)
#chars = [str(chr(x)) for x in data.view(np.uint8)]
#aes_seed = str('').join(chars)
#aes = Crypto.Random.Fortuna.FortunaGenerator.AESGenerator()
#aes.reseed(aes_seed)
#aes.pseudo_random_data(10)
orig_rand_idxs = ut.random_indexes(len(orig_grouped_annots), seed=3340258)
orig_sample_size = 75
random_annot_groups = ut.take(orig_grouped_annots, orig_rand_idxs)
orig_annot_sample = random_annot_groups[:orig_sample_size]
# OOOPS MADE ERROR REDO ----
filter_kw = {
'multiple': None,
'view': ['right'],
'minqual': 'good',
'is_known': True,
'min_pername': 2,
}
filter_kw_ = ut.dict_union(
filter_kw, {
'min_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime':
ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
})
refiltered_sample = [
ibs.filter_annots_general(annot.aids, filter_kw=filter_kw_)
for annot in orig_annot_sample
]
is_ok = (np.array(ut.lmap(len, refiltered_sample)) >= 2)
ok_part_orig_sample = ut.compress(orig_annot_sample, is_ok)
ok_part_orig_nids = [x.nids[0] for x in ok_part_orig_sample]
# Now compute real sample
aids = ibs.filter_annots_general(filter_kw=filter_kw_)
all_annots = ibs.annots(aids)
unique_nids, grouped_annots_ = all_annots.group(all_annots.nids)
grouped_annots = grouped_annots_
# Ensure we get everything
#grouped_annots = [ibs.annots(aids_) for aids_ in ibs.get_name_aids(unique_nids)]
pop = len(grouped_annots)
pername_list = ut.lmap(len, grouped_annots)
groups = ibeis.annots.AnnotGroups(grouped_annots, ibs)
match_tags = [ut.unique(ut.flatten(t)) for t in groups.match_tags]
tag_case_hist = ut.dict_hist(ut.flatten(match_tags))
print('name_pop = %r' % (pop, ))
print('Annots per Multiton Name' +
ut.repr3(ut.get_stats(pername_list, use_median=True)))
print('Name Tag Hist ' + ut.repr3(tag_case_hist))
print('Percent Photobomb: %.2f%%' %
(tag_case_hist['photobomb'] / pop * 100))
print('Percent Split: %.2f%%' % (tag_case_hist['splitcase'] / pop * 100))
# Remove the ok part from this sample
remain_unique_nids = ut.setdiff(unique_nids, ok_part_orig_nids)
remain_grouped_annots = [
ibs.annots(aids_) for aids_ in ibs.get_name_aids(remain_unique_nids)
]
sample_size = 75
import vtool as vt
vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.05)
remain_rand_idxs = ut.random_indexes(len(remain_grouped_annots),
seed=3340258)
remain_sample_size = sample_size - len(ok_part_orig_nids)
remain_random_annot_groups = ut.take(remain_grouped_annots,
remain_rand_idxs)
remain_annot_sample = remain_random_annot_groups[:remain_sample_size]
annot_sample_nofilter = ok_part_orig_sample + remain_annot_sample
# Filter out all bad parts
annot_sample_filter = [
ibs.annots(ibs.filter_annots_general(annot.aids, filter_kw=filter_kw_))
for annot in annot_sample_nofilter
]
annot_sample = annot_sample_filter
win = None
from ibeis.viz import viz_graph2
for annots in ut.InteractiveIter(annot_sample):
if win is not None:
win.close()
win = viz_graph2.make_qt_graph_interface(ibs,
aids=annots.aids,
init_mode='rereview')
print(win)
sample_groups = ibeis.annots.AnnotGroups(annot_sample, ibs)
flat_tags = [ut.unique(ut.flatten(t)) for t in sample_groups.match_tags]
print('Using Split and Photobomb')
is_positive = ['photobomb' in t or 'splitcase' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=.95)
print('Only Photobomb')
is_positive = ['photobomb' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=.95)
print('Only SplitCase')
is_positive = ['splitcase' in t for t in flat_tags]
num_positive = sum(is_positive)
vt.calc_error_bars_from_sample(sample_size,
num_positive,
pop,
conf_level=.95)
def test_inc_query(ibs_gt, num_initial=0):
"""
entry point for interactive query tests
test_interactive_incremental_queries
Args:
ibs (list) : IBEISController object
qaid_list (list) : list of annotation-ids to query
CommandLine:
python dev.py -t inc --db PZ_MTEST --qaid 1:30:3 --cmd
python dev.py --db PZ_MTEST --allgt --cmd
python dev.py --db PZ_MTEST --allgt -t inc
python dev.py --db PZ_MTEST --allgt -t inc
CommandLine:
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --interact-after 444440 --noqcache
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:1 --interact-after 444440 --noqcache
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:1
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:2
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3 --ninit 5000
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0
# Writes out test script
python -c "import utool as ut; ut.write_modscript_alias('Tinc.sh', 'ibeis.algo.hots.qt_inc_automatch')"
sh Tinc.sh --test-test_inc_query:0
sh Tinc.sh --test-test_inc_query:1
sh Tinc.sh --test-test_inc_query:2
sh Tinc.sh --test-test_inc_query:3
utprof.py -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:3 --num-init 5000
sh Tinc.sh --test-test_inc_query:0 --ninit 10
sh Tinc.sh --test-test_inc_query:0 --ninit 10 --verbose-debug --verbose-helpful
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --ia 10
# Runs into a merge case
python -m ibeis.algo.hots.qt_inc_automatch --test-test_inc_query:0 --ia 30
Example0:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('testdb1')
>>> test_inc_query(ibs_gt)
Example1:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('PZ_MTEST')
>>> test_inc_query(ibs_gt)
Example2:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('GZ_ALL')
>>> test_inc_query(ibs_gt)
Example3:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.automated_matcher import * # NOQA
>>> ibs_gt = ibeis.opendb('PZ_Master0')
>>> test_inc_query(ibs_gt)
"""
from ibeis import main_module
main_module._preload()
guitool.ensure_qtapp()
num_initial
self = IncQueryHarness()
num_initial = ut.get_argval(('--num-initial', '--num-init', '--ninit'),
int, 0)
interactive_after = ut.get_argval(('--interactive-after', '--ia'),
type_=int,
default=None)
# Add information to an empty database from a groundtruth database
ibs, aid_list1, aid1_to_aid2 = ah.setup_incremental_test(ibs_gt)
if True or interactive_after is not None:
back = main_module._init_gui()
back.connect_ibeis_control(ibs)
else:
back = None
interactive = self.test_incremental_query(
ibs_gt,
ibs,
aid_list1,
aid1_to_aid2,
interactive_after=interactive_after,
num_initial=num_initial,
back=back)
if back is None and interactive_after is None and interactive:
# need to startup backend
back = main_module._init_gui()
back.connect_ibeis_control(ibs)
if interactive or interactive_after is not None:
guitool.qtapp_loop()