def print_latexsum(ibs, testres, verbose=True):
r"""
Args:
ibs (IBEISController): ibeis controller object
testres (?):
CommandLine:
python -m ibeis.expt.experiment_printres --exec-print_latexsum
python -m ibeis.scripts.gen_cand_expts --exec-gen_script
python -m ibeis --tf print_latexsum -t candidacy --db PZ_Master0 -a controlled --rank-lt-list=1,5,10,100
python -m ibeis --tf print_latexsum -t candidacy --db PZ_MTEST -a controlled --rank-lt-list=1,5,10,100
Example:
>>> # SCRIPT
>>> from ibeis.expt.experiment_printres import * # NOQA
>>> from ibeis.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts()
>>> tabular_str2 = print_latexsum(ibs, testres)
"""
print('==========================')
print('[harn] LaTeX: %s' % testres.testnameid)
print('==========================')
# Create configuration latex table
X_LIST = testres.get_X_LIST()
criteria_lbls = [r'#ranks $\leq$ %d' % X for X in X_LIST]
dbname = ibs.get_dbname()
cfg_score_title = dbname + ' rank scores'
nLessX_dict = testres.get_nLessX_dict()
cfgscores = np.array([nLessX_dict[int(X)] for X in X_LIST]).T
# For mat row labels
row_lbls = testres.get_short_cfglbls()
# Order cdf list by rank0
row_lbls = ut.sortedby(row_lbls, cfgscores.T[0], reverse=True)
cfgscores = np.array(
ut.sortedby(cfgscores.tolist(), cfgscores.T[0], reverse=True))
cmdaug = testres.get_title_aug()
#if testres.common_acfg is not None:
# cfgname = testres.common_acfg['common']['_cfgname']
# cmdaug += '_' + cfgname
#if hasattr(testres, 'common_cfgdict'):
# cmdaug += '_' + (testres.common_cfgdict['_cfgname'])
# cfg_score_title += ' ' + cmdaug
tabular_kwargs = dict(title=cfg_score_title,
out_of=testres.nQuery,
bold_best=True,
flip=False,
SHORTEN_ROW_LBLS=False)
col_lbls = criteria_lbls
tabular_str = ut.util_latex.make_score_tabular(row_lbls, col_lbls,
cfgscores, **tabular_kwargs)
#latex_formater.render(tabular_str)
cmdname = ut.latex_sanitize_command_name('Expmt' + ibs.get_dbname() + '_' +
cmdaug + 'Table')
tabular_str2 = ut.latex_newcommand(cmdname, tabular_str)
print(tabular_str2)
return tabular_str2
def delete_empty_directories(self):
"""
ut.ensuredir(self.dpath + '/foo')
ut.ensuredir(self.dpath + '/foo/bar')
ut.ensuredir(self.dpath + '/foo/bar/baz')
self.delete_empty_directories()
"""
import os
# for root, dirs, files in os.walk(self.dpath, topdown=False):
# if len(files) == 0 and len(os.listdir(root)) == 0:
# print('Remove %s' % root)
# os.rmdir(root)
if True:
# Find all directories with no files
subdirs = ut.glob(self.dpath, '*', recursive=True, fullpath=False,
with_files=False, with_dirs=True)
freq = {d: 0 for d in subdirs + ['']}
for path in self.rel_fpath_list:
while True:
path = dirname(path)
freq[path] += 1
if not path:
break
to_delete = [k for k, v in freq.items() if v == 0]
# Remove deep dirs first
to_delete = ut.sortedby(to_delete, map(len, to_delete))[::-1]
for d in to_delete:
dpath = join(self.dpath, d)
print('Remove %s' % dpath)
os.rmdir(dpath)
def delete_empty_directories(self):
"""
ut.ensuredir(self.dpath + '/foo')
ut.ensuredir(self.dpath + '/foo/bar')
ut.ensuredir(self.dpath + '/foo/bar/baz')
self.delete_empty_directories()
"""
import os
# for root, dirs, files in os.walk(self.dpath, topdown=False):
# if len(files) == 0 and len(os.listdir(root)) == 0:
# print('Remove %s' % root)
# os.rmdir(root)
if True:
# Find all directories with no files
subdirs = ut.glob(self.dpath,
'*',
recursive=True,
fullpath=False,
with_files=False,
with_dirs=True)
freq = {d: 0 for d in subdirs + ['']}
for path in self.rel_fpath_list:
while True:
path = dirname(path)
freq[path] += 1
if not path:
break
to_delete = [k for k, v in freq.items() if v == 0]
# Remove deep dirs first
to_delete = ut.sortedby(to_delete, map(len, to_delete))[::-1]
for d in to_delete:
dpath = join(self.dpath, d)
print('Remove %s' % dpath)
os.rmdir(dpath)
def group_review():
prefill = request.args.get('prefill', '')
if len(prefill) > 0:
ibs = current_app.ibs
aid_list = ibs.get_valid_aids()
bad_species_list, bad_viewpoint_list = ibs.validate_annot_species_viewpoint_cnn(aid_list)
GROUP_BY_PREDICTION = True
if GROUP_BY_PREDICTION:
grouped_dict = ut.group_items(bad_viewpoint_list, ut.get_list_column(bad_viewpoint_list, 3))
grouped_list = grouped_dict.values()
regrouped_items = ut.flatten(ut.sortedby(grouped_list, map(len, grouped_list)))
candidate_aid_list = ut.get_list_column(regrouped_items, 0)
else:
candidate_aid_list = [ bad_viewpoint[0] for bad_viewpoint in bad_viewpoint_list]
elif request.args.get('aid_list', None) is not None:
aid_list = request.args.get('aid_list', '')
if len(aid_list) > 0:
aid_list = aid_list.replace('[', '')
aid_list = aid_list.replace(']', '')
aid_list = aid_list.strip().split(',')
candidate_aid_list = [ int(aid_.strip()) for aid_ in aid_list ]
else:
candidate_aid_list = ''
else:
candidate_aid_list = ''
return appf.template(None, 'group_review', candidate_aid_list=candidate_aid_list, mode_list=appf.VALID_TURK_MODES)
def _change_row_indices(model):
""" Non-Qt Helper """
if model.sortcolumn is not None:
print("using: sortcolumn=%r" % model.sortcolumn)
column_data = model.col_data_list[model.sortcolumn]
indices = list(range(len(column_data)))
model.row_sortx = utool.sortedby(indices, column_data, reverse=model.sortreverse)
elif len(model.col_data_list) > 0:
model.row_sortx = list(range(len(model.col_data_list[0])))
else:
model.row_sortx = []
def print_factors(model, factor_list):
if hasattr(model, 'var2_cpd'):
semtypes = [model.var2_cpd[f.variables[0]].ttype for f in factor_list]
else:
semtypes = [0] * len(factor_list)
for type_, factors in ut.group_items(factor_list, semtypes).items():
logger.info('Result Factors (%r)' % (type_, ))
factors = ut.sortedby(factors, [f.variables[0] for f in factors])
for fs_ in ut.ichunks(factors, 4):
ut.colorprint(ut.hz_str([f._str('phi', 'psql') for f in fs_]),
'yellow')
def get_backup_fpaths(ibs):
fname, ext = splitext(ibs.sqldb_fname)
backups = sorted(ut.glob(ibs.backupdir, '*%s' % ext))
#backup_info = [ut.get_file_info(fpath) for fpath in backups]
modified = [ut.get_file_info(fpath)['last_modified'] for fpath in backups]
unixtimes = [ut.util_time.exiftime_to_unixtime(tag) for tag in modified]
backups = ut.sortedby(backups, unixtimes)
return backups
#backup_uuids = [ut.get_file_uuid(fpath) for fpath in backups]
#backup_hashes = [ut.get_file_hash(fpath) for fpath in backups]
#backup_bytes = [ut.get_file_nBytes(fpath) for fpath in backups]
pass
def print_factors(model, factor_list):
if hasattr(model, 'var2_cpd'):
semtypes = [model.var2_cpd[f.variables[0]].ttype
for f in factor_list]
else:
semtypes = [0] * len(factor_list)
for type_, factors in ut.group_items(factor_list, semtypes).items():
print('Result Factors (%r)' % (type_,))
factors = ut.sortedby(factors, [f.variables[0] for f in factors])
for fs_ in ut.ichunks(factors, 4):
ut.colorprint(ut.hz_str([f._str('phi', 'psql') for f in fs_]),
'yellow')
def print_database_structure(cur):
import utool as ut
tablename_list = ut.get_tablenames(cur)
colinfos_list = [ut.get_table_columninfo_list(cur, tablename) for tablename in tablename_list]
numrows_list = [ut.get_table_num_rows(cur, tablename) for tablename in tablename_list]
for tablename, colinfo_list, num_rows in ut.sortedby(list(zip(tablename_list, colinfos_list, numrows_list)), numrows_list):
print('+-------------')
print('tablename = %r' % (tablename,))
print('num_rows = %r' % (num_rows,))
#print(ut.list_str(colinfo_list))
print(ut.list_str(ut.get_primary_columninfo(cur, tablename)))
print(ut.list_str(ut.get_nonprimary_columninfo(cur, tablename)))
print('+-------------')
def print_database_structure(cur):
import utool as ut
tablename_list = ut.get_tablenames(cur)
colinfos_list = [ut.get_table_columninfo_list(cur, tablename) for tablename in tablename_list]
numrows_list = [ut.get_table_num_rows(cur, tablename) for tablename in tablename_list]
for tablename, colinfo_list, num_rows in ut.sortedby(list(zip(tablename_list, colinfos_list, numrows_list)), numrows_list):
print('+-------------')
print('tablename = %r' % (tablename,))
print('num_rows = %r' % (num_rows,))
#print(ut.repr4(colinfo_list))
print(ut.repr4(ut.get_primary_columninfo(cur, tablename)))
print(ut.repr4(ut.get_nonprimary_columninfo(cur, tablename)))
print('+-------------')
def _change_row_indices(model):
""" Non-Qt Helper """
if model.sortcolumn is not None:
print('using: sortcolumn=%r' % model.sortcolumn)
column_data = model.col_data_list[model.sortcolumn]
indices = list(range(len(column_data)))
model.row_sortx = utool.sortedby(indices,
column_data,
reverse=model.sortreverse)
elif len(model.col_data_list) > 0:
model.row_sortx = list(range(len(model.col_data_list[0])))
else:
model.row_sortx = []
def get_cached_vocabs():
import parse
# Parse some of the training data from fname
parse_str = '{}nC={num_cent},{}_DPTS(({num_dpts},{dim}){}'
smkdir = ut.get_app_resource_dir('smk')
fname_list = ut.glob(smkdir, 'akmeans*')
fpath_list = [join(smkdir, fname) for fname in fname_list]
result_list = [parse.parse(parse_str, fpath) for fpath in fpath_list]
nCent_list = [int(res['num_cent']) for res in result_list]
nDpts_list = [int(res['num_dpts']) for res in result_list]
key_list = zip(nCent_list, nDpts_list)
fpath_sorted = ut.sortedby(fpath_list, key_list, reverse=True)
return fpath_sorted
def get_cached_vocabs():
import parse
# Parse some of the training data from fname
parse_str = '{}nC={num_cent},{}_DPTS(({num_dpts},{dim}){}'
smkdir = ut.get_app_resource_dir('smk')
fname_list = ut.glob(smkdir, 'akmeans*')
fpath_list = [join(smkdir, fname) for fname in fname_list]
result_list = [parse.parse(parse_str, fpath) for fpath in fpath_list]
nCent_list = [int(res['num_cent']) for res in result_list]
nDpts_list = [int(res['num_dpts']) for res in result_list]
key_list = zip(nCent_list, nDpts_list)
fpath_sorted = ut.sortedby(fpath_list, key_list, reverse=True)
return fpath_sorted
def rectify_labels(G, labels):
# Ensure labels are rebased and
# are different between different connected compoments
graph = G.copy()
node_to_annot_idx = nx.get_node_attributes(graph, 'annot_idx')
cut_edges = []
for u, v in graph.edges():
idx1 = node_to_annot_idx[u]
idx2 = node_to_annot_idx[v]
if labels[idx1] != labels[idx2]:
cut_edges.append((u, v))
graph.remove_edges_from(cut_edges)
ccs_nodes = list(nx.connected_components(graph))
ccs_idxs = ut.unflat_take(node_to_annot_idx, ccs_nodes)
# Make consistent sorting
ccs_idxs = [sorted(idxs) for idxs in ccs_idxs]
ccs_idxs = ut.sortedby(ccs_idxs, ut.take_column(ccs_idxs, 0))
labels = ut.ungroup([[c] * len(x) for c, x in enumerate(ccs_idxs)], ccs_idxs)
labels = np.array(labels)
return labels
def print_latexsum(ibs, testres, verbose=True):
r"""
Args:
ibs (IBEISController): ibeis controller object
testres (?):
CommandLine:
python -m ibeis.expt.experiment_printres --exec-print_latexsum
python -m ibeis.scripts.gen_cand_expts --exec-gen_script
python -m ibeis --tf print_latexsum -t candidacy --db PZ_Master0 -a controlled --rank-lt-list=1,5,10,100
python -m ibeis --tf print_latexsum -t candidacy --db PZ_MTEST -a controlled --rank-lt-list=1,5,10,100
Example:
>>> # SCRIPT
>>> from ibeis.expt.experiment_printres import * # NOQA
>>> from ibeis.init import main_helpers
>>> ibs, testres = main_helpers.testdata_expts()
>>> tabular_str2 = print_latexsum(ibs, testres)
"""
print('==========================')
print('[harn] LaTeX: %s' % testres.testnameid)
print('==========================')
# Create configuration latex table
X_LIST = testres.get_X_LIST()
criteria_lbls = [r'#ranks $\leq$ %d' % X for X in X_LIST]
dbname = ibs.get_dbname()
cfg_score_title = dbname + ' rank scores'
nLessX_dict = testres.get_nLessX_dict()
cfgscores = np.array([nLessX_dict[int(X)] for X in X_LIST]).T
# For mat row labels
row_lbls = testres.get_short_cfglbls()
# Order cdf list by rank0
row_lbls = ut.sortedby(row_lbls, cfgscores.T[0], reverse=True)
cfgscores = np.array(ut.sortedby(cfgscores.tolist(), cfgscores.T[0], reverse=True))
cmdaug = testres.get_title_aug()
#if testres.common_acfg is not None:
# cfgname = testres.common_acfg['common']['_cfgname']
# cmdaug += '_' + cfgname
#if hasattr(testres, 'common_cfgdict'):
# cmdaug += '_' + (testres.common_cfgdict['_cfgname'])
# cfg_score_title += ' ' + cmdaug
#else:
# #ut.embed()
# assert False, 'need common_cfgdict'
tabular_kwargs = dict(
title=cfg_score_title,
out_of=testres.nQuery,
bold_best=True,
flip=False,
SHORTEN_ROW_LBLS=False
)
col_lbls = criteria_lbls
tabular_str = ut.util_latex.make_score_tabular(
row_lbls, col_lbls, cfgscores, **tabular_kwargs)
#latex_formater.render(tabular_str)
cmdname = ut.latex_sanitize_command_name('Expmt' + ibs.get_dbname() + '_' + cmdaug + 'Table')
tabular_str2 = ut.latex_newcommand(cmdname, tabular_str)
print(tabular_str2)
return tabular_str2
def get_key_order(cfg):
keys = [k for k in cfg.keys() if k not in nonlbl_keys]
sortorder = [
2 * k.startswith('q') + 1 * k.startswith('d') for k in keys
]
return ut.sortedby(keys, sortorder)[::-1]
def temp_model(num_annots,
num_names,
score_evidence=[],
name_evidence=[],
other_evidence={},
noquery=False,
verbose=None,
**kwargs):
if verbose is None:
verbose = ut.VERBOSE
method = kwargs.pop('method', None)
model = make_name_model(num_annots, num_names, verbose=verbose, **kwargs)
if verbose:
model.print_priors(ignore_ttypes=[MATCH_TTYPE, SCORE_TTYPE])
model, evidence, soft_evidence = update_model_evidence(
model, name_evidence, score_evidence, other_evidence)
if verbose and len(soft_evidence) != 0:
model.print_priors(ignore_ttypes=[MATCH_TTYPE, SCORE_TTYPE],
title='Soft Evidence',
color='green')
# if verbose:
# ut.colorprint('\n --- Soft Evidence ---', 'white')
# for ttype, cpds in model.ttype2_cpds.items():
# if ttype != MATCH_TTYPE:
# for fs_ in ut.ichunks(cpds, 4):
# ut.colorprint(ut.hz_str([f._cpdstr('psql') for f in fs_]),
# 'green')
if verbose:
ut.colorprint('\n --- Inference ---', 'red')
if (len(evidence) > 0 or len(soft_evidence) > 0) and not noquery:
evidence = model._ensure_internal_evidence(evidence)
query_vars = []
query_vars += ut.list_getattr(model.ttype2_cpds[NAME_TTYPE],
'variable')
# query_vars += ut.list_getattr(model.ttype2_cpds[MATCH_TTYPE], 'variable')
query_vars = ut.setdiff(query_vars, evidence.keys())
# query_vars = ut.setdiff(query_vars, soft_evidence.keys())
query_results = cluster_query(model, query_vars, evidence,
soft_evidence, method)
else:
query_results = {}
factor_list = query_results['factor_list']
if verbose:
if verbose:
logger.info('+--------')
semtypes = [model.var2_cpd[f.variables[0]].ttype for f in factor_list]
for type_, factors in ut.group_items(factor_list, semtypes).items():
logger.info('Result Factors (%r)' % (type_, ))
factors = ut.sortedby(factors, [f.variables[0] for f in factors])
for fs_ in ut.ichunks(factors, 4):
ut.colorprint(ut.hz_str([f._str('phi', 'psql') for f in fs_]),
'yellow')
logger.info('MAP assignments')
top_assignments = query_results.get('top_assignments', [])
tmp = []
for lbl, val in top_assignments:
tmp.append('%s : %.4f' % (ut.repr2(lbl), val))
logger.info(ut.align('\n'.join(tmp), ' :'))
logger.info('L_____\n')
showkw = dict(evidence=evidence,
soft_evidence=soft_evidence,
**query_results)
from wbia.algo.hots import pgm_viz
pgm_viz.show_model(model, **showkw)
return (model, evidence, query_results)
def make_clusters(infr):
import itertools
import networkx as nx
cm_list = infr.cm_list
graph = infr.make_graph()
# hack for orig aids
orig_aid2_nid = {}
for cm in cm_list:
orig_aid2_nid[cm.qaid] = cm.qnid
for daid, dnid in zip(cm.daid_list, cm.dnid_list):
orig_aid2_nid[daid] = dnid
cluster_aids = []
cluster_nids = []
connected = list(nx.connected_components(graph))
for comp in connected:
cluster_nids.append([])
cluster_aids.append([])
for x in comp:
if isinstance(x, tuple):
cluster_nids[-1].append(x[1])
else:
cluster_aids[-1].append(x)
# Make first part of inference output
qaid_list = [cm.qaid for cm in cm_list]
qaid_set = set(qaid_list)
#start_nid = 9001
# Find an nid that doesn't exist in the database
start_nid = len(infr.qreq_.ibs._get_all_known_name_rowids()) + 1
next_new_nid = itertools.count(start_nid)
cluster_tuples = []
for aids, nids in zip(cluster_aids, cluster_nids):
other_nid_clusters = cluster_nids[:]
other_nid_clusters.remove(nids)
other_nids = ut.flatten(other_nid_clusters)
split_case = len(ut.list_intersection(other_nids, nids)) > 0
merge_case = len(nids) > 1
new_name = len(nids) == 0
#print('[chip_match > AnnotInference > make_inference] WARNING:
# EXEMPLAR FLAG SET TO TRUE, NEEDS TO BE IMPLEMENTED')
error_flag = (split_case << 1) + (merge_case << 2) + (new_name << 3)
strflags = ['split', 'merge', 'new']
error_flag = ut.compress(strflags, [split_case, merge_case, new_name])
#error_flag = split_case or merge_case
# <HACK>
# SET EXEMPLARS
ibs = infr.qreq_.ibs
viewpoint_texts = ibs.get_annot_yaw_texts(aids)
view_to_aids = ut.group_items(aids, viewpoint_texts)
num_wanted_exemplars_per_view = 4
hack_set_these_qaids_as_exemplars = set([])
for view, aids_ in view_to_aids.items():
heuristic_exemplar_aids = set(aids) - qaid_set
heuristic_non_exemplar_aids = set(aids).intersection(qaid_set)
num_needed_exemplars = (num_wanted_exemplars_per_view -
len(heuristic_exemplar_aids))
# Choose the best query annots to fill out exemplars
if len(heuristic_non_exemplar_aids) == 0:
continue
quality_ints = ibs.get_annot_qualities(heuristic_non_exemplar_aids)
okish = ibs.const.QUALITY_TEXT_TO_INT[ibs.const.QUAL_OK] - .1
quality_ints = [x if x is None else okish for x in quality_ints]
aids_ = ut.sortedby(heuristic_non_exemplar_aids, quality_ints)[::-1]
chosen = aids_[:num_needed_exemplars]
for qaid_ in chosen:
hack_set_these_qaids_as_exemplars.add(qaid_)
# </HACK>
if not error_flag and not new_name:
new_nid = nids[0]
else:
new_nid = six.next(next_new_nid)
for aid in aids:
if aid not in qaid_set:
if len(error_flag) == 0:
continue
orig_nid = orig_aid2_nid[aid]
exemplar_flag = aid in hack_set_these_qaids_as_exemplars
#clusters is list 4 tuple: (aid, orig_name_uuid, new_name_uuid, error_flag)
tup = (aid, orig_nid, new_nid, exemplar_flag, error_flag)
cluster_tuples.append(tup)
return cluster_tuples
def sortself(group):
val_list = [(c.cmc, c.name) for c in group.cards]
group.cards = ut.sortedby(group.cards, val_list)
def dev_train_distinctiveness(species=None):
r"""
Args:
ibs (IBEISController): wbia controller object
species (None):
CommandLine:
python -m wbia.algo.hots.distinctiveness_normalizer --test-dev_train_distinctiveness
alias dev_train_distinctiveness='python -m wbia.algo.hots.distinctiveness_normalizer --test-dev_train_distinctiveness'
# Publishing (uses cached normalizers if available)
dev_train_distinctiveness --species GZ --publish
dev_train_distinctiveness --species PZ --publish
dev_train_distinctiveness --species PZ --retrain
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.algo.hots.distinctiveness_normalizer import * # NOQA
>>> import wbia
>>> species = ut.get_argval('--species', str, 'zebra_grevys')
>>> dev_train_distinctiveness(species)
"""
import wbia
# if 'species' not in vars() or species is None:
# species = 'zebra_grevys'
if species == 'zebra_grevys':
dbname = 'GZ_ALL'
elif species == 'zebra_plains':
dbname = 'PZ_Master0'
ibs = wbia.opendb(dbname)
global_distinctdir = ibs.get_global_distinctiveness_modeldir()
cachedir = global_distinctdir
dstcnvs_normer = DistinctivnessNormalizer(species, cachedir=cachedir)
try:
if ut.get_argflag('--retrain'):
raise IOError('force cache miss')
with ut.Timer('loading distinctiveness'):
dstcnvs_normer.load(cachedir)
# Cache hit
logger.info('distinctivness model cache hit')
except IOError:
logger.info('distinctivness model cache miss')
with ut.Timer('training distinctiveness'):
# Need to train
# Add one example from each name
# TODO: add one exemplar per viewpoint for each name
# max_vecs = 1E6
# max_annots = 975
max_annots = 975
# ibs.fix_and_clean_database()
nid_list = ibs.get_valid_nids()
aids_list = ibs.get_name_aids(nid_list)
# remove junk
aids_list = ibs.unflat_map(ibs.filter_junk_annotations, aids_list)
# remove empty
aids_list = [aids for aids in aids_list if len(aids) > 0]
num_annots_list = list(map(len, aids_list))
aids_list = ut.sortedby(aids_list, num_annots_list, reverse=True)
# take only one annot per name
aid_list = ut.get_list_column(aids_list, 0)
# Keep only a certain number of annots for distinctiveness mapping
aid_list_ = ut.listclip(aid_list, max_annots)
logger.info('total num named annots = %r' % (sum(num_annots_list)))
logger.info(
'training distinctiveness using %d/%d singleton annots' %
(len(aid_list_), len(aid_list)))
# vec
# FIXME: qreq_ params for config rowid
vecs_list = ibs.get_annot_vecs(aid_list_)
num_vecs = sum(list(map(len, vecs_list)))
logger.info('num_vecs = %r' % (num_vecs, ))
vecs = np.vstack(vecs_list)
logger.info('vecs size = %r' % (ut.get_object_size_str(vecs), ))
dstcnvs_normer.init_support(vecs)
dstcnvs_normer.save(global_distinctdir)
if ut.get_argflag('--publish'):
dstcnvs_normer.publish()
def show_qres_analysis(ibs, cm, qreq_=None, **kwargs):
"""
Wrapper around show_qres.
KWARGS:
aid_list - show matches against aid_list (default top 3)
Args:
ibs (IBEISController): ibeis controller object
cm (ChipMatch): object of feature correspondences and scores
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
N, show_gt, show_query, aid_list, figtitle, viz_name_score, viz_name_score
CommandLine:
python -m ibeis.viz.viz_qres --exec-show_qres_analysis --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.viz.viz_qres import * # NOQA
>>> import ibeis
>>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> daids = ibs.get_valid_aids(species=species)
>>> qaids = ibs.get_valid_aids(species=species)
>>> qres_list, qreq_ = ibs.query_chips(
>>> [1], [2, 3, 4, 5, 6, 7, 8, 9],
>>> cfgdict=dict(), return_request=True)
>>> cm = qres_list[0]
>>> kwargs = dict(show_query=False, viz_name_score=True,
>>> show_timedelta=True, N=3, show_gf=True)
>>> show_qres_analysis(ibs, cm, qreq_, **kwargs)
>>> ut.show_if_requested()
"""
if ut.NOT_QUIET:
print('[show_qres] cm.show_analysis()')
# Parse arguments
N = kwargs.get('N', DEFAULT_NTOP)
show_gt = kwargs.pop('show_gt', True)
show_gf = kwargs.pop('show_gf', False)
show_query = kwargs.pop('show_query', True)
aid_list = kwargs.pop('aid_list', None)
figtitle = kwargs.pop('figtitle', None)
#viz_name_score = kwargs.get('viz_name_score', qreq_ is not None)
#viz_name_score = kwargs.get('viz_name_score', False)
viz_name_score = kwargs.get('viz_name_score', True)
failed_to_match = False
# Debug printing
#print('[analysis] noshow_gt = %r' % noshow_gt)
#print('[analysis] show_query = %r' % show_query)
#print('[analysis] aid_list = %r' % aid_list)
if aid_list is None:
# Compare to aid_list instead of using top ranks
#print('[analysis] showing top aids')
top_aids = cm.get_top_aids(N)
if len(top_aids) == 0:
failed_to_match = True
print('WARNING! No matches found for this query')
if figtitle is None:
if len(top_aids) == 0:
figtitle = 'WARNING: no matches found!' + ibsfuncs.aidstr(cm.qaid)
else:
topscore = cm.get_annot_scores(top_aids)[0]
figtitle = ('q%s -- topscore=%r' % (ibsfuncs.aidstr(cm.qaid), topscore))
else:
print('[analysis] showing a given list of aids')
top_aids = aid_list
if figtitle is None:
figtitle = 'comparing to ' + ibsfuncs.aidstr(top_aids) + figtitle
# Get any groundtruth if you are showing it
showgt_aids = []
if show_gt:
# Get the missed groundtruth annotations
#if isinstance(cm, chip_match.ChipMatch):
assert qreq_ is not None
matchable_aids = qreq_.daids
#matchable_aids = cm.daid_list
#else:
# matchable_aids = cm.daids
#matchable_aids = ibs.get_recognition_database_aids()
#matchable_aids = list(cm.aid2_fm.keys())
_gtaids = ibs.get_annot_groundtruth(cm.qaid, daid_list=matchable_aids)
if viz_name_score:
# Only look at the groundtruth if a name isnt in the top list
_gtnids = ibs.get_annot_name_rowids(_gtaids)
top_nids = ibs.get_annot_name_rowids(top_aids)
_valids = ~np.in1d(_gtnids, top_nids)
_gtaids = ut.compress(_gtaids, _valids)
# No need to display highly ranked groundtruth. It will already show up
_gtaids = np.setdiff1d(_gtaids, top_aids)
# Sort missed grountruth by score
_gtscores = cm.get_annot_scores(_gtaids)
_gtaids = ut.sortedby(_gtaids, _gtscores, reverse=True)
if viz_name_score:
if len(_gtaids) > 1:
_gtaids = _gtaids[0:1]
else:
if len(_gtaids) > 3:
# Hack to not show too many unmatched groundtruths
#_isexmp = ibs.get_annot_exemplar_flags(_gtaids)
_gtaids = _gtaids[0:3]
showgt_aids = _gtaids
if show_gf:
# Show only one top-scoring groundfalse example
top_nids = ibs.get_annot_name_rowids(top_aids)
is_groundfalse = top_nids != ibs.get_annot_name_rowids(cm.qaid)
gf_idxs = np.nonzero(is_groundfalse)[0]
if len(gf_idxs) > 0:
best_gf_idx = gf_idxs[0]
isvalid = ~is_groundfalse
isvalid[best_gf_idx] = True
# Filter so there is only one groundfalse
top_aids = top_aids.compress(isvalid)
else:
# seems like there were no results. Must be bad feature detections
# maybe too much spatial verification
top_aids = []
if len(showgt_aids) != 0:
# Hack to just include gtaids in normal list
top_aids = np.append(top_aids, showgt_aids)
showgt_aids = []
if viz_name_score:
# Make sure that there is only one of each name in the list
top_nids = ibs.get_annot_name_rowids(top_aids)
top_aids = ut.compress(top_aids, ut.flag_unique_items(top_nids))
return show_qres(ibs, cm, gt_aids=showgt_aids, top_aids=top_aids,
figtitle=figtitle, show_query=show_query, qreq_=qreq_,
failed_to_match=failed_to_match,
**kwargs)
def confirm(self, event=None):
"""
CommandLine:
python -m ibeis.viz.interact.interact_query_decision --test-confirm
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.interact.interact_query_decision import * # NOQA
>>> import utool as ut
>>> # build test data
>>> import ibeis
>>> ibs = ibeis.opendb('testdb1')
>>> self = ibs
>>> self.ibs = ibs
>>> selected_aids = ut.get_list_column(ibs.get_name_aids(ibs.get_valid_nids()), 0)
>>> comfirm_res = 'jeff'
>>> # execute function
>>> #result = self.confirm(event)
>>> # verify results
>>> #print(result)
"""
print('[interact_query_decision] Confirming selected animals.')
selected_aids = [aid for aid in self.comp_aids
if aid is not None and self.aid_checkbox_states[aid]]
if len(selected_aids) == 0:
print('[interact_query_decision] Confirming no match.')
chosen_aids = []
if self.other_checkbox_states['none']:
chosen_aids = 'newname'
elif self.other_checkbox_states['junk']:
chosen_aids = 'junk'
else:
msg = 'INTERACT_QUERY_DECISION IMPOSSIBLE STATE'
raise AssertionError(msg)
elif len(selected_aids) == 1:
print('[interact_query_decision] Confirming single match')
chosen_aids = selected_aids
else:
print('[interact_query_decision] Confirming merge')
msg = ut.textblock(
'''
You have selected more than one animal as a match to the query
animal. By doing this you are telling IBEIS that these are ALL
the SAME ANIMAL. \n\n\nIf this is not what you want, click
Cancel. If it is what you want, choose one of the names below
as the name to keep.
''')
selected_names = self.ibs.get_annot_names(selected_aids)
options = selected_names
parent = None
title = 'Confirm Merge'
merge_name = guitool.user_option(parent, msg=msg, title=title,
options=options)
if merge_name is None:
print('[interact_query_decision] cancelled merge')
self.update_callback()
self.backend_callback()
self.show_page()
return
else:
print('[interact_query_decision] confirmed merge')
is_merge_name = [merge_name == name_ for name_ in selected_names]
chosen_aids = ut.sortedby(selected_aids, is_merge_name)[::-1]
print('[interact_query_decision] Calling update callbacks')
self.update_callback()
self.backend_callback()
print('[interact_query_decision] Calling decision callback')
print('[interact_query_decision] self.name_decision_callback = %r' % (self.name_decision_callback,))
if isinstance(chosen_aids, six.string_types):
# hack for string non-match commands
chosen_names = chosen_aids
else:
chosen_names = self.ibs.get_annot_names(chosen_aids)
self.name_decision_callback(chosen_names)
print('[interact_query_decision] sent name_decision_callback(chosen_names=%r)' % (chosen_names,))
def sort_module_functions():
from os.path import dirname, join
import utool as ut
import ibeis.control
import re
#import re
#regex = r'[^@]*\ndef'
modfpath = dirname(ibeis.control.__file__)
fpath = join(modfpath, 'manual_annot_funcs.py')
#fpath = join(modfpath, 'manual_dependant_funcs.py')
#fpath = join(modfpath, 'manual_lblannot_funcs.py')
#fpath = join(modfpath, 'manual_name_species_funcs.py')
text = ut.read_from(fpath, verbose=False)
lines = text.splitlines()
indent_list = [ut.get_indentation(line) for line in lines]
isfunc_list = [line.startswith('def ') for line in lines]
isblank_list = [len(line.strip(' ')) == 0 for line in lines]
isdec_list = [line.startswith('@') for line in lines]
tmp = ['def' if isfunc else indent for isfunc, indent in zip(isfunc_list, indent_list)]
tmp = ['b' if isblank else t for isblank, t in zip(isblank_list, tmp)]
tmp = ['@' if isdec else t for isdec, t in zip(isdec_list, tmp)]
#print('\n'.join([str((t, count + 1)) for (count, t) in enumerate(tmp)]))
block_list = re.split('\n\n\n', text, flags=re.MULTILINE)
#for block in block_list:
# print('#====')
# print(block)
isfunc_list = [re.search('^def ', block, re.MULTILINE) is not None for block in block_list]
whole_varname = ut.whole_word(ut.REGEX_VARNAME)
funcname_regex = r'def\s+' + ut.named_field('funcname', whole_varname)
def findfuncname(block):
match = re.search(funcname_regex, block)
return match.group('funcname')
funcnameblock_list = [findfuncname(block) if isfunc else None
for isfunc, block in zip(isfunc_list, block_list)]
funcblock_list = ut.filter_items(block_list, isfunc_list)
funcname_list = ut.filter_items(funcnameblock_list, isfunc_list)
nonfunc_list = ut.filterfalse_items(block_list, isfunc_list)
nonfunc_list = ut.filterfalse_items(block_list, isfunc_list)
ismain_list = [re.search('^if __name__ == ["\']__main__["\']', nonfunc) is not None
for nonfunc in nonfunc_list]
mainblock_list = ut.filter_items(nonfunc_list, ismain_list)
nonfunc_list = ut.filterfalse_items(nonfunc_list, ismain_list)
newtext_list = []
for nonfunc in nonfunc_list:
newtext_list.append(nonfunc)
newtext_list.append('\n')
#funcname_list
for funcblock in ut.sortedby(funcblock_list, funcname_list):
newtext_list.append(funcblock)
newtext_list.append('\n')
for mainblock in mainblock_list:
newtext_list.append(mainblock)
newtext = '\n'.join(newtext_list)
print('newtext = %s' % (newtext,))
print('len(newtext) = %r' % (len(newtext),))
print('len(text) = %r' % (len(text),))
backup_fpath = ut.augpath(fpath, augext='.bak', augdir='_backup', ensure=True)
ut.write_to(backup_fpath, text)
ut.write_to(fpath, newtext)
def make_clusters(infr):
import itertools
import networkx as nx
cm_list = infr.cm_list
graph = infr.make_graph()
# hack for orig aids
orig_aid2_nid = {}
for cm in cm_list:
orig_aid2_nid[cm.qaid] = cm.qnid
for daid, dnid in zip(cm.daid_list, cm.dnid_list):
orig_aid2_nid[daid] = dnid
cluster_aids = []
cluster_nids = []
connected = list(nx.connected_components(graph))
for comp in connected:
cluster_nids.append([])
cluster_aids.append([])
for x in comp:
if isinstance(x, tuple):
cluster_nids[-1].append(x[1])
else:
cluster_aids[-1].append(x)
# Make first part of inference output
qaid_list = [cm.qaid for cm in cm_list]
qaid_set = set(qaid_list)
#start_nid = 9001
# Find an nid that doesn't exist in the database
start_nid = len(infr.qreq_.ibs._get_all_known_name_rowids()) + 1
next_new_nid = itertools.count(start_nid)
cluster_tuples = []
for aids, nids in zip(cluster_aids, cluster_nids):
other_nid_clusters = cluster_nids[:]
other_nid_clusters.remove(nids)
other_nids = ut.flatten(other_nid_clusters)
split_case = len(ut.list_intersection(other_nids, nids)) > 0
merge_case = len(nids) > 1
new_name = len(nids) == 0
#print('[chip_match > AnnotInference > make_inference] WARNING:
# EXEMPLAR FLAG SET TO TRUE, NEEDS TO BE IMPLEMENTED')
error_flag = (split_case << 1) + (merge_case << 2) + (
new_name << 3)
strflags = ['split', 'merge', 'new']
error_flag = ut.compress(strflags,
[split_case, merge_case, new_name])
#error_flag = split_case or merge_case
# <HACK>
# SET EXEMPLARS
ibs = infr.qreq_.ibs
viewpoint_texts = ibs.get_annot_yaw_texts(aids)
view_to_aids = ut.group_items(aids, viewpoint_texts)
num_wanted_exemplars_per_view = 4
hack_set_these_qaids_as_exemplars = set([])
for view, aids_ in view_to_aids.items():
heuristic_exemplar_aids = set(aids) - qaid_set
heuristic_non_exemplar_aids = set(aids).intersection(qaid_set)
num_needed_exemplars = (num_wanted_exemplars_per_view -
len(heuristic_exemplar_aids))
# Choose the best query annots to fill out exemplars
if len(heuristic_non_exemplar_aids) == 0:
continue
quality_ints = ibs.get_annot_qualities(
heuristic_non_exemplar_aids)
okish = ibs.const.QUALITY_TEXT_TO_INT[ibs.const.QUAL_OK] - .1
quality_ints = [
x if x is None else okish for x in quality_ints
]
aids_ = ut.sortedby(heuristic_non_exemplar_aids,
quality_ints)[::-1]
chosen = aids_[:num_needed_exemplars]
for qaid_ in chosen:
hack_set_these_qaids_as_exemplars.add(qaid_)
# </HACK>
if not error_flag and not new_name:
new_nid = nids[0]
else:
new_nid = six.next(next_new_nid)
for aid in aids:
if aid not in qaid_set:
if len(error_flag) == 0:
continue
orig_nid = orig_aid2_nid[aid]
exemplar_flag = aid in hack_set_these_qaids_as_exemplars
#clusters is list 4 tuple: (aid, orig_name_uuid, new_name_uuid, error_flag)
tup = (aid, orig_nid, new_nid, exemplar_flag, error_flag)
cluster_tuples.append(tup)
return cluster_tuples
def split_analysis(ibs):
"""
CommandLine:
python -m ibeis.other.dbinfo split_analysis --show
python -m ibeis split_analysis --show
python -m ibeis split_analysis --show --good
Ignore:
# mount
sshfs -o idmap=user lev:/ ~/lev
# unmount
fusermount -u ~/lev
Example:
>>> # DISABLE_DOCTEST GGR
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> dbdir = '/media/danger/GGR/GGR-IBEIS'
>>> dbdir = dbdir if ut.checkpath(dbdir) else ut.truepath('~/lev/media/danger/GGR/GGR-IBEIS')
>>> ibs = ibeis.opendb(dbdir=dbdir, allow_newdir=False)
>>> import guitool_ibeis as gt
>>> gt.ensure_qtapp()
>>> win = split_analysis(ibs)
>>> ut.quit_if_noshow()
>>> import plottool_ibeis as pt
>>> gt.qtapp_loop(qwin=win)
>>> #ut.show_if_requested()
"""
#nid_list = ibs.get_valid_nids(filter_empty=True)
import datetime
day1 = datetime.date(2016, 1, 30)
day2 = datetime.date(2016, 1, 31)
filter_kw = {
'multiple': None,
#'view': ['right'],
#'minqual': 'good',
'is_known': True,
'min_pername': 1,
}
aids1 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day1, 1.0)),
})
)
aids2 = ibs.filter_annots_general(filter_kw=ut.dict_union(
filter_kw, {
'min_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 0.0)),
'max_unixtime': ut.datetime_to_posixtime(ut.date_to_datetime(day2, 1.0)),
})
)
all_aids = aids1 + aids2
all_annots = ibs.annots(all_aids)
print('%d annots on day 1' % (len(aids1)) )
print('%d annots on day 2' % (len(aids2)) )
print('%d annots overall' % (len(all_annots)) )
print('%d names overall' % (len(ut.unique(all_annots.nids))) )
nid_list, annots_list = all_annots.group(all_annots.nids)
REVIEWED_EDGES = True
if REVIEWED_EDGES:
aids_list = [annots.aids for annots in annots_list]
#aid_pairs = [annots.get_am_aidpairs() for annots in annots_list] # Slower
aid_pairs = ibs.get_unflat_am_aidpairs(aids_list) # Faster
else:
# ALL EDGES
aid_pairs = [annots.get_aidpairs() for annots in annots_list]
speeds_list = ibs.unflat_map(ibs.get_annotpair_speeds, aid_pairs)
import vtool_ibeis as vt
max_speeds = np.array([vt.safe_max(s, nans=False) for s in speeds_list])
nan_idx = np.where(np.isnan(max_speeds))[0]
inf_idx = np.where(np.isinf(max_speeds))[0]
bad_idx = sorted(ut.unique(ut.flatten([inf_idx, nan_idx])))
ok_idx = ut.index_complement(bad_idx, len(max_speeds))
print('#nan_idx = %r' % (len(nan_idx),))
print('#inf_idx = %r' % (len(inf_idx),))
print('#ok_idx = %r' % (len(ok_idx),))
ok_speeds = max_speeds[ok_idx]
ok_nids = ut.take(nid_list, ok_idx)
ok_annots = ut.take(annots_list, ok_idx)
sortx = np.argsort(ok_speeds)[::-1]
sorted_speeds = np.array(ut.take(ok_speeds, sortx))
sorted_annots = np.array(ut.take(ok_annots, sortx))
sorted_nids = np.array(ut.take(ok_nids, sortx)) # NOQA
sorted_speeds = np.clip(sorted_speeds, 0, 100)
#idx = vt.find_elbow_point(sorted_speeds)
#EXCESSIVE_SPEED = sorted_speeds[idx]
# http://www.infoplease.com/ipa/A0004737.html
# http://www.speedofanimals.com/animals/zebra
#ZEBRA_SPEED_MAX = 64 # km/h
#ZEBRA_SPEED_RUN = 50 # km/h
ZEBRA_SPEED_SLOW_RUN = 20 # km/h
#ZEBRA_SPEED_FAST_WALK = 10 # km/h
#ZEBRA_SPEED_WALK = 7 # km/h
MAX_SPEED = ZEBRA_SPEED_SLOW_RUN
#MAX_SPEED = ZEBRA_SPEED_WALK
#MAX_SPEED = EXCESSIVE_SPEED
flags = sorted_speeds > MAX_SPEED
flagged_ok_annots = ut.compress(sorted_annots, flags)
inf_annots = ut.take(annots_list, inf_idx)
flagged_annots = inf_annots + flagged_ok_annots
print('MAX_SPEED = %r km/h' % (MAX_SPEED,))
print('%d annots with infinite speed' % (len(inf_annots),))
print('%d annots with large speed' % (len(flagged_ok_annots),))
print('Marking all pairs of annots above the threshold as non-matching')
from ibeis.algo.graph import graph_iden
import networkx as nx
progkw = dict(freq=1, bs=True, est_window=len(flagged_annots))
bad_edges_list = []
good_edges_list = []
for annots in ut.ProgIter(flagged_annots, lbl='flag speeding names', **progkw):
edge_to_speeds = annots.get_speeds()
bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
good_edges = [edge for edge, speed in edge_to_speeds.items() if speed <= MAX_SPEED]
bad_edges_list.append(bad_edges)
good_edges_list.append(good_edges)
all_bad_edges = ut.flatten(bad_edges_list)
good_edges_list = ut.flatten(good_edges_list)
print('num_bad_edges = %r' % (len(ut.flatten(bad_edges_list)),))
print('num_bad_edges = %r' % (len(ut.flatten(good_edges_list)),))
if 1:
from ibeis.viz import viz_graph2
import guitool_ibeis as gt
gt.ensure_qtapp()
if ut.get_argflag('--good'):
print('Looking at GOOD (no speed problems) edges')
aid_pairs = good_edges_list
else:
print('Looking at BAD (speed problems) edges')
aid_pairs = all_bad_edges
aids = sorted(list(set(ut.flatten(aid_pairs))))
infr = graph_iden.AnnotInference(ibs, aids, verbose=False)
infr.initialize_graph()
# Use random scores to randomize sort order
rng = np.random.RandomState(0)
scores = (-rng.rand(len(aid_pairs)) * 10).tolist()
infr.graph.add_edges_from(aid_pairs)
if True:
edge_sample_size = 250
pop_nids = ut.unique(ibs.get_annot_nids(ut.unique(ut.flatten(aid_pairs))))
sorted_pairs = ut.sortedby(aid_pairs, scores)[::-1][0:edge_sample_size]
sorted_nids = ibs.get_annot_nids(ut.take_column(sorted_pairs, 0))
sample_size = len(ut.unique(sorted_nids))
am_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(*sorted_pairs))
flags = ut.not_list(ut.flag_None_items(am_rowids))
#am_rowids = ut.compress(am_rowids, flags)
positive_tags = ['SplitCase', 'Photobomb']
flags_list = [ut.replace_nones(ibs.get_annotmatch_prop(tag, am_rowids), 0)
for tag in positive_tags]
print('edge_case_hist: ' + ut.repr3(
['%s %s' % (txt, sum(flags_)) for flags_, txt in zip(flags_list, positive_tags)]))
is_positive = ut.or_lists(*flags_list)
num_positive = sum(ut.lmap(any, ut.group_items(is_positive, sorted_nids).values()))
pop = len(pop_nids)
print('A positive is any edge flagged as a %s' % (ut.conj_phrase(positive_tags, 'or'),))
print('--- Sampling wrt edges ---')
print('edge_sample_size = %r' % (edge_sample_size,))
print('edge_population_size = %r' % (len(aid_pairs),))
print('num_positive_edges = %r' % (sum(is_positive)))
print('--- Sampling wrt names ---')
print('name_population_size = %r' % (pop,))
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level=.95)
nx.set_edge_attributes(infr.graph, name='score', values=dict(zip(aid_pairs, scores)))
win = viz_graph2.AnnotGraphWidget(infr=infr, use_image=False,
init_mode=None)
win.populate_edge_model()
win.show()
return win
# Make review interface for only bad edges
infr_list = []
iter_ = list(zip(flagged_annots, bad_edges_list))
for annots, bad_edges in ut.ProgIter(iter_, lbl='creating inference', **progkw):
aids = annots.aids
nids = [1] * len(aids)
infr = graph_iden.AnnotInference(ibs, aids, nids, verbose=False)
infr.initialize_graph()
infr.reset_feedback()
infr_list.append(infr)
# Check which ones are user defined as incorrect
#num_positive = 0
#for infr in infr_list:
# flag = np.any(infr.get_feedback_probs()[0] == 0)
# num_positive += flag
#print('num_positive = %r' % (num_positive,))
#pop = len(infr_list)
#print('pop = %r' % (pop,))
iter_ = list(zip(infr_list, bad_edges_list))
for infr, bad_edges in ut.ProgIter(iter_, lbl='adding speed edges', **progkw):
flipped_edges = []
for aid1, aid2 in bad_edges:
if infr.graph.has_edge(aid1, aid2):
flipped_edges.append((aid1, aid2))
infr.add_feedback((aid1, aid2), NEGTV)
nx.set_edge_attributes(infr.graph, name='_speed_split', values='orig')
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'new' for edge in bad_edges})
nx.set_edge_attributes(infr.graph, name='_speed_split', values={edge: 'flip' for edge in flipped_edges})
#for infr in ut.ProgIter(infr_list, lbl='flagging speeding edges', **progkw):
# annots = ibs.annots(infr.aids)
# edge_to_speeds = annots.get_speeds()
# bad_edges = [edge for edge, speed in edge_to_speeds.items() if speed > MAX_SPEED]
def inference_stats(infr_list_):
relabel_stats = []
for infr in infr_list_:
num_ccs, num_inconsistent = infr.relabel_using_reviews()
state_hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, 'decision').values())
if POSTV not in state_hist:
state_hist[POSTV] = 0
hist = ut.dict_hist(nx.get_edge_attributes(infr.graph, '_speed_split').values())
subgraphs = infr.positive_connected_compoments()
subgraph_sizes = [len(g) for g in subgraphs]
info = ut.odict([
('num_nonmatch_edges', state_hist[NEGTV]),
('num_match_edges', state_hist[POSTV]),
('frac_nonmatch_edges', state_hist[NEGTV] / (state_hist[POSTV] + state_hist[NEGTV])),
('num_inconsistent', num_inconsistent),
('num_ccs', num_ccs),
('edges_flipped', hist.get('flip', 0)),
('edges_unchanged', hist.get('orig', 0)),
('bad_unreviewed_edges', hist.get('new', 0)),
('orig_size', len(infr.graph)),
('new_sizes', subgraph_sizes),
])
relabel_stats.append(info)
return relabel_stats
relabel_stats = inference_stats(infr_list)
print('\nAll Split Info:')
lines = []
for key in relabel_stats[0].keys():
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_incon_list = np.array(ut.take_column(relabel_stats, 'num_inconsistent'))
can_split_flags = num_incon_list == 0
print('Can trivially split %d / %d' % (sum(can_split_flags), len(can_split_flags)))
splittable_infrs = ut.compress(infr_list, can_split_flags)
relabel_stats = inference_stats(splittable_infrs)
print('\nTrival Split Info:')
lines = []
for key in relabel_stats[0].keys():
if key in ['num_inconsistent']:
continue
data = ut.take_column(relabel_stats, key)
if key == 'new_sizes':
data = ut.flatten(data)
lines.append('stats(%s) = %s' % (
key, ut.repr2(ut.get_stats(data, use_median=True), precision=2)))
print('\n'.join(ut.align_lines(lines, '=')))
num_match_edges = np.array(ut.take_column(relabel_stats, 'num_match_edges'))
num_nonmatch_edges = np.array(ut.take_column(relabel_stats, 'num_nonmatch_edges'))
flags1 = np.logical_and(num_match_edges > num_nonmatch_edges, num_nonmatch_edges < 3)
reasonable_infr = ut.compress(splittable_infrs, flags1)
new_sizes_list = ut.take_column(relabel_stats, 'new_sizes')
flags2 = [len(sizes) == 2 and sum(sizes) > 4 and (min(sizes) / max(sizes)) > .3
for sizes in new_sizes_list]
reasonable_infr = ut.compress(splittable_infrs, flags2)
print('#reasonable_infr = %r' % (len(reasonable_infr),))
for infr in ut.InteractiveIter(reasonable_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
rest = ~np.logical_or(flags1, flags2)
nonreasonable_infr = ut.compress(splittable_infrs, rest)
rng = np.random.RandomState(0)
random_idx = ut.random_indexes(len(nonreasonable_infr) - 1, 15, rng=rng)
random_infr = ut.take(nonreasonable_infr, random_idx)
for infr in ut.InteractiveIter(random_infr):
annots = ibs.annots(infr.aids)
edge_to_speeds = annots.get_speeds()
print('max_speed = %r' % (max(edge_to_speeds.values())),)
infr.initialize_visual_node_attrs()
infr.show_graph(use_image=True, only_reviewed=True)
#import scipy.stats as st
#conf_interval = .95
#st.norm.cdf(conf_interval)
# view-source:http://www.surveysystem.com/sscalc.htm
#zval = 1.96 # 95 percent confidence
#zValC = 3.8416 #
#zValC = 6.6564
#import statsmodels.stats.api as sms
#es = sms.proportion_effectsize(0.5, 0.75)
#sms.NormalIndPower().solve_power(es, power=0.9, alpha=0.05, ratio=1)
pop = 279
num_positive = 3
sample_size = 15
conf_level = .95
#conf_level = .99
vt.calc_error_bars_from_sample(sample_size, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(sample_size + 38 / 3, num_positive, pop, conf_level)
print('---')
vt.calc_error_bars_from_sample(15 + 38, num_positive=3, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(15, num_positive=3, pop=675, conf_level=.95)
pop = 279
#err_frac = .05 # 5%
err_frac = .10 # 10%
conf_level = .95
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
pop = 675
vt.calc_sample_from_error_bars(err_frac, pop, conf_level)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.95, prior=.1)
vt.calc_sample_from_error_bars(.05, pop, conf_level=.68, prior=.2)
vt.calc_sample_from_error_bars(.10, pop, conf_level=.68)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.95)
vt.calc_error_bars_from_sample(100, num_positive=5, pop=675, conf_level=.68)
def show_qres_analysis(ibs, cm, qreq_=None, **kwargs):
"""
Wrapper around show_qres.
KWARGS:
aid_list - show matches against aid_list (default top 3)
Args:
ibs (IBEISController): ibeis controller object
cm (ChipMatch): object of feature correspondences and scores
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
N, show_gt, show_query, aid_list, figtitle, viz_name_score, viz_name_score
CommandLine:
python -m ibeis.viz.viz_qres --exec-show_qres_analysis --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.viz.viz_qres import * # NOQA
>>> import ibeis
>>> species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> daids = ibs.get_valid_aids(species=species)
>>> qaids = ibs.get_valid_aids(species=species)
>>> qres_list, qreq_ = ibs.query_chips(
>>> [1], [2, 3, 4, 5, 6, 7, 8, 9],
>>> cfgdict=dict(), return_request=True)
>>> cm = qres_list[0]
>>> kwargs = dict(show_query=False, viz_name_score=True,
>>> show_timedelta=True, N=3, show_gf=True)
>>> show_qres_analysis(ibs, cm, qreq_, **kwargs)
>>> ut.show_if_requested()
"""
if ut.NOT_QUIET:
print('[show_qres] cm.show_analysis()')
# Parse arguments
N = kwargs.get('N', DEFAULT_NTOP)
show_gt = kwargs.pop('show_gt', True)
show_gf = kwargs.pop('show_gf', False)
show_query = kwargs.pop('show_query', True)
aid_list = kwargs.pop('aid_list', None)
figtitle = kwargs.pop('figtitle', None)
#viz_name_score = kwargs.get('viz_name_score', qreq_ is not None)
#viz_name_score = kwargs.get('viz_name_score', False)
viz_name_score = kwargs.get('viz_name_score', True)
failed_to_match = False
# Debug printing
#print('[analysis] noshow_gt = %r' % noshow_gt)
#print('[analysis] show_query = %r' % show_query)
#print('[analysis] aid_list = %r' % aid_list)
if aid_list is None:
# Compare to aid_list instead of using top ranks
#print('[analysis] showing top aids')
top_aids = cm.get_top_aids(N)
if len(top_aids) == 0:
failed_to_match = True
print('WARNING! No matches found for this query')
if figtitle is None:
if len(top_aids) == 0:
figtitle = 'WARNING: no matches found!' + ibsfuncs.aidstr(
cm.qaid)
else:
topscore = cm.get_annot_scores(top_aids)[0]
figtitle = ('q%s -- topscore=%r' %
(ibsfuncs.aidstr(cm.qaid), topscore))
else:
print('[analysis] showing a given list of aids')
top_aids = aid_list
if figtitle is None:
figtitle = 'comparing to ' + ibsfuncs.aidstr(top_aids) + figtitle
# Get any groundtruth if you are showing it
showgt_aids = []
if show_gt:
# Get the missed groundtruth annotations
#if isinstance(cm, chip_match.ChipMatch):
assert qreq_ is not None
matchable_aids = qreq_.daids
#matchable_aids = cm.daid_list
#else:
# matchable_aids = cm.daids
#matchable_aids = ibs.get_recognition_database_aids()
#matchable_aids = list(cm.aid2_fm.keys())
_gtaids = ibs.get_annot_groundtruth(cm.qaid, daid_list=matchable_aids)
if viz_name_score:
# Only look at the groundtruth if a name isnt in the top list
_gtnids = ibs.get_annot_name_rowids(_gtaids)
top_nids = ibs.get_annot_name_rowids(top_aids)
_valids = ~np.in1d(_gtnids, top_nids)
_gtaids = ut.compress(_gtaids, _valids)
# No need to display highly ranked groundtruth. It will already show up
_gtaids = np.setdiff1d(_gtaids, top_aids)
# Sort missed grountruth by score
_gtscores = cm.get_annot_scores(_gtaids)
_gtaids = ut.sortedby(_gtaids, _gtscores, reverse=True)
if viz_name_score:
if len(_gtaids) > 1:
_gtaids = _gtaids[0:1]
else:
if len(_gtaids) > 3:
# Hack to not show too many unmatched groundtruths
#_isexmp = ibs.get_annot_exemplar_flags(_gtaids)
_gtaids = _gtaids[0:3]
showgt_aids = _gtaids
if show_gf:
# Show only one top-scoring groundfalse example
top_nids = ibs.get_annot_name_rowids(top_aids)
is_groundfalse = top_nids != ibs.get_annot_name_rowids(cm.qaid)
gf_idxs = np.nonzero(is_groundfalse)[0]
if len(gf_idxs) > 0:
best_gf_idx = gf_idxs[0]
isvalid = ~is_groundfalse
isvalid[best_gf_idx] = True
# Filter so there is only one groundfalse
top_aids = top_aids.compress(isvalid)
else:
# seems like there were no results. Must be bad feature detections
# maybe too much spatial verification
top_aids = []
if len(showgt_aids) != 0:
# Hack to just include gtaids in normal list
top_aids = np.append(top_aids, showgt_aids)
showgt_aids = []
if viz_name_score:
# Make sure that there is only one of each name in the list
top_nids = ibs.get_annot_name_rowids(top_aids)
top_aids = ut.compress(top_aids, ut.flag_unique_items(top_nids))
return show_qres(ibs,
cm,
gt_aids=showgt_aids,
top_aids=top_aids,
figtitle=figtitle,
show_query=show_query,
qreq_=qreq_,
failed_to_match=failed_to_match,
**kwargs)
def print_unused():
print(ut.indentjoin(ut.sortedby(unused_keys, map(len, unused_keys))))
print('len(unused_keys) = %r' % (len(unused_keys), ))
def test_model(num_annots, num_names, score_evidence=[], name_evidence=[],
other_evidence={}, noquery=False, verbose=None,
**kwargs):
if verbose is None:
verbose = ut.VERBOSE
method = kwargs.pop('method', None)
model = make_name_model(num_annots, num_names, verbose=verbose, **kwargs)
if verbose:
model.print_priors(ignore_ttypes=['match', 'score'])
model, evidence, soft_evidence = update_model_evidence(
model, name_evidence, score_evidence, other_evidence)
if verbose and len(soft_evidence) != 0:
model.print_priors(ignore_ttypes=['match', 'score'],
title='Soft Evidence', color='green')
#if verbose:
# ut.colorprint('\n --- Soft Evidence ---', 'white')
# for ttype, cpds in model.ttype2_cpds.items():
# if ttype != 'match':
# for fs_ in ut.ichunks(cpds, 4):
# ut.colorprint(ut.hz_str([f._cpdstr('psql') for f in fs_]),
# 'green')
if verbose:
ut.colorprint('\n --- Inference ---', 'red')
if (len(evidence) > 0 or len(soft_evidence) > 0) and not noquery:
evidence = model._ensure_internal_evidence(evidence)
query_vars = []
query_vars += ut.list_getattr(model.ttype2_cpds['name'], 'variable')
#query_vars += ut.list_getattr(model.ttype2_cpds['match'], 'variable')
query_vars = ut.setdiff(query_vars, evidence.keys())
#query_vars = ut.setdiff(query_vars, soft_evidence.keys())
query_results = cluster_query(model, query_vars, evidence,
soft_evidence, method)
else:
query_results = {}
factor_list = query_results['factor_list']
if verbose:
if verbose:
print('+--------')
semtypes = [model.var2_cpd[f.variables[0]].ttype
for f in factor_list]
for type_, factors in ut.group_items(factor_list, semtypes).items():
print('Result Factors (%r)' % (type_,))
factors = ut.sortedby(factors, [f.variables[0] for f in factors])
for fs_ in ut.ichunks(factors, 4):
ut.colorprint(ut.hz_str([f._str('phi', 'psql') for f in fs_]),
'yellow')
print('MAP assignments')
top_assignments = query_results.get('top_assignments', [])
tmp = []
for lbl, val in top_assignments:
tmp.append('%s : %.4f' % (ut.repr2(lbl), val))
print(ut.align('\n'.join(tmp), ' :'))
print('L_____\n')
showkw = dict(evidence=evidence,
soft_evidence=soft_evidence,
**query_results)
pgm_viz.show_model(model, **showkw)
return (model, evidence, query_results)
def dev_train_distinctiveness(species=None):
r"""
Args:
ibs (IBEISController): ibeis controller object
species (None):
CommandLine:
python -m ibeis.algo.hots.distinctiveness_normalizer --test-dev_train_distinctiveness
alias dev_train_distinctiveness='python -m ibeis.algo.hots.distinctiveness_normalizer --test-dev_train_distinctiveness'
# Publishing (uses cached normalizers if available)
dev_train_distinctiveness --species GZ --publish
dev_train_distinctiveness --species PZ --publish
dev_train_distinctiveness --species PZ --retrain
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.distinctiveness_normalizer import * # NOQA
>>> import ibeis
>>> species = ut.get_argval('--species', str, 'zebra_grevys')
>>> dev_train_distinctiveness(species)
"""
import ibeis
#if 'species' not in vars() or species is None:
# species = 'zebra_grevys'
if species == 'zebra_grevys':
dbname = 'GZ_ALL'
elif species == 'zebra_plains':
dbname = 'PZ_Master0'
ibs = ibeis.opendb(dbname)
global_distinctdir = ibs.get_global_distinctiveness_modeldir()
cachedir = global_distinctdir
dstcnvs_normer = DistinctivnessNormalizer(species, cachedir=cachedir)
try:
if ut.get_argflag('--retrain'):
raise IOError('force cache miss')
with ut.Timer('loading distinctiveness'):
dstcnvs_normer.load(cachedir)
# Cache hit
print('distinctivness model cache hit')
except IOError:
print('distinctivness model cache miss')
with ut.Timer('training distinctiveness'):
# Need to train
# Add one example from each name
# TODO: add one exemplar per viewpoint for each name
#max_vecs = 1E6
#max_annots = 975
max_annots = 975
#ibs.fix_and_clean_database()
nid_list = ibs.get_valid_nids()
aids_list = ibs.get_name_aids(nid_list)
# remove junk
aids_list = ibs.unflat_map(ibs.filter_junk_annotations, aids_list)
# remove empty
aids_list = [aids for aids in aids_list if len(aids) > 0]
num_annots_list = list(map(len, aids_list))
aids_list = ut.sortedby(aids_list, num_annots_list, reverse=True)
# take only one annot per name
aid_list = ut.get_list_column(aids_list, 0)
# Keep only a certain number of annots for distinctiveness mapping
aid_list_ = ut.listclip(aid_list, max_annots)
print('total num named annots = %r' % (sum(num_annots_list)))
print('training distinctiveness using %d/%d singleton annots' %
(len(aid_list_), len(aid_list)))
# vec
# FIXME: qreq_ params for config rowid
vecs_list = ibs.get_annot_vecs(aid_list_)
num_vecs = sum(list(map(len, vecs_list)))
print('num_vecs = %r' % (num_vecs,))
vecs = np.vstack(vecs_list)
print('vecs size = %r' % (ut.get_object_size_str(vecs),))
dstcnvs_normer.init_support(vecs)
dstcnvs_normer.save(global_distinctdir)
if ut.get_argflag('--publish'):
dstcnvs_normer.publish()
def get_key_order(cfg):
keys = [k for k in cfg.keys() if k not in nonlbl_keys]
sortorder = [2 * k.startswith('q') + 1 * k.startswith('d')
for k in keys]
return ut.sortedby(keys, sortorder)[::-1]
def confirm(self, event=None):
"""
CommandLine:
python -m ibeis.viz.interact.interact_query_decision --test-confirm
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.interact.interact_query_decision import * # NOQA
>>> import utool as ut
>>> # build test data
>>> import ibeis
>>> ibs = ibeis.opendb('testdb1')
>>> self = ibs
>>> self.ibs = ibs
>>> selected_aids = ut.get_list_column(ibs.get_name_aids(ibs.get_valid_nids()), 0)
>>> comfirm_res = 'jeff'
>>> # execute function
>>> #result = self.confirm(event)
>>> # verify results
>>> #print(result)
"""
print('[interact_query_decision] Confirming selected animals.')
selected_aids = [
aid for aid in self.comp_aids
if aid is not None and self.aid_checkbox_states[aid]
]
if len(selected_aids) == 0:
print('[interact_query_decision] Confirming no match.')
chosen_aids = []
if self.other_checkbox_states['none']:
chosen_aids = 'newname'
elif self.other_checkbox_states['junk']:
chosen_aids = 'junk'
else:
msg = 'INTERACT_QUERY_DECISION IMPOSSIBLE STATE'
raise AssertionError(msg)
elif len(selected_aids) == 1:
print('[interact_query_decision] Confirming single match')
chosen_aids = selected_aids
else:
print('[interact_query_decision] Confirming merge')
msg = ut.textblock('''
You have selected more than one animal as a match to the query
animal. By doing this you are telling IBEIS that these are ALL
the SAME ANIMAL. \n\n\nIf this is not what you want, click
Cancel. If it is what you want, choose one of the names below
as the name to keep.
''')
selected_names = self.ibs.get_annot_names(selected_aids)
options = selected_names
parent = None
title = 'Confirm Merge'
merge_name = guitool.user_option(parent,
msg=msg,
title=title,
options=options)
if merge_name is None:
print('[interact_query_decision] cancelled merge')
self.update_callback()
self.backend_callback()
self.show_page()
return
else:
print('[interact_query_decision] confirmed merge')
is_merge_name = [
merge_name == name_ for name_ in selected_names
]
chosen_aids = ut.sortedby(selected_aids, is_merge_name)[::-1]
print('[interact_query_decision] Calling update callbacks')
self.update_callback()
self.backend_callback()
print('[interact_query_decision] Calling decision callback')
print('[interact_query_decision] self.name_decision_callback = %r' %
(self.name_decision_callback, ))
if isinstance(chosen_aids, six.string_types):
# hack for string non-match commands
chosen_names = chosen_aids
else:
chosen_names = self.ibs.get_annot_names(chosen_aids)
self.name_decision_callback(chosen_names)
print(
'[interact_query_decision] sent name_decision_callback(chosen_names=%r)'
% (chosen_names, ))
def test_zotero_sql():
r"""
"C:\Program Files (x86)\Mozilla Firefox\firefox.exe"
"C:\Program Files (x86)\Mozilla Firefox\firefox.exe" -profile "C:\Users\joncrall\AppData\Roaming\Mozilla\Firefox\Profiles\7kadig32.default" -repl 4242
References:
http://www.cogsci.nl/blog/tutorials/97-writing-a-command-line-zotero-client-in-9-lines-of-code
https://forums.zotero.org/discussion/2919/command-line-export-to-bib-file/
http://www.curiousjason.com/zoterotobibtex.html
https://addons.mozilla.org/en-US/firefox/addon/mozrepl/
# bibtex plugin
https://github.com/ZotPlus/zotero-better-bibtex
https://groups.google.com/forum/#!forum/zotero-dev
Ignore:
C:\Users\joncrall\AppData\Roaming\Zotero\Zotero\Profiles\xrmkwlkz.default\zotero\translators
"""
cur = zotero.cur # NOQA
#ut.rrrr()
# ENTIRE DATABASE INFO
ut.print_database_structure(cur)
tablename_list = ut.get_tablenames(cur)
colinfos_list = [ut.get_table_columninfo_list(cur, tablename) for tablename in tablename_list] # NOQA
numrows_list = [ut.get_table_num_rows(cur, tablename) for tablename in tablename_list] # NOQA
tablename = 'items'
colnames = ('itemID',) # NOQA
colinfo_list = ut.get_table_columninfo_list(cur, tablename) # NOQA
itemtype_id_list = ut.get_table_rows(cur, 'items', ('itemTypeID',))
ut.get_table_columninfo_list(cur, 'itemTypeFields')
ut.get_table_rows(cur, 'itemTypeFields', ('fieldID',), where='itemTypeID=?', params=itemtype_ids) # NOQA
ut.get_table_rows(cur, 'itemTypeFields', ('orderIndex',), where='itemTypeID=?', params=itemtype_ids) # NOQA
ut.get_table_rows(cur, 'itemTypeFields', ('',), where='itemTypeID=?', params=itemtype_ids) # NOQA
itemData # NOQA
# Item Table INFO
ut.get_table_columninfo_list(cur, 'tags')
ut.get_table_columninfo_list(cur, 'items')
ut.get_table_columninfo_list(cur, 'itemTypeFields')
ut.get_table_columninfo_list(cur, 'itemData')
ut.get_table_columninfo_list(cur, 'itemDataValues')
ut.get_table_columninfo_list(cur, 'fields')
ut.get_table_columninfo_list(cur, 'fieldsCombined')
ut.get_table_rows(cur, 'fields', ('fieldName',))
# The ID of each item in the database
itemid_list = ut.get_table_rows(cur, 'items', ('itemID',))
# The type of each item
itemtype_id_list = ut.get_list_column(ut.get_table_rows(cur, 'items', ('itemTypeID',), where='itemID=?', params=itemid_list), 0)
# The different types of items
itemtype_ids = list(set(itemtype_id_list))
# The fields of each item type
fieldids_list_ = ut.get_table_rows(cur, 'itemTypeFields', ('fieldID',), where='itemTypeID=?', params=itemtype_ids)
orderids_list_ = ut.get_table_rows(cur, 'itemTypeFields', ('orderIndex',), where='itemTypeID=?', params=itemtype_ids)
fieldids_list = [ut.sortedby(f, o) for f, o in zip(fieldids_list_, orderids_list_)]
itemtypeid2_fields = dict(zip(itemtype_ids, fieldids_list))
itemid_fieldids_list = [[(itemID[0], fieldID[0]) for fieldID in itemtypeid2_fields[itemTypeID]] for itemID, itemTypeID in list(zip(itemid_list, itemtype_id_list))[0:7]]
flat_list, cumsum_list = ut.invertible_flatten2(itemid_fieldids_list)
# Get field values
flat_valueID_list = ut.get_table_rows(cur, 'itemData', ('valueID',), where='itemID=? and fieldID=?', params=flat_list)
valueIDs_list = ut.unflatten2(flat_valueID_list, cumsum_list)
filtered_itemid_fieldids_list = [[if_ for if_, v in zip(ifs, vs) if len(v) > 0] for ifs, vs in zip(itemid_fieldids_list, valueIDs_list)]
filtered_flat_list, filtered_cumsum_list = ut.invertible_flatten2(filtered_itemid_fieldids_list)
# Get field values
filt_flat_valueID_list = ut.get_table_rows(cur, 'itemData', ('valueID',), where='itemID=? and fieldID=?', params=filtered_flat_list)
filt_flat_valueID_list_ = ut.get_list_column(filt_flat_valueID_list, 0)
filt_flat_fieldname_list = ut.get_table_rows(cur, 'fields', ('fieldName',), where='fieldID=?', params=ut.get_list_column(filtered_flat_list, [1]))
filt_flat_value_list = ut.get_table_rows(cur, 'itemDataValues', ('value',), where='valueID=?', params=filt_flat_valueID_list_) # NOQA
#
filt_fieldname_list = ut.unflatten2(filt_flat_fieldname_list, filtered_cumsum_list) # NOQA
filt_valueIDs_list = ut.unflatten2(filt_flat_valueID_list, filtered_cumsum_list) # NOQA
ut.get_table_rows(cur, 'itemTypeFields', ('fieldID', 'orderIndex'), where='itemTypeID=?', params=itemtype_ids)
all_values = ut.get_list_column(ut.get_table_rows(cur, 'itemDataValues', ('value',)), 0)
import re
import six
for value in all_values:
if isinstance(value, six.string_types) and re.search('CVPR', value):
print(value)
def sort_module_functions():
from os.path import dirname, join
import utool as ut
import ibeis.control
import re
#import re
#regex = r'[^@]*\ndef'
modfpath = dirname(ibeis.control.__file__)
fpath = join(modfpath, 'manual_annot_funcs.py')
#fpath = join(modfpath, 'manual_dependant_funcs.py')
#fpath = join(modfpath, 'manual_lblannot_funcs.py')
#fpath = join(modfpath, 'manual_name_species_funcs.py')
text = ut.read_from(fpath, verbose=False)
lines = text.splitlines()
indent_list = [ut.get_indentation(line) for line in lines]
isfunc_list = [line.startswith('def ') for line in lines]
isblank_list = [len(line.strip(' ')) == 0 for line in lines]
isdec_list = [line.startswith('@') for line in lines]
tmp = [
'def' if isfunc else indent
for isfunc, indent in zip(isfunc_list, indent_list)
]
tmp = ['b' if isblank else t for isblank, t in zip(isblank_list, tmp)]
tmp = ['@' if isdec else t for isdec, t in zip(isdec_list, tmp)]
#print('\n'.join([str((t, count + 1)) for (count, t) in enumerate(tmp)]))
block_list = re.split('\n\n\n', text, flags=re.MULTILINE)
#for block in block_list:
# print('#====')
# print(block)
isfunc_list = [
re.search('^def ', block, re.MULTILINE) is not None
for block in block_list
]
whole_varname = ut.whole_word(ut.REGEX_VARNAME)
funcname_regex = r'def\s+' + ut.named_field('funcname', whole_varname)
def findfuncname(block):
match = re.search(funcname_regex, block)
return match.group('funcname')
funcnameblock_list = [
findfuncname(block) if isfunc else None
for isfunc, block in zip(isfunc_list, block_list)
]
funcblock_list = ut.filter_items(block_list, isfunc_list)
funcname_list = ut.filter_items(funcnameblock_list, isfunc_list)
nonfunc_list = ut.filterfalse_items(block_list, isfunc_list)
nonfunc_list = ut.filterfalse_items(block_list, isfunc_list)
ismain_list = [
re.search('^if __name__ == ["\']__main__["\']', nonfunc) is not None
for nonfunc in nonfunc_list
]
mainblock_list = ut.filter_items(nonfunc_list, ismain_list)
nonfunc_list = ut.filterfalse_items(nonfunc_list, ismain_list)
newtext_list = []
for nonfunc in nonfunc_list:
newtext_list.append(nonfunc)
newtext_list.append('\n')
#funcname_list
for funcblock in ut.sortedby(funcblock_list, funcname_list):
newtext_list.append(funcblock)
newtext_list.append('\n')
for mainblock in mainblock_list:
newtext_list.append(mainblock)
newtext = '\n'.join(newtext_list)
print('newtext = %s' % (newtext, ))
print('len(newtext) = %r' % (len(newtext), ))
print('len(text) = %r' % (len(text), ))
backup_fpath = ut.augpath(fpath,
augext='.bak',
augdir='_backup',
ensure=True)
ut.write_to(backup_fpath, text)
ut.write_to(fpath, newtext)
