def delete_features(ibs, feat_rowid_list, config2_=None):
""" deletes images from the database that belong to fids"""
from ibeis.algo.preproc import preproc_feat
if ut.VERBOSE:
print('[ibs] deleting %d features' % len(feat_rowid_list))
# remove non-sql external dependeinces of these rowids
preproc_feat.on_delete(ibs, feat_rowid_list)
# remove dependants of these rowids
featweight_rowid_list = ut.filter_Nones(ibs.get_feat_featweight_rowids(feat_rowid_list, config2_=config2_, ensure=False))
ibs.delete_featweight(featweight_rowid_list)
# remove these rowids
ibs.dbcache.delete_rowids(const.FEATURE_TABLE, feat_rowid_list)
num_deleted = len(ut.filter_Nones(feat_rowid_list))
return num_deleted
def delete_annot_featweight(ibs, aid_list, config2_=None):
""" annot.featweight.delete(aid_list)
Args:
aid_list
TemplateInfo:
Tdeleter_rl_depenant
root = annot
leaf = featweight
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control._autogen_featweight_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> aid_list = ibs._get_all_aids()[:1]
>>> # Ensure there are some leafs to delete
>>> featweight_rowid_list = ibs.get_annot_featweight_rowids(aid_list, config2_=config2_, ensure=True)
>>> num_deleted1 = ibs.delete_annot_featweight(aid_list, config2_=config2_)
>>> num_deleted2 = ibs.delete_annot_featweight(aid_list, config2_=config2_)
>>> # The first delete should remove everything
>>> ut.assert_eq(num_deleted1, len(featweight_rowid_list))
>>> # The second delete should have removed nothing
>>> ut.assert_eq(num_deleted2, 0)
"""
if ut.VERBOSE:
print('[ibs] deleting %d annots leaf nodes' % len(aid_list))
# Delete any dependants
_featweight_rowid_list = ibs.get_annot_featweight_rowids(
aid_list, config2_=config2_, ensure=False)
featweight_rowid_list = ut.filter_Nones(_featweight_rowid_list)
num_deleted = ibs.delete_featweight(featweight_rowid_list)
return num_deleted
def delete_annotmatch(ibs, annotmatch_rowid_list):
r""" annotmatch.delete(annotmatch_rowid_list)
delete annotmatch rows
Args:
annotmatch_rowid_list
Returns:
int: num_deleted
TemplateInfo:
Tdeleter_native_tbl
tbl = annotmatch
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control.manual_annotmatch_funcs import * # NOQA
>>> ibs, config2_ = testdata_annotmatch()
>>> annotmatch_rowid_list = ibs._get_all_annotmatch_rowids()[:2]
>>> num_deleted = ibs.delete_annotmatch(annotmatch_rowid_list)
>>> print('num_deleted = %r' % (num_deleted,))
"""
#from ibeis.algo.preproc import preproc_annotmatch
# NO EXTERN IMPORT
if ut.VERBOSE:
print('[ibs] deleting %d annotmatch rows' % len(annotmatch_rowid_list))
# Prepare: Delete externally stored data (if any)
#preproc_annotmatch.on_delete(ibs, annotmatch_rowid_list, config2_=config2_)
# NO EXTERN DELETE
# Finalize: Delete self
ibs.db.delete_rowids(const.ANNOTMATCH_TABLE, annotmatch_rowid_list)
num_deleted = len(ut.filter_Nones(annotmatch_rowid_list))
return num_deleted
def delete_annotmatch(ibs, annotmatch_rowid_list, config2_=None):
r""" annotmatch.delete(annotmatch_rowid_list)
delete annotmatch rows
Args:
annotmatch_rowid_list
Returns:
int: num_deleted
TemplateInfo:
Tdeleter_native_tbl
tbl = annotmatch
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control._autogen_annotmatch_funcs import * # NOQA
>>> ibs, config2_ = testdata_annotmatch()
>>> annotmatch_rowid_list = ibs._get_all_annotmatch_rowids()[:2]
>>> num_deleted = ibs.delete_annotmatch(annotmatch_rowid_list)
>>> print('num_deleted = %r' % (num_deleted,))
"""
#from ibeis.algo.preproc import preproc_annotmatch
# NO EXTERN IMPORT
if ut.VERBOSE:
print('[ibs] deleting %d annotmatch rows' % len(annotmatch_rowid_list))
# Prepare: Delete externally stored data (if any)
#preproc_annotmatch.on_delete(ibs, annotmatch_rowid_list, config2_=config2_)
# NO EXTERN DELETE
# Finalize: Delete self
ibs.db.delete_rowids(const.ANNOTMATCH_TABLE, annotmatch_rowid_list)
num_deleted = len(ut.filter_Nones(annotmatch_rowid_list))
return num_deleted
def delete_annot_feats(ibs, aid_list, config2_=None):
""" annot.feat.delete(aid_list)
Args:
aid_list
TemplateInfo:
Tdeleter_rl_depenant
root = annot
leaf = feat
CommandLine:
python -m ibeis.control.manual_feat_funcs --test-delete_annot_feats
python -m ibeis.control.manual_feat_funcs --test-delete_annot_feats --verb-control
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control.manual_feat_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> aid_list = ibs._get_all_aids()[:1]
>>> fids_list = ibs.get_annot_feat_rowids(aid_list, config2_=config2_, ensure=True)
>>> num_deleted1 = ibs.delete_annot_feats(aid_list, config2_=config2_)
>>> ut.assert_eq(num_deleted1, len(fids_list))
>>> num_deleted2 = ibs.delete_annot_feats(aid_list, config2_=config2_)
>>> ut.assert_eq(num_deleted2, 0)
"""
if ut.VERBOSE:
print('[ibs] deleting %d annots leaf nodes' % len(aid_list))
# Delete any dependants
_feat_rowid_list = ibs.get_annot_feat_rowids(
aid_list, config2_=config2_, ensure=False)
feat_rowid_list = ut.filter_Nones(_feat_rowid_list)
num_deleted = ibs.delete_feat(feat_rowid_list)
return num_deleted
def delete_chip_feats(ibs, chip_rowid_list, config2_=None):
""" chip.feat.delete(chip_rowid_list)
Args:
chip_rowid_list
TemplateInfo:
Tdeleter_rl_depenant
parent = chip
leaf = feat
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control.manual_feat_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> chip_rowid_list = ibs._get_all_chip_rowids()[::3]
>>> ibs.delete_chip_feats(chip_rowid_list, config2_=config2_)
"""
if ut.VERBOSE:
print('[ibs] deleting %d chips leaf nodes' % len(chip_rowid_list))
# Delete any dependants
_feat_rowid_list = ibs.get_chip_feat_rowid(
chip_rowid_list, config2_=config2_, ensure=False)
feat_rowid_list = ut.filter_Nones(_feat_rowid_list)
num_deleted = ibs.delete_feat(feat_rowid_list)
return num_deleted
def delete_gar(ibs, gar_rowid_list, config2_=None):
""" gar.delete(gar_rowid_list)
delete gar rows
Args:
gar_rowid_list
Returns:
int: num_deleted
TemplateInfo:
Tdeleter_native_tbl
tbl = gar
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.control.manual_garelate_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> gar_rowid_list = ibs._get_all_gar_rowids()[:2]
>>> num_deleted = ibs.delete_gar(gar_rowid_list)
>>> print('num_deleted = %r' % (num_deleted,))
"""
# from wbia.algo.preproc import preproc_gar
# NO EXTERN IMPORT
if ut.VERBOSE:
logger.info('[ibs] deleting %d gar rows' % len(gar_rowid_list))
# Prepare: Delete externally stored data (if any)
# preproc_gar.on_delete(ibs, gar_rowid_list, config2_=config2_)
# NO EXTERN DELETE
# Finalize: Delete self
ibs.db.delete_rowids(const.GA_RELATION_TABLE, gar_rowid_list)
num_deleted = len(ut.filter_Nones(gar_rowid_list))
return num_deleted
def _get_cm_edge_data(infr, edges, cm_list=None):
symmetric = True
if cm_list is None:
cm_list = infr.cm_list
# Find scores for the edges that exist in the graph
edge_to_data = ut.ddict(dict)
aid_to_cm = {cm.qaid: cm for cm in cm_list}
for u, v in edges:
if symmetric:
u, v = e_(u, v)
cm1 = aid_to_cm.get(u, None)
cm2 = aid_to_cm.get(v, None)
scores = []
ranks = []
for cm in ut.filter_Nones([cm1, cm2]):
for aid in [u, v]:
idx = cm.daid2_idx.get(aid, None)
if idx is None:
continue
score = cm.annot_score_list[idx]
rank = cm.get_annot_ranks([aid])[0]
scores.append(score)
ranks.append(rank)
if len(scores) == 0:
score = None
rank = None
else:
# Choose whichever one gave the best score
idx = vt.safe_argmax(scores, nans=False)
score = scores[idx]
rank = ranks[idx]
edge_to_data[(u, v)]['score'] = score
edge_to_data[(u, v)]['rank'] = rank
return edge_to_data
def delete_feat_featweight(ibs, feat_rowid_list, config2_=None):
""" feat.featweight.delete(feat_rowid_list)
Args:
feat_rowid_list
Returns:
int: num_deleted
TemplateInfo:
Tdeleter_rl_depenant
parent = feat
leaf = featweight
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.control._autogen_featweight_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> feat_rowid_list = ibs._get_all_feat_rowids()[:2]
>>> ibs.delete_feat_featweight(feat_rowid_list, config2_=config2_)
"""
if ut.VERBOSE:
print('[ibs] deleting %d feats leaf nodes' % len(feat_rowid_list))
# Delete any dependants
_featweight_rowid_list = ibs.get_feat_featweight_rowids(
feat_rowid_list, config2_=config2_, ensure=False)
featweight_rowid_list = ut.filter_Nones(_featweight_rowid_list)
num_deleted = ibs.delete_featweight(featweight_rowid_list)
return num_deleted
def ingest_rawdata(ibs, ingestable, localize=False):
"""
Ingests rawdata into an ibeis database.
if ingest_type == 'named_folders':
Converts folder structure where folders = name, to ibsdb
if ingest_type == 'named_images':
Converts imgname structure where imgnames = name_id.ext, to ibsdb
"""
img_dir = ingestable.img_dir
ingest_type = ingestable.ingest_type
fmtkey = ingestable.fmtkey
adjust_percent = ingestable.adjust_percent
postingest_func = ingestable.postingest_func
print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
# Get images in the image directory
gpath_list = list_ingestable_images(img_dir, recursive=True)
# Parse structure for image names
if ingest_type == 'named_folders':
name_list = ibsfuncs.get_name_text_from_parent_folder(gpath_list, img_dir, fmtkey)
pass
if ingest_type == 'named_images':
name_list = ibsfuncs.get_name_text_from_gnames(gpath_list, img_dir, fmtkey)
if ingest_type == 'unknown':
name_list = [constants.INDIVIDUAL_KEY for _ in xrange(len(gpath_list))]
# Add Images
gid_list_ = ibs.add_images(gpath_list)
# <DEBUG>
#print('added: ' + utool.indentjoin(map(str, zip(gid_list_, gpath_list))))
unique_gids = list(set(gid_list_))
print("[ingest] Length gid list: %d" % len(gid_list_))
print("[ingest] Length unique gid list: %d" % len(unique_gids))
assert len(gid_list_) == len(gpath_list)
for gid in gid_list_:
if gid is None:
print('[ingest] big fat warning')
# </DEBUG>
gid_list = utool.filter_Nones(gid_list_)
unique_gids, unique_names, unique_notes = ibsfuncs.resolve_name_conflicts(
gid_list, name_list)
# Add ANNOTATIONs with names and notes
aid_list = ibsfuncs.use_images_as_annotations(ibs, unique_gids,
name_list=unique_names,
notes_list=unique_notes,
adjust_percent=adjust_percent)
if localize:
ibsfuncs.localize_images(ibs)
if postingest_func is not None:
postingest_func(ibs)
# Print to show success
#ibs.print_image_table()
#ibs.print_tables()
#ibs.print_annotation_table()
#ibs.print_alr_table()
#ibs.print_lblannot_table()
#ibs.print_image_table()
return aid_list
def config_graph_subattrs(cfg, depc):
# TODO: if this hack is fully completed need a way of getting the
# full config belonging to both chip + feat
# cfg = request.config.feat_cfg
import networkx as netx
tablename = ut.invert_dict(depc.configclass_dict)[cfg.__class__]
#tablename = cfg.get_config_name()
ancestors = netx.dag.ancestors(depc.graph, tablename)
subconfigs_ = ut.dict_take(depc.configclass_dict, ancestors, None)
subconfigs = ut.filter_Nones(subconfigs_) # NOQA
def delete_rows(table, rowid_list):
#from ibeis.algo.preproc import preproc_feat
if table.on_delete is not None:
table.on_delete()
if ut.VERBOSE:
print('deleting %d rows' % len(rowid_list))
# Finalize: Delete table
table.db.delete_rowids(table.tablename, rowid_list)
num_deleted = len(ut.filter_Nones(rowid_list))
return num_deleted
def config_graph_subattrs(cfg, depc):
# TODO: if this hack is fully completed need a way of getting the
# full config belonging to both chip + feat
# cfg = request.config.feat_cfg
import networkx as netx
tablename = ut.invert_dict(depc.configclass_dict)[cfg.__class__]
#tablename = cfg.get_config_name()
ancestors = netx.dag.ancestors(depc.graph, tablename)
subconfigs_ = ut.dict_take(depc.configclass_dict, ancestors, None)
subconfigs = ut.filter_Nones(subconfigs_) # NOQA
def get_review_rowids_between(ibs, aids1, aids2=None, method=None):
"""
Find staging rowids between sets of aids
Doctest:
>>> from wbia.control.manual_review_funcs import *
>>> import wbia
>>> ibs = wbia.opendb('PZ_MTEST')
>>> aids1 = aids2 = [1, 2, 3, 4, 5, 6]
>>> rowids_between = ibs.get_review_rowids_between
>>> ids1 = sorted(rowids_between(aids1, aids2, method=1))
>>> ids2 = sorted(rowids_between(aids1, aids2, method=2))
>>> assert len(ub.find_duplicates(ids1)) == 0
>>> assert len(ub.find_duplicates(ids2)) == 0
>>> assert ids1 == ids2
"""
if aids2 is None:
aids2 = aids1
if method is None:
if len(aids1) * len(aids2) > 5000:
method = 1
else:
method = 2
if method == 1:
# Strategy 1: get all existing rows and see what intersects
# This is better when the enumerated set of rows would be larger than
# the database size
rowids11 = set(ut.flatten(ibs.get_review_rowids_from_aid1(aids1)))
rowids12 = set(ut.flatten(ibs.get_review_rowids_from_aid2(aids1)))
if aids1 is aids2:
rowids = list(reduce(set.intersection, [rowids11, rowids12]))
else:
rowids21 = set(ut.flatten(ibs.get_review_rowids_from_aid1(aids2)))
rowids22 = set(ut.flatten(ibs.get_review_rowids_from_aid2(aids2)))
rowids = list(
reduce(set.intersection, [rowids11, rowids12, rowids21, rowids22])
)
elif method == 2:
# Strategy 2: enumerate what rows could exist and see what does exist
# This is better when the enumerated set of rows would be smaller than
# the database size
edges = list(ut.product_nonsame(aids1, aids2))
if len(edges) == 0:
rowids = []
else:
rowids = ibs.get_review_rowids_from_edges(edges, directed=True)
if rowids is None:
rowids = []
rowids = ut.filter_Nones(rowids)
rowids = ut.flatten(rowids)
else:
raise ValueError('no method=%r' % (method,))
return rowids
def delete_annot_chips(ibs, aid_list, qreq_=None):
""" Clears annotation data but does not remove the annotation """
_cid_list = ibs.get_annot_chip_rowids(aid_list, ensure=False, qreq_=qreq_)
cid_list = ut.filter_Nones(_cid_list)
ibs.delete_chips(cid_list)
# HACK FIX: if annot chips are None then the image thumbnail
# will not be invalidated
if len(_cid_list) != len(cid_list):
aid_list_ = [aid for aid, _cid in zip(aid_list, _cid_list) if _cid is None]
gid_list_ = ibs.get_annot_gids(aid_list_)
ibs.delete_image_thumbs(gid_list_)
def get_annotmatch_rowids_in_cliques(ibs, aids_list):
# Equivalent call:
#ibs.get_annotmatch_rowids_between_groups(ibs, aids_list, aids_list)
import itertools
ams_list = [
ibs.get_annotmatch_rowid_from_undirected_superkey(*zip(
*itertools.combinations(aids, 2)))
for aids in ut.ProgIter(aids_list, lbl='loading clique am rowids')
]
ams_list = [[] if ams is None else ut.filter_Nones(ams)
for ams in ams_list]
return ams_list
def make_netx_graph_from_aids(ibs, aids_list, full=False):
#aids_list, nid_list = ibs.group_annots_by_name(aid_list)
unique_aids = list(ut.flatten(aids_list))
aids1, aids2 = get_name_rowid_edges_from_aids2(ibs, aids_list)
if not full:
annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2)
annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)
return make_netx_graph_from_aidpairs(ibs, aids1, aids2, unique_aids=unique_aids)
def delete_chips(ibs, cid_list, verbose=ut.VERBOSE, qreq_=None):
""" deletes images from the database that belong to gids"""
from ibeis.model.preproc import preproc_chip
if verbose:
print('[ibs] deleting %d annotation-chips' % len(cid_list))
# Delete chip-images from disk
#preproc_chip.delete_chips(ibs, cid_list, verbose=verbose)
preproc_chip.on_delete(ibs, cid_list, verbose=verbose)
# Delete chip features from sql
_fid_list = ibs.get_chip_fids(cid_list, ensure=False, qreq_=qreq_)
fid_list = ut.filter_Nones(_fid_list)
ibs.delete_features(fid_list)
# Delete chips from sql
ibs.dbcache.delete_rowids(const.CHIP_TABLE, cid_list)
def delete_annot_chips(ibs, aid_list, config2_=None):
r"""
Clears annotation data (does not remove the annotation)
RESTful:
Method: DELETE
URL: /api/annot_chip/
"""
_cid_list = ibs.get_annot_chip_rowids(aid_list, ensure=False, config2_=config2_)
cid_list = ut.filter_Nones(_cid_list)
ibs.delete_chips(cid_list)
# HACK FIX: if annot chips are None then the image thumbnail
# will not be invalidated
if len(_cid_list) != len(cid_list):
aid_list_ = [aid for aid, _cid in zip(aid_list, _cid_list) if _cid is None]
gid_list_ = ibs.get_annot_gids(aid_list_)
ibs.delete_image_thumbs(gid_list_)
def _new_repo_url(repo, host=None, user=None, reponame=None, fmt=None):
import utool as ut
if reponame is None:
reponame = repo.reponame
if host is None:
host = "github.com"
if fmt is None:
fmt = "ssh"
if host is not None:
assert user is not None, "github needs a user"
url_fmts = {"https": ("https://", "/"), "ssh": ("git@", ":")}
prefix, sep = url_fmts[fmt]
parts = [prefix, host, sep, user, "/", reponame, ".git"]
parts = ut.filter_Nones(parts)
url = "".join(parts)
return url
def remove_existing_fpaths(fpath_list, verbose=VERBOSE, quiet=QUIET,
strict=False, print_caller=PRINT_CALLER, lbl='files'):
""" checks existance before removing. then tries to remove exisint paths """
import utool as ut
if print_caller:
print(util_dbg.get_caller_name(range(1, 4)) + ' called remove_existing_fpaths')
fpath_list_ = ut.filter_Nones(fpath_list)
exists_list = list(map(exists, fpath_list_))
if verbose:
nTotal = len(fpath_list)
nValid = len(fpath_list_)
nExist = sum(exists_list)
print('[util_path.remove_existing_fpaths] requesting delete of %d %s' % (nTotal, lbl))
if nValid != nTotal:
print('[util_path.remove_existing_fpaths] trying to delete %d/%d non None %s ' % (nValid, nTotal, lbl))
print('[util_path.remove_existing_fpaths] %d/%d exist and need to be deleted' % (nExist, nValid))
existing_fpath_list = ut.filter_items(fpath_list_, exists_list)
return remove_fpaths(existing_fpath_list, verbose=verbose, quiet=quiet,
strict=strict, print_caller=False, lbl=lbl)
def _new_repo_url(repo, host=None, user=None, reponame=None, fmt=None):
import utool as ut
if reponame is None:
reponame = repo.reponame
if host is None:
host = 'github.com'
if fmt is None:
fmt = 'ssh'
if host is not None:
assert user is not None, 'github needs a user'
url_fmts = {
'https': ('https://', '/'),
'ssh': ('git@', ':'),
}
prefix, sep = url_fmts[fmt]
parts = [prefix, host, sep, user, '/', reponame, '.git']
parts = ut.filter_Nones(parts)
url = ''.join(parts)
return url
def _new_remote_url(repo, host=None, user=None, reponame=None, fmt=None):
import utool as ut
if reponame is None:
reponame = repo.reponame
if host is None:
host = 'github.com'
if fmt is None:
fmt = 'ssh'
if host == 'github.com':
assert user is not None, 'github needs a user'
url_fmts = {
'https': ('https://', '/'),
'ssh': ('git@', ':'),
}
prefix, sep = url_fmts[fmt]
user_ = '' if user is None else user + '/'
parts = [prefix, host, sep, user_, reponame, '.git']
parts = ut.filter_Nones(parts)
url = ''.join(parts)
return url
def get_default_cell_template_list(ibs):
cells = notebook_cells
noexample = ut.get_argflag('--noexample')
cell_template_list = [
cells.initialize,
None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,
cells.pipe_config_info,
cells.annot_config_info,
None if noexample else cells.timestamp_distribution,
None if noexample else cells.example_annotations,
None if noexample else cells.example_names,
cells.per_annotation_accuracy,
cells.per_name_accuracy,
cells.timedelta_distribution,
cells.config_overlap,
#cells.dbsize_expt,
None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,
cells.all_annot_scoresep,
cells.success_annot_scoresep,
cells.easy_success_cases,
cells.hard_success_cases,
cells.failure_type1_cases,
cells.failure_type2_cases,
#cells.investigate_specific_case,
#cells.view_intereseting_tags,
# TODO:
# show query chips
]
cell_template_list = ut.filter_Nones(cell_template_list)
return cell_template_list
def delete_chips(ibs, cid_list, verbose=ut.VERBOSE, config2_=None):
r"""
deletes images from the database that belong to gids
RESTful:
Method: DELETE
URL: /api/chip/
"""
from ibeis.algo.preproc import preproc_chip
if verbose:
print('[ibs] deleting %d annotation-chips' % len(cid_list))
# Delete sql-external (on-disk) information
preproc_chip.on_delete(ibs, cid_list)
# Delete sql-dependencies
fid_list = ut.filter_Nones(ibs.get_chip_feat_rowid(cid_list, config2_=config2_, ensure=False))
aid_list = ibs.get_chip_aids(cid_list)
gid_list = ibs.get_annot_gids(aid_list)
ibs.delete_image_thumbs(gid_list)
ibs.delete_annot_chip_thumbs(aid_list)
ibs.delete_features(fid_list, config2_=config2_)
# Delete chips from sql
ibs.dbcache.delete_rowids(const.CHIP_TABLE, cid_list)
def delete_gar(ibs, gar_rowid_list, config2_=None):
""" gar.delete(gar_rowid_list)
delete gar rows
Args:
gar_rowid_list
Returns:
int: num_deleted
TemplateInfo:
Tdeleter_native_tbl
tbl = gar
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control.manual_garelate_funcs import * # NOQA
>>> ibs, config2_ = testdata_ibs()
>>> gar_rowid_list = ibs._get_all_gar_rowids()[:2]
>>> num_deleted = ibs.delete_gar(gar_rowid_list)
>>> print('num_deleted = %r' % (num_deleted,))
"""
#from ibeis.algo.preproc import preproc_gar
# NO EXTERN IMPORT
if ut.VERBOSE:
print('[ibs] deleting %d gar rows' %
len(gar_rowid_list))
# Prepare: Delete externally stored data (if any)
#preproc_gar.on_delete(ibs, gar_rowid_list, config2_=config2_)
# NO EXTERN DELETE
# Finalize: Delete self
ibs.db.delete_rowids(
const.GA_RELATION_TABLE, gar_rowid_list)
num_deleted = len(
ut.filter_Nones(gar_rowid_list))
return num_deleted
def build_alias_map(regex_map, tag_vocab):
"""
Constructs explicit mapping. Order of items in regex map matters.
Items at top are given preference.
Example:
>>> tags_list = [['t1', 't2'], [], ['t3'], ['t4', 't5']]
>>> tag_vocab = ut.flat_unique(*tags_list)
>>> regex_map = [('t[3-4]', 'A9'), ('t0', 'a0')]
>>> unmapped = list(set(tag_vocab) - set(alias_map.keys()))
"""
import utool as ut
import re
alias_map = ut.odict([])
for pats, new_tag in reversed(regex_map):
pats = ut.ensure_iterable(pats)
for pat in pats:
flags = [re.match(pat, t) for t in tag_vocab]
for old_tag in ut.compress(tag_vocab, flags):
alias_map[old_tag] = new_tag
identity_map = ut.take_column(regex_map, 1)
for tag in ut.filter_Nones(identity_map):
alias_map[tag] = tag
return alias_map
def ingest_rawdata(ibs, ingestable, localize=False):
"""
Ingests rawdata into an ibeis database.
if ingest_type == 'named_folders':
Converts folder structure where folders = name, to ibsdb
if ingest_type == 'named_images':
Converts imgname structure where imgnames = name_id.ext, to ibsdb
CommandLine:
python ibeis/dbio/ingest_database.py --db seals_drop2
Args:
ibs (IBEISController): ibeis controller object
ingestable (?):
localize (bool): (default = False)
Returns:
list: aid_list - list of annotation rowids
CommandLine:
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata
python -m ibeis.dbio.ingest_database --exec-ingest_rawdata --db snow-leopards --imgdir /raid/raw_rsync/snow-leopards
Example:
>>> # SCRIPT
>>> # General ingest script
>>> from ibeis.dbio.ingest_database import * # NOQA
>>> import ibeis
>>> dbname = ut.get_argval('--db', str, None) # 'snow-leopards')
>>> force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
>>> img_dir = ut.get_argval('--imgdir', type_=str, default=None)
>>> assert img_dir is not None, 'specify img dir'
>>> assert dbname is not None, 'specify dbname'
>>> ingestable = Ingestable(dbname,
>>> img_dir=img_dir,
>>> ingest_type='unknown',
>>> fmtkey=None,
>>> species=None,
>>> images_as_annots=False,
>>> adjust_percent=0.00)
>>> from ibeis.control import IBEISControl
>>> dbdir = ibeis.sysres.db_to_dbdir(dbname, allow_newdir=True, use_sync=False)
>>> ut.ensuredir(dbdir, verbose=True)
>>> ibs = IBEISControl.request_IBEISController(dbdir)
>>> localize = False
>>> aid_list = ingest_rawdata(ibs, ingestable, localize)
>>> result = ('aid_list = %s' % (str(aid_list),))
>>> print(result)
"""
print('[ingest_rawdata] Ingestable' + str(ingestable))
if ingestable.zipfile is not None:
zipfile_fpath = ut.truepath(join(ibeis.sysres.get_workdir(), ingestable.zipfile))
ingestable.img_dir = ut.unarchive_file(zipfile_fpath)
img_dir = realpath(ingestable.img_dir)
ingest_type = ingestable.ingest_type
fmtkey = ingestable.fmtkey
adjust_percent = ingestable.adjust_percent
species_text = ingestable.species
postingest_func = ingestable.postingest_func
print('[ingest] ingesting rawdata: img_dir=%r, injest_type=%r' % (img_dir, ingest_type))
# Get images in the image directory
unzipped_file_base_dir = join(ibs.get_dbdir(), 'unzipped_files')
def extract_zipfile_images(ibs, ingestable):
import utool as ut # NOQA
zipfile_list = ut.glob(ingestable.img_dir, '*.zip', recursive=True)
if len(zipfile_list) > 0:
print('Found zipfile_list = %r' % (zipfile_list,))
ut.ensuredir(unzipped_file_base_dir)
for zipfile in zipfile_list:
unziped_file_relpath = dirname(relpath(relpath(realpath(zipfile), realpath(ingestable.img_dir))))
unzipped_file_dir = join(unzipped_file_base_dir, unziped_file_relpath)
ut.ensuredir(unzipped_file_dir)
ut.unzip_file(zipfile, output_dir=unzipped_file_dir, overwrite=False)
gpath_list = ut.list_images(unzipped_file_dir, fullpath=True, recursive=True)
else:
gpath_list = []
return gpath_list
def list_images(img_dir):
""" lists images that are not in an internal cache """
import utool as ut # NOQA
ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
gpath_list = ut.list_images(img_dir,
fullpath=True,
recursive=True,
ignore_list=ignore_list)
return gpath_list
# FIXME ensure python3 works with this
gpath_list1 = ut.ensure_unicode_strlist(list_images(img_dir))
gpath_list2 = ut.ensure_unicode_strlist(extract_zipfile_images(ibs, ingestable))
gpath_list = gpath_list1 + gpath_list2
#__STR__ = const.__STR__
#map(__STR__, gpath_list)
#ut.embed()
# Parse structure for image names
if ingest_type == 'named_folders':
name_list1 = get_name_texts_from_parent_folder(gpath_list1, img_dir, fmtkey)
name_list2 = get_name_texts_from_parent_folder(gpath_list2, unzipped_file_base_dir, fmtkey)
name_list = name_list1 + name_list2
pass
elif ingest_type == 'named_images':
name_list = get_name_texts_from_gnames(gpath_list, img_dir, fmtkey)
elif ingest_type == 'unknown':
name_list = [const.UNKNOWN for _ in range(len(gpath_list))]
else:
raise NotImplementedError('unknwon ingest_type=%r' % (ingest_type,))
# Find names likely to be the same?
RECTIFY_NAMES_HUERISTIC = True
if RECTIFY_NAMES_HUERISTIC:
names = sorted(list(set(name_list)))
splitchars = [' ', '/']
def multisplit(str_, splitchars):
import utool as ut
n = [str_]
for char in splitchars:
n = ut.flatten([_.split(char) for _ in n])
return n
groupids = [multisplit(n1, splitchars)[0] for n1 in names]
grouped_names = ut.group_items(names, groupids)
fixed_names = {newkey: key for key, val in grouped_names.items() if len(val) > 1 for newkey in val}
name_list = [fixed_names.get(name, name) for name in name_list]
# Add Images
gpath_list = [gpath.replace('\\', '/') for gpath in gpath_list]
gid_list_ = ibs.add_images(gpath_list)
# <DEBUG>
#print('added: ' + ut.indentjoin(map(str, zip(gid_list_, gpath_list))))
unique_gids = list(set(gid_list_))
print("[ingest] Length gid list: %d" % len(gid_list_))
print("[ingest] Length unique gid list: %d" % len(unique_gids))
assert len(gid_list_) == len(gpath_list)
for gid in gid_list_:
if gid is None:
print('[ingest] big fat warning')
# </DEBUG>
gid_list = ut.filter_Nones(gid_list_)
unique_gids, unique_names, unique_notes = resolve_name_conflicts(
gid_list, name_list)
# Add ANNOTATIONs with names and notes
if ingestable.images_as_annots:
aid_list = ibs.use_images_as_annotations(unique_gids,
adjust_percent=adjust_percent)
ibs.set_annot_names(aid_list, unique_names)
ibs.set_annot_notes(aid_list, unique_notes)
if species_text is not None:
ibs.set_annot_species(aid_list, [species_text] * len(aid_list))
if localize:
ibs.localize_images()
if postingest_func is not None:
postingest_func(ibs)
def get_annotmatch_rowids_between(ibs, aids1, aids2, method=None):
"""
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.annotmatch_funcs import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb('PZ_MTEST')
>>> aids1 = aids2 = [1, 2, 3, 4, 5, 6]
>>> rowids_between = ibs.get_annotmatch_rowids_between
>>> ams1 = sorted(rowids_between(aids1, aids2, method=1))
>>> ams2 = sorted(rowids_between(aids1, aids2, method=2))
>>> assert len(ub.find_duplicates(ams1)) == 0
>>> assert len(ub.find_duplicates(ams2)) == 0
>>> assert sorted(ams2) == sorted(ams1)
"""
if method is None:
if len(aids1) * len(aids2) > 5000:
method = 1
else:
method = 2
if method == 1:
# Strategy 1: get all existing rows and see what intersects
# This is better when the enumerated set of rows would be larger than
# the database size
unflat_rowids1L = ibs.get_annotmatch_rowids_from_aid1(aids1)
unflat_rowids1R = ibs.get_annotmatch_rowids_from_aid2(aids1)
unflat_rowids2L = ibs.get_annotmatch_rowids_from_aid1(aids2)
unflat_rowids2R = ibs.get_annotmatch_rowids_from_aid2(aids2)
am_rowids1L = {
r
for r in ut.iflatten(unflat_rowids1L) if r is not None
}
am_rowids1R = {
r
for r in ut.iflatten(unflat_rowids1R) if r is not None
}
am_rowids2L = {
r
for r in ut.iflatten(unflat_rowids2L) if r is not None
}
am_rowids2R = {
r
for r in ut.iflatten(unflat_rowids2R) if r is not None
}
ams12 = am_rowids1L.intersection(am_rowids2R)
ams21 = am_rowids2L.intersection(am_rowids1R)
ams = sorted(ams12.union(ams21))
# ams = sorted(am_rowids1.intersection(am_rowids2))
# rowids2 = ibs.get_annotmatch_rowids_from_aid2(aid_list)
# unflat_rowids1 = ibs.get_annotmatch_rowids_from_aid(aids1)
# unflat_rowids2 = ibs.get_annotmatch_rowids_from_aid(aids2)
# am_rowids1 = {r for r in ut.iflatten(unflat_rowids1) if r is not None}
# am_rowids2 = {r for r in ut.iflatten(unflat_rowids2) if r is not None}
# ams = sorted(am_rowids1.intersection(am_rowids2))
# ams = ut.isect(am_rowids1, am_rowids2)
elif method == 2:
# Strategy 2: enumerate what rows could exist and see what does exist
# This is better when the enumerated set of rows would be smaller than
# the database size
edges = list(ut.product_nonsame(aids1, aids2))
if len(edges) == 0:
ams = []
else:
aids1_, aids2_ = ut.listT(edges)
# ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1_, aids2_)
ams = ibs.get_annotmatch_rowid_from_superkey(aids1_, aids2_)
if ams is None:
ams = []
ams = ut.filter_Nones(ams)
return ams
def make_score_tabular(row_lbls,
col_lbls,
values,
title=None,
out_of=None,
bold_best=False,
flip=False,
bigger_is_better=True,
multicol_lbls=None,
FORCE_INT=False,
precision=None,
SHORTEN_ROW_LBLS=False,
col_align='l',
col_sep='|',
multicol_sep='|',
centerline=True,
astable=False,
table_position='',
AUTOFIX_LATEX=True,
**kwargs):
r"""
makes a LaTeX tabular for displaying scores or errors
Args:
row_lbls (list of str):
col_lbls (list of str):
values (ndarray):
title (str): (default = None)
out_of (None): (default = None)
bold_best (bool): (default = True)
flip (bool): (default = False)
table_position (str) : eg '[h]'
Returns:
str: tabular_str
CommandLine:
python -m utool.util_latex --test-make_score_tabular:0 --show
python -m utool.util_latex --test-make_score_tabular:1 --show
python -m utool.util_latex --test-make_score_tabular:2 --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2], [3.2, 4]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
>>> multicol_lbls = [('spam', 1), ('eggs', 2)]
>>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
"""
import utool as ut
if flip:
bigger_is_better = not bigger_is_better
flip_repltups = [('<=', '>'), ('>', '<='), ('\\leq', '\\gt'),
('\\geq', '\\lt'), ('score', 'error')]
col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
if title is not None:
title = replace_all(title, flip_repltups)
if out_of is not None:
values = out_of - values
# Abbreviate based on common substrings
common_rowlbl = None
if SHORTEN_ROW_LBLS:
if isinstance(row_lbls, list):
row_lbl_list = row_lbls
else:
row_lbl_list = row_lbls.flatten().tolist()
# Split the rob labels into the alg components
#algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
longest = long_substr(row_lbl_list)
common_strs = []
while len(longest) > 10:
common_strs += [longest]
row_lbl_list = [
row.replace(longest, '...') for row in row_lbl_list
]
longest = long_substr(row_lbl_list)
common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
row_lbls = row_lbl_list
if len(row_lbl_list) == 1:
common_rowlbl = row_lbl_list[0]
row_lbls = ['0']
# Stack values into a tabular body
# TODO: need ability to specify datatypes
def ensurelist(row_values):
try:
return row_values.tolist()
except AttributeError:
return row_values
if False:
# Numpy formatting
def padvec(shape=(1, 1)):
pad = np.array([[' ' for c in range(shape[1])]
for r in range(shape[0])])
return pad
col_lbls = ensure_rowvec(col_lbls)
row_lbls = ensure_colvec(row_lbls)
_0 = np.vstack([padvec(), row_lbls])
_1 = np.vstack([col_lbls, values])
body = np.hstack([_0, _1])
body = [[str_ for str_ in row] for row in body]
else:
assert len(row_lbls) == len(values)
body = [[' '] + col_lbls]
body += [[row_lbl] + ensurelist(row_values)
for row_lbl, row_values in zip(row_lbls, values)]
#import utool as ut
# Fix things in each body cell
DO_PERCENT = True
try:
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if precision is not None:
# Hack
if ut.is_float(body[r][c]):
fmtstr = '%.' + str(precision) + 'f'
body[r][c] = fmtstr % (float(body[r][c]), )
# Force integer
if FORCE_INT:
body[r][c] = str(int(float(body[r][c])))
body[r][c] = str(body[r][c])
# Remove bad formatting;
if AUTOFIX_LATEX:
body[r][c] = escape_latex(body[r][c])
except Exception as ex:
import utool as ut
print('len(row_lbls) = %r' % (len(row_lbls), ))
print('len(col_lbls) = %r' % (len(col_lbls), ))
print('len(values) = %r' % (values, ))
print('ut.depth_profile(values) = %r' % (ut.depth_profile(values), ))
util_dbg.printex(ex, keys=['r', 'c'])
raise
# Bold the best values
if bold_best:
best_col_scores = values.max(0) if bigger_is_better else values.min(0)
rows_to_bold = [
np.where(values[:, colx] == best_col_scores[colx])[0]
for colx in range(len(values.T))
]
for colx, rowx_list in enumerate(rows_to_bold):
for rowx in rowx_list:
body[rowx + 1][colx +
1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'
# More fixing after the bold is in place
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if out_of is not None:
body[r][c] = body[r][c] + '/' + str(out_of)
if DO_PERCENT:
percent = ' = %.1f%%' % float(
100 * values[r - 1, c - 1] / out_of)
body[r][c] += escape_latex(percent)
# Align columns for pretty printing
body = np.array(body)
ALIGN_BODY = True
if ALIGN_BODY:
new_body_cols = []
for col in body.T:
colstrs = list(map(str, ensurelist(col)))
collens = list(map(len, colstrs))
maxlen = max(collens)
newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
new_body_cols += [newcols]
body = np.array(new_body_cols).T
# Build Body (and row layout)
HLINE_SEP = True
rowvalsep = ''
colvalsep = ' & '
endl = '\\\\\n'
hline = r'\hline'
#extra_rowsep_pos_list = [1] # rows to insert an extra hline after
extra_rowsep_pos_list = [] # rows to insert an extra hline after
if HLINE_SEP:
rowvalsep = hline + '\n'
# rowstr list holds blocks of rows
rowstr_list = [colvalsep.join(row) + endl for row in body]
#rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
#rowstr_list = [(
# ('' if len(row) == 0 else row[0])
# if len(row) <= 1 else
# row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
# for row in body]
rowsep_list = [rowvalsep for row in rowstr_list[0:-1]
] # should be len 1 less than rowstr_list
# Insert multicolumn names
if multicol_lbls is not None:
# TODO: label of the row labels
multicol_sep
multicols = [
latex_multicolumn(multicol, size, 'c' + multicol_sep)
for multicol, size in multicol_lbls
]
multicol_str = latex_multirow(
'', 2) + colvalsep + colvalsep.join(multicols) + endl
ncols = sum([tup[1] for tup in multicol_lbls])
mcol_sep = '\\cline{2-%d}\n' % (ncols + 1, )
rowstr_list = [multicol_str] + rowstr_list
rowsep_list = [mcol_sep] + rowsep_list
#extra_rowsep_pos_list += [1]
# Insert title
if title is not None and not astable:
tex_title = latex_multicolumn(title, len(body[0])) + endl
rowstr_list = [tex_title] + rowstr_list
rowsep_list = [rowvalsep] + rowsep_list
#extra_rowsep_pos_list += [2]
# Apply an extra hline (for label)
#extra_rowsep_pos_list = []
for pos in sorted(extra_rowsep_pos_list)[::-1]:
rowstr_list.insert(pos, '')
rowsep_list.insert(pos, rowvalsep)
#tabular_body = rowvalsep.join(rowstr_list)
from six.moves import zip_longest
tabular_body = ''.join([
row if sep is None else row + sep
for row, sep in zip_longest(rowstr_list, rowsep_list)
])
# Build Column Layout
col_align_list = [col_align] * len(body[0])
#extra_collayoutsep_pos_list = [1]
extra_collayoutsep_pos_list = []
for pos in sorted(extra_collayoutsep_pos_list)[::-1]:
col_align_list.insert(pos, '')
#col_layaout_sep_list = rowlblcol_sep # TODO
rowlblcol_sep = '|'
# Build build internal seprations between column alignments
# Defaults to just the normal col_sep
col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
# Adjust for the separations between row labels and the actual row data
if len(col_align_sep_list) > 0:
col_align_sep_list[0] = rowlblcol_sep
# Continue multicolumn sepratation
if multicol_lbls is not None:
multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
for offset in multicol_offsets:
if offset < len(col_align_sep_list):
col_align_sep_list[offset] = multicol_sep
from six.moves import zip_longest
_tmp = [
ut.filter_Nones(tup)
for tup in zip_longest(col_align_list, col_align_sep_list)
]
col_layout = ''.join(ut.flatten(_tmp))
#if len(col_align_list) > 1:
# col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
#else:
# col_layout = col_sep.join(col_align_list)
tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
tabular_tail = r'\end{tabular}'
if centerline:
tabular_head = r'\centerline{' + '\n' + tabular_head
tabular_tail = tabular_tail + '}'
if astable:
#tabular_head = r'\begin{centering}' + '\n' + tabular_head
tabular_head = r'\centering' + '\n' + tabular_head
tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head
lblstr = latex_sanitize_command_name(kwargs.get('label', title))
caption = title
if AUTOFIX_LATEX:
caption = escape_latex(caption)
caption = '\n% ---\n' + caption + '\n% ---\n'
#tabular_head = r'\end{centering}' + '\n' + tabular_head
tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (
lblstr, caption, lblstr)
tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
topsep = '\\hline\n' if True else '\\toprule\n'
botsep = '\\hline\n' if True else '\\bottomrule\n'
tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail
if common_rowlbl is not None:
#tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
pass
return tabular_str
def stagger_group(list_):
return ut.filter_Nones(ut.iflatten(zip_longest(*list_)))
def get_dbinfo(ibs, verbose=True,
with_imgsize=False,
with_bytes=False,
with_contrib=False,
with_agesex=False,
with_header=True,
short=False,
tag='dbinfo',
aid_list=None):
"""
Returns dictionary of digestable database information
Infostr is a string summary of all the stats. Prints infostr in addition to
returning locals
Args:
ibs (IBEISController):
verbose (bool):
with_imgsize (bool):
with_bytes (bool):
Returns:
dict:
CommandLine:
python -m ibeis.other.dbinfo --exec-get_dbinfo:0
python -m ibeis.other.dbinfo --test-get_dbinfo:1
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a ctrl
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0
Example1:
>>> # SCRIPT
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> defaultdb = 'testdb1'
>>> ibs, aid_list = ibeis.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
>>> kwargs = ut.get_kwdefaults(get_dbinfo)
>>> kwargs['verbose'] = False
>>> kwargs['aid_list'] = aid_list
>>> kwargs = ut.parse_dict_from_argv(kwargs)
>>> output = get_dbinfo(ibs, **kwargs)
>>> result = (output['info_str'])
>>> print(result)
>>> #ibs = ibeis.opendb(defaultdb='testdb1')
>>> # <HACK FOR FILTERING>
>>> #from ibeis.expt import cfghelpers
>>> #from ibeis.expt import annotation_configs
>>> #from ibeis.init import filter_annots
>>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
>>> # annotation_configs.TEST_NAMES)
>>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
>>> # ut.get_list_column(named_defaults_dict, 'qcfg')))
>>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
>>> #aid_list = ibs.get_valid_aids()
>>> # </HACK FOR FILTERING>
Example1:
>>> # ENABLE_DOCTEST
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> verbose = True
>>> short = True
>>> #ibs = ibeis.opendb(db='GZ_ALL')
>>> #ibs = ibeis.opendb(db='PZ_Master0')
>>> ibs = ibeis.opendb('testdb1')
>>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
>>> ibs.delete_contributors(ibs.get_valid_contrib_rowids())
>>> ibs.delete_empty_nids()
>>> #ibs = ibeis.opendb(db='PZ_MTEST')
>>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
>>> result = (output['info_str'])
>>> print(result)
+============================
DB Info: testdb1
DB Notes: None
DB NumContrib: 0
----------
# Names = 7
# Names (unassociated) = 0
# Names (singleton) = 5
# Names (multiton) = 2
----------
# Annots = 13
# Annots (unknown) = 4
# Annots (singleton) = 5
# Annots (multiton) = 4
----------
# Img = 13
L============================
"""
# TODO Database size in bytes
# TODO: occurrence, contributors, etc...
# Basic variables
request_annot_subset = False
_input_aid_list = aid_list # NOQA
if aid_list is None:
valid_aids = ibs.get_valid_aids()
valid_nids = ibs.get_valid_nids()
valid_gids = ibs.get_valid_gids()
else:
if isinstance(aid_list, str):
# Hack to get experiment stats on aids
acfg_name_list = [aid_list]
print('Specified custom aids via acfgname %s' % (acfg_name_list,))
from ibeis.expt import experiment_helpers
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs, acfg_name_list)
aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
#aid_list =
if verbose:
print('Specified %d custom aids' % (len(aid_list,)))
request_annot_subset = True
valid_aids = aid_list
valid_nids = list(
set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False)) -
{const.UNKNOWN_NAME_ROWID}
)
valid_gids = list(set(ibs.get_annot_gids(aid_list)))
#associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation
FILTER_HACK = True
if FILTER_HACK:
# HUGE HACK - get only images and names with filtered aids
valid_aids_ = ibs.filter_aids_custom(valid_aids)
valid_nids_ = ibs.filter_nids_custom(valid_nids)
valid_gids_ = ibs.filter_gids_custom(valid_gids)
if verbose:
print('Filtered %d names' % (len(valid_nids) - len(valid_nids_)))
print('Filtered %d images' % (len(valid_gids) - len(valid_gids_)))
print('Filtered %d annots' % (len(valid_aids) - len(valid_aids_)))
valid_gids = valid_gids_
valid_nids = valid_nids_
valid_aids = valid_aids_
#associated_nids = ut.compress(associated_nids, map(any,
#ibs.unflat_map(ibs.get_annot_custom_filterflags,
# ibs.get_name_aids(associated_nids))))
# Image info
if verbose:
print('Checking Image Info')
gx2_aids = ibs.get_image_aids(valid_gids)
if FILTER_HACK:
gx2_aids = [ibs.filter_aids_custom(aids) for aids in gx2_aids] # HACK FOR FILTER
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
gx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in gx2_aids]
gx2_nAnnots = np.array(list(map(len, gx2_aids)))
image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
gx2_nAnnots_stats = ut.get_stats_str(gx2_nAnnots, newlines=True, use_median=True)
image_reviewed_list = ibs.get_image_reviewed(valid_gids)
# Name stats
if verbose:
print('Checking Name Info')
nx2_aids = ibs.get_name_aids(valid_nids)
if FILTER_HACK:
nx2_aids = [ibs.filter_aids_custom(aids) for aids in nx2_aids] # HACK FOR FILTER
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
nx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in nx2_aids]
associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))
ibs.check_name_mapping_consistency(nx2_aids)
# Occurrence Info
def compute_annot_occurrence_ids(ibs, aid_list):
from ibeis.algo.preproc import preproc_occurrence
gid_list = ibs.get_annot_gids(aid_list)
gid2_aids = ut.group_items(aid_list, gid_list)
flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False)
occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()}
return occurid2_aids
import utool
with utool.embed_on_exception_context:
occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
nid2_occurxs = ut.ddict(list)
for occurx, nids in enumerate(occur_unique_nids):
for nid in nids:
nid2_occurxs[nid].append(occurx)
nid2_occurx_single = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1}
nid2_occurx_resight = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1}
singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())
singlesight_annot_stats = ut.get_stats(list(map(len, singlesight_encounters)), use_median=True, use_sum=True)
resight_name_stats = ut.get_stats(list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True)
try:
aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0)
undirected_tags = ibs.get_aidpair_tags(aid_pairs.T[0], aid_pairs.T[1], directed=False)
tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
pair_tag_info = ut.map_dict_vals(len, tag_dict)
num_reviewed_pairs = sum(ibs.get_annot_pair_is_reviewed(aid_pairs.T[0], aid_pairs.T[1]))
pair_tag_info['num_reviewed'] = num_reviewed_pairs
except Exception:
pair_tag_info = {}
#print(ut.dict_str(pair_tag_info))
# Annot Stats
# TODO: number of images where chips cover entire image
# TODO: total image coverage of annotation
# TODO: total annotation overlap
"""
ax2_unknown = ibs.is_aid_unknown(valid_aids)
ax2_nid = ibs.get_annot_name_rowids(valid_aids)
assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
"""
#
if verbose:
print('Checking Annot Species')
unknown_aids = ut.compress(valid_aids, ibs.is_aid_unknown(valid_aids))
species_list = ibs.get_annot_species_texts(valid_aids)
species2_aids = ut.group_items(valid_aids, species_list)
species2_nAids = {key: len(val) for key, val in species2_aids.items()}
if verbose:
print('Checking Multiton/Singleton Species')
nx2_nAnnots = np.array(list(map(len, nx2_aids)))
# Seperate singleton / multitons
multiton_nxs = np.where(nx2_nAnnots > 1)[0]
singleton_nxs = np.where(nx2_nAnnots == 1)[0]
unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
valid_nxs = np.hstack([multiton_nxs, singleton_nxs])
num_names_with_gt = len(multiton_nxs)
# Annot Info
if verbose:
print('Checking Annot Info')
multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
assert len(set(multiton_nxs)) == len(multiton_nxs)
if len(multiton_aids_list) == 0:
multiton_aids = np.array([], dtype=np.int)
else:
multiton_aids = np.hstack(multiton_aids_list)
assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
singleton_aids = ut.take(nx2_aids, singleton_nxs)
multiton_nid2_nannots = list(map(len, multiton_aids_list))
# Image size stats
if with_imgsize:
if verbose:
print('Checking ImageSize Info')
gpath_list = ibs.get_image_paths(valid_gids)
def wh_print_stats(wh_list):
if len(wh_list) == 0:
return '{empty}'
wh_list = np.asarray(wh_list)
stat_dict = OrderedDict(
[( 'max', wh_list.max(0)),
( 'min', wh_list.min(0)),
('mean', wh_list.mean(0)),
( 'std', wh_list.std(0))])
def arr2str(var):
return ('[' + (
', '.join(list(map(lambda x: '%.1f' % x, var)))
) + ']')
ret = (',\n '.join([
'%s:%s' % (key, arr2str(val))
for key, val in stat_dict.items()
]))
return '{\n ' + ret + '\n}'
print('reading image sizes')
# Image size stats
img_size_list = ibs.get_image_sizes(valid_gids)
img_size_stats = wh_print_stats(img_size_list)
# Chip size stats
annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
annotation_bbox_arr = np.array(annotation_bbox_list)
if len(annotation_bbox_arr) == 0:
annotation_size_list = []
else:
annotation_size_list = annotation_bbox_arr[:, 2:4]
chip_size_stats = wh_print_stats(annotation_size_list)
imgsize_stat_lines = [
(' # Img in dir = %d' % len(gpath_list)),
(' Image Size Stats = %s' % (img_size_stats,)),
(' * Chip Size Stats = %s' % (chip_size_stats,)),
]
else:
imgsize_stat_lines = []
if verbose:
print('Building Stats String')
multiton_stats = ut.get_stats_str(multiton_nid2_nannots, newlines=True, use_median=True)
# Time stats
unixtime_list = ibs.get_image_unixtime(valid_gids)
unixtime_list = ut.list_replace(unixtime_list, -1, float('nan'))
#valid_unixtime_list = [time for time in unixtime_list if time != -1]
#unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
if ut.get_argflag('--hackshow-unixtime'):
show_time_distributions(ibs, unixtime_list)
ut.show_if_requested()
unixtime_statstr = ut.get_timestats_str(unixtime_list, newlines=True, full=True)
# GPS stats
gps_list_ = ibs.get_image_gps(valid_gids)
gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
gps_list = ut.compress(gps_list_, gpsvalid_list)
def get_annot_age_stats(aid_list):
annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
age_dict = ut.ddict((lambda : 0))
for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
if (min_age is None or min_age < 12) and max_age < 12:
age_dict['Infant'] += 1
elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
age_dict['Juvenile'] += 1
elif 36 <= min_age and (36 <= max_age or max_age is None):
age_dict['Adult'] += 1
else:
print('Found UNKNOWN Age: %r, %r' % (min_age, max_age, ))
age_dict['UNKNOWN'] += 1
return age_dict
def get_annot_sex_stats(aid_list):
annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(set(annot_sextext_list) - set(sex_keys))
sextext2_nAnnots = ut.odict([(key, len(sextext2_aids.get(key, []))) for key in sex_keys])
# Filter 0's
sextext2_nAnnots = {key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0}
return sextext2_nAnnots
if verbose:
print('Checking Other Annot Stats')
qualtext2_nAnnots = ibs.get_annot_qual_stats(valid_aids)
yawtext2_nAnnots = ibs.get_annot_yaw_stats(valid_aids)
agetext2_nAnnots = get_annot_age_stats(valid_aids)
sextext2_nAnnots = get_annot_sex_stats(valid_aids)
if verbose:
print('Checking Contrib Stats')
# Contributor Statistics
# hack remove colon for image alignment
def fix_tag_list(tag_list):
return [None if tag is None else tag.replace(':', ';') for tag in tag_list]
image_contrib_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
annot_contrib_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
contrib_tag_to_gids = ut.group_items(valid_gids, image_contrib_tags)
contrib_tag_to_aids = ut.group_items(valid_aids, annot_contrib_tags)
contrib_tag_to_qualstats = {key: ibs.get_annot_qual_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}
contrib_tag_to_viewstats = {key: ibs.get_annot_yaw_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}
contrib_tag_to_nImages = {key: len(val) for key, val in six.iteritems(contrib_tag_to_gids)}
contrib_tag_to_nAnnots = {key: len(val) for key, val in six.iteritems(contrib_tag_to_aids)}
if verbose:
print('Summarizing')
# Summarize stats
num_names = len(valid_nids)
num_names_unassociated = len(valid_nids) - len(associated_nids)
num_names_singleton = len(singleton_nxs)
num_names_multiton = len(multiton_nxs)
num_singleton_annots = len(singleton_aids)
num_multiton_annots = len(multiton_aids)
num_unknown_annots = len(unknown_aids)
num_annots = len(valid_aids)
if with_bytes:
if verbose:
print('Checking Disk Space')
ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))
if True:
if verbose:
print('Check asserts')
try:
bad_aids = np.intersect1d(multiton_aids, unknown_aids)
_num_names_total_check = num_names_singleton + num_names_unassociated + num_names_multiton
_num_annots_total_check = num_unknown_annots + num_singleton_annots + num_multiton_annots
assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
assert _num_names_total_check == num_names, 'inconsistent num names'
#if not request_annot_subset:
# dont check this if you have an annot subset
assert _num_annots_total_check == num_annots, 'inconsistent num annots'
except Exception as ex:
ut.printex(ex, keys=[
'_num_names_total_check',
'num_names',
'_num_annots_total_check',
'num_annots',
'num_names_singleton',
'num_names_multiton',
'num_unknown_annots',
'num_multiton_annots',
'num_singleton_annots',
])
raise
# Get contributor statistics
contrib_rowids = ibs.get_valid_contrib_rowids()
num_contributors = len(contrib_rowids)
# print
num_tabs = 5
def align2(str_):
return ut.align(str_, ':', ' :')
def align_dict2(dict_):
str_ = ut.dict_str(dict_)
return align2(str_)
header_block_lines = (
[('+============================'), ] + (
[
('+ singleton := single sighting'),
('+ multiton := multiple sightings'),
('--' * num_tabs),
] if not short and with_header else []
)
)
source_block_lines = [
('DB Info: ' + ibs.get_dbname()),
('DB Notes: ' + ibs.get_dbnotes()),
('DB NumContrib: %d' % num_contributors),
]
bytes_block_lines = [
('--' * num_tabs),
('DB Bytes: '),
(' +- dbdir nBytes: ' + dbdir_space),
(' | +- _ibsdb nBytes: ' + ibsdir_space),
(' | | +-imgdir nBytes: ' + imgdir_space),
(' | | +-cachedir nBytes: ' + cachedir_space),
] if with_bytes else []
name_block_lines = [
('--' * num_tabs),
('# Names = %d' % num_names),
('# Names (unassociated) = %d' % num_names_unassociated),
('# Names (singleton) = %d' % num_names_singleton),
('# Names (multiton) = %d' % num_names_multiton),
]
subset_str = ' ' if not request_annot_subset else '(SUBSET)'
annot_block_lines = [
('--' * num_tabs),
('# Annots %s = %d' % (subset_str, num_annots,)),
('# Annots (unknown) = %d' % num_unknown_annots),
('# Annots (singleton) = %d' % num_singleton_annots),
('# Annots (multiton) = %d' % num_multiton_annots),
]
annot_per_basic_block_lines = [
('--' * num_tabs),
('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)),
('# Annots per Species = %s' % (align_dict2(species2_nAids),)),
] if not short else []
occurrence_block_lines = [
('--' * num_tabs),
('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
] if not short else []
annot_per_qualview_block_lines = [
None if short else '# Annots per Viewpoint = %s' % align_dict2(yawtext2_nAnnots),
None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
]
annot_per_agesex_block_lines = [
'# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
'# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
] if not short and with_agesex else []
contrib_block_lines = [
'# Images per contributor = ' + align_dict2(contrib_tag_to_nImages),
'# Annots per contributor = ' + align_dict2(contrib_tag_to_nAnnots),
'# Quality per contributor = ' + ut.dict_str(contrib_tag_to_qualstats, sorted_=True),
'# Viewpoint per contributor = ' + ut.dict_str(contrib_tag_to_viewstats, sorted_=True),
] if with_contrib else []
img_block_lines = [
('--' * num_tabs),
('# Img = %d' % len(valid_gids)),
None if short else ('# Img reviewed = %d' % sum(image_reviewed_list)),
None if short else ('# Img with gps = %d' % len(gps_list)),
#('# Img with timestamp = %d' % len(valid_unixtime_list)),
None if short else ('Img Time Stats = %s' % (align2(unixtime_statstr),)),
]
info_str_lines = (
header_block_lines +
bytes_block_lines +
source_block_lines +
name_block_lines +
annot_block_lines +
annot_per_basic_block_lines +
occurrence_block_lines +
annot_per_qualview_block_lines +
annot_per_agesex_block_lines +
img_block_lines +
contrib_block_lines +
imgsize_stat_lines +
[('L============================'), ]
)
info_str = '\n'.join(ut.filter_Nones(info_str_lines))
info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
if verbose:
print(info_str2)
locals_ = locals()
return locals_
def make_netx_graph_from_aid_groups(ibs,
aids_list,
only_reviewed_matches=True,
invis_edges=None,
ensure_edges=None,
temp_nids=None,
allow_directed=False):
r"""
Args:
ibs (ibeis.IBEISController): image analysis api
aids_list (list):
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_graph import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> aids_list = [[1, 2, 3, 4], [5, 6, 7]]
>>> invis_edges = [(1, 5)]
>>> only_reviewed_matches = True
>>> graph = make_netx_graph_from_aid_groups(ibs, aids_list,
>>> only_reviewed_matches,
>>> invis_edges)
>>> list(nx.connected_components(graph.to_undirected()))
"""
#aids_list, nid_list = ibs.group_annots_by_name(aid_list)
unique_aids = list(ut.flatten(aids_list))
# grouped version
unflat_edges = (list(itertools.product(aids, aids)) for aids in aids_list)
aid_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
aids1 = ut.get_list_column(aid_pairs, 0)
aids2 = ut.get_list_column(aid_pairs, 1)
if only_reviewed_matches:
annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(
aids1, aids2)
annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)
graph = make_netx_graph_from_aidpairs(ibs,
aids1,
aids2,
unique_aids=unique_aids)
if ensure_edges is not None:
if ensure_edges == 'all':
ensure_edges = list(ut.upper_diag_self_prodx(list(graph.nodes())))
ensure_edges_ = []
for edge in ensure_edges:
edge = tuple(edge)
redge = tuple(edge[::-1]) # HACK
if graph.has_edge(*edge):
ensure_edges_.append(edge)
pass
#nx.set_edge_attributes(graph, 'weight', {edge: .001})
elif (not allow_directed) and graph.has_edge(*redge):
ensure_edges_.append(redge)
#nx.set_edge_attributes(graph, 'weight', {redge: .001})
pass
else:
ensure_edges_.append(edge)
#graph.add_edge(*edge, weight=.001)
graph.add_edge(*edge)
if temp_nids is None:
unique_nids = ibs.get_annot_nids(list(graph.nodes()))
else:
# HACK
unique_nids = [1] * len(list(graph.nodes()))
#unique_nids = temp_nids
nx.set_node_attributes(graph, 'nid', dict(zip(graph.nodes(), unique_nids)))
import plottool as pt
ensure_names_are_connected(graph, aids_list)
# Color edges by nid
color_by_nids(graph, unique_nids=unique_nids)
if invis_edges:
for edge in invis_edges:
if graph.has_edge(*edge):
nx.set_edge_attributes(graph, 'style', {edge: 'invis'})
nx.set_edge_attributes(graph, 'invisible', {edge: True})
else:
graph.add_edge(*edge, style='invis', invisible=True)
# Hack color images orange
if ensure_edges:
nx.set_edge_attributes(
graph, 'color', {tuple(edge): pt.ORANGE
for edge in ensure_edges_})
return graph
def get_default_cell_template_list(ibs):
"""
Defines the order of ipython notebook cells
"""
cells = notebook_cells
noexample = ut.get_argflag('--noexample')
asreport = ut.get_argflag('--asreport')
withtags = ut.get_argflag('--withtags')
cell_template_list = []
info_cells = [
cells.pipe_config_info,
cells.annot_config_info,
cells.timestamp_distribution,
]
dev_analysis = [
cells.config_overlap,
#cells.dbsize_expt,
None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,
cells.all_annot_scoresep,
cells.success_annot_scoresep,
]
cell_template_list += [
cells.introduction if asreport else None,
cells.initialize,
None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,
]
if not asreport:
cell_template_list += info_cells
if not noexample:
cell_template_list += [
cells.example_annotations,
cells.example_names,
]
cell_template_list += [
cells.per_annotation_accuracy,
cells.per_name_accuracy,
cells.easy_success_cases,
cells.hard_success_cases,
cells.failure_type1_cases,
cells.failure_type2_cases,
cells.timedelta_distribution,
]
if withtags:
cell_template_list += [
cells.investigate_specific_case,
cells.view_intereseting_tags,
]
if asreport:
# Append our debug stuff at the bottom
cell_template_list += [cells.IGNOREAFTER]
cell_template_list += info_cells
cell_template_list += dev_analysis
cell_template_list += [
cells.config_disagree_cases,
]
cell_template_list = ut.filter_Nones(cell_template_list)
return cell_template_list
def get_default_cell_template_list(ibs):
"""
Defines the order of ipython notebook cells
"""
cells = notebook_cells
noexample = not ut.get_argflag('--examples')
asreport = ut.get_argflag('--asreport')
withtags = ut.get_argflag('--withtags')
cell_template_list = []
info_cells = [
cells.pipe_config_info,
cells.annot_config_info,
# cells.per_encounter_stats,
cells.timestamp_distribution,
]
dev_analysis = [
cells.config_overlap,
#cells.dbsize_expt,
# None if ibs.get_dbname() == 'humpbacks' else cells.feat_score_sep,
cells.all_annot_scoresep,
cells.success_annot_scoresep,
]
cell_template_list += [
cells.introduction if asreport else None,
cells.nb_init,
cells.db_init,
None if ibs.get_dbname() != 'humpbacks' else cells.fluke_select,
]
if not asreport:
cell_template_list += info_cells
if not noexample:
cell_template_list += [
cells.example_annotations,
cells.example_names,
]
cell_template_list += [
cells.per_annotation_accuracy,
cells.per_name_accuracy,
cells.easy_success_cases,
cells.hard_success_cases,
cells.failure_type1_cases,
cells.failure_type2_cases,
cells.total_failure_cases,
cells.timedelta_distribution,
]
if withtags:
cell_template_list += [
cells.investigate_specific_case,
cells.view_intereseting_tags,
]
if asreport:
# Append our debug stuff at the bottom
cell_template_list += [cells.IGNOREAFTER]
cell_template_list += info_cells
cell_template_list += dev_analysis
cell_template_list += [
cells.config_disagree_cases,
]
cell_template_list = ut.filter_Nones(cell_template_list)
cell_template_list = ut.lmap(ut.normalize_cells, cell_template_list)
if not asreport:
# Remove all of the extra fluff
cell_template_list = [(header.split('\n')[0], code, None)
for (header, code, footer) in cell_template_list]
return cell_template_list
def convert_ggr2018_to_wbia(ggr_path,
dbdir=None,
purge=True,
dry_run=False,
apply_updates=True,
**kwargs):
r"""Convert the raw GGR2 (2018) data to an wbia database.
Args
ggr_path (str): Directory to folder *containing* raw GGR 2018 data
dbdir (str): Output directory
CommandLine:
python -m wbia convert_ggr2018_to_wbia
Example:
>>> # SCRIPT
>>> from wbia.dbio.ingest_ggr import * # NOQA
>>> default_ggr_path = join('/', 'data', 'wbia', 'GGR2', 'GGR2018data')
>>> default_dbdir = join('/', 'data', 'wbia', 'GGR2-IBEIS')
>>> dbdir = ut.get_argval('--dbdir', type_=str, default=default_dbdir)
>>> ggr_path = ut.get_argval('--ggr', type_=str, default=default_ggr_path)
>>> result = convert_ggr2018_to_wbia(ggr_path, dbdir=dbdir, purge=False, dry_run=True, apply_updates=False)
>>> print(result)
"""
ALLOWED_NUMBERS = list(range(1, 250))
ALLOWED_LETTERS = ['A', 'B', 'C', 'D', 'E', 'F']
################################################################################
if apply_updates:
_fix_ggr2018_directory_structure(ggr_path)
################################################################################
blacklist_filepath_set = set([
join(ggr_path, 'Cameras info.numbers'),
join(ggr_path, 'Cameras info.xlsx'),
join(ggr_path, 'GGR_photos_MRC_29.1.18.ods'),
join(ggr_path, 'Cameras info-2.numbers'),
])
# Check root files
direct = Directory(ggr_path)
for filepath in direct.files(recursive=False):
try:
assert filepath in blacklist_filepath_set
ut.delete(filepath)
except AssertionError:
logger.info('Unresolved root file found in %r' % (filepath, ))
continue
################################################################################
if purge:
ut.delete(dbdir)
ibs = wbia.opendb(dbdir=dbdir)
################################################################################
# Check folder structure
assert exists(ggr_path)
direct = Directory(ggr_path, recursive=0)
direct1_list = direct.directories()
direct1_list.sort(key=lambda x: int(x.base()), reverse=False)
for direct1 in direct1_list:
if not dry_run:
logger.info('Processing directory: %r' % (direct1, ))
base1 = direct1.base()
try:
int(base1)
except ValueError:
logger.info('Error found in %r' % (direct1, ))
continue
try:
assert len(direct1.files(recursive=False)) == 0
except AssertionError:
logger.info('Files found in %r' % (direct1, ))
continue
seen_letter_list = []
direct1_ = Directory(direct1.absolute_directory_path, recursive=0)
direct2_list = direct1_.directories()
direct2_list.sort(key=lambda x: x.base(), reverse=False)
for direct2 in direct2_list:
base2 = direct2.base()
try:
assert base2.startswith(base1)
except AssertionError:
logger.info('Folder name heredity conflict %r with %r' %
(direct2, direct1))
continue
try:
assert len(base2) >= 2
assert ' ' not in base2
number = base2[:-1]
letter = base2[-1]
number = int(number)
letter = letter.upper()
assert number in ALLOWED_NUMBERS
assert letter in ALLOWED_LETTERS
seen_letter_list.append(letter)
except ValueError:
logger.info('Error found in %r' % (direct2, ))
continue
except AssertionError:
logger.info('Folder name format error found in %r' %
(direct2, ))
continue
direct2_ = Directory(direct2.absolute_directory_path,
recursive=True,
images=True)
try:
assert len(direct2_.directories()) == 0
except AssertionError:
logger.info('Folders exist in file only level %r' %
(direct2, ))
continue
filepath_list = sorted(direct2_.files())
if not dry_run:
try:
gid_list = ibs.add_images(filepath_list)
gid_list = ut.filter_Nones(gid_list)
gid_list = sorted(list(set(gid_list)))
imageset_text = 'GGR2,%d,%s' % (number, letter)
note_list = [
'%s,%05d' % (imageset_text, index + 1)
for index, gid in enumerate(gid_list)
]
ibs.set_image_notes(gid_list, note_list)
ibs.set_image_imagesettext(gid_list,
[imageset_text] * len(gid_list))
except Exception as ex: # NOQA
ut.embed()
seen_letter_set = set(seen_letter_list)
try:
assert len(seen_letter_set) == len(seen_letter_list)
except AssertionError:
logger.info('Duplicate letters in %r with letters %r' %
(direct1, seen_letter_list))
continue
try:
assert 'A' in seen_letter_set
except AssertionError:
logger.info('WARNING: A camera not found in %r' % (direct1, ))
continue
return ibs
def get_layer_info_str(output_layer, batch_size=128):
r"""
Args:
output_layer (lasange.layers.Layer):
CommandLine:
python -m ibeis_cnn.net_strs --test-get_layer_info_str:0
python -m ibeis_cnn.net_strs --test-get_layer_info_str:1
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis_cnn.net_strs import * # NOQA
>>> from ibeis_cnn import models
>>> model = models.DummyModel(data_shape=(24, 24, 3), autoinit=True)
>>> output_layer = model.output_layer
>>> result = get_layer_info_str(output_layer)
>>> result = '\n'.join([x.rstrip() for x in result.split('\n')])
>>> print(result)
Network Structure:
index Layer Outputs Bytes OutShape Params
0 Input 1,728 55,296 (8, 3, 24, 24) []
1 Conv2D 7,744 249,600 (8, 16, 22, 22) [W(16,3,3,3, {t,r}), b(16, {t})]
2 Conv2D 7,056 229,952 (8, 16, 21, 21) [W(16,16,2,2, {t,r}), b(16, {t})]
3 Dense 8 226,080 (8, 8) [W(7056,8, {t,r}), b(8, {t})]
4 Dense 5 340 (8, 5) [W(8,5, {t,r}), b(5, {t})]
...this model has 57,989 learnable parameters
...this model will use 761,268 bytes = 743.43 KB
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis_cnn.net_strs import * # NOQA
>>> from ibeis_cnn import models
>>> model = models.mnist.MNISTModel(batch_size=128, output_dims=10,
>>> data_shape=(24, 24, 3))
>>> model.init_arch()
>>> output_layer = model.output_layer
>>> result = get_layer_info_str(output_layer)
>>> result = '\n'.join([x.rstrip() for x in result.split('\n')])
>>> print(result)
"""
import ibeis_cnn.__LASAGNE__ as lasagne
info_lines = []
_print = info_lines.append
with warnings.catch_warnings():
warnings.filterwarnings('ignore', '.*topo.*')
nn_layers = lasagne.layers.get_all_layers(output_layer)
_print('Network Structure:')
columns_ = ut.ddict(list)
for index, layer in enumerate(nn_layers):
layer_info = get_layer_info(layer)
columns_['index'].append(index)
columns_['name'].append(layer_info['name'])
#columns_['type'].append(getattr(layer, 'type', None))
#columns_['layer'].append(layer_info['classname'])
columns_['layer'].append(layer_info['classalias'])
columns_['num_outputs'].append('{:,}'.format(int(layer_info['num_outputs'])))
columns_['output_shape'].append(str(layer_info['output_shape'] ))
columns_['params'].append(layer_info['param_str'])
columns_['param_type'].append(layer_info['param_type_str'])
columns_['mem'].append(layer_info['total_memory'])
columns_['bytes'].append('{:,}'.format(int(layer_info['total_bytes'])))
#ut.embed()
header_nice = {
'index' : 'index',
'name' : 'Name',
'layer' : 'Layer',
'type' : 'Type',
'num_outputs' : 'Outputs',
'output_shape' : 'OutShape',
'params' : 'Params',
'param_type' : 'ParamType',
'mem' : 'Mem',
'bytes' : 'Bytes',
}
header_align = {
'index' : '<',
'params' : '<',
'bytes' : '>',
'num_outputs' : '>',
}
def get_col_maxval(key):
header_len = len(header_nice[key])
val_len = max(list(map(len, map(str, columns_[key]))))
return max(val_len, header_len)
header_order = ['index']
if len(ut.filter_Nones(columns_['name'])) > 0:
header_order += ['name']
header_order += ['layer', 'num_outputs']
#header_order += ['mem']
header_order += ['bytes']
header_order += ['output_shape', 'params' ]
#'param_type']
max_len = {key: str(get_col_maxval(key) + 1) for key, col in six.iteritems(columns_)}
fmtstr = ' ' + ' '.join(
[
'{:' + align + len_ + '}'
for align, len_ in zip(ut.dict_take(header_align, header_order, '<'),
ut.dict_take(max_len, header_order))
]
)
_print(fmtstr.format(*ut.dict_take(header_nice, header_order)))
row_list = zip(*ut.dict_take(columns_, header_order))
for row in row_list:
try:
row = ['' if _ is None else _ for _ in row]
str_ = fmtstr.format(*row)
_print(str_)
except TypeError:
print('Error printing %r with args %r' % (fmtstr, row, ))
total_bytes = count_bytes(output_layer)
num_params = lasagne.layers.count_params(output_layer)
_print('...this model has {:,} learnable parameters'.format(num_params))
_print('...this model will use ~{:,} bytes = {} per input'.format(
total_bytes, ut.byte_str2(total_bytes)))
_print('...this model will use ~{:,} bytes = {} per batch with a batch size of {}'.format(
total_bytes * batch_size, ut.byte_str2(total_bytes * batch_size), batch_size))
info_str = '\n'.join(info_lines)
return info_str
def make_inference(infr):
cm_list = infr.cm_list
unique_nids, prob_names = infr.make_prob_names()
cluster_tuples = infr.make_clusters()
# Make pair list for output
if infr.user_feedback is not None:
keys = list(
zip(infr.user_feedback['aid1'], infr.user_feedback['aid2']))
feedback_lookup = ut.make_index_lookup(keys)
user_feedback = infr.user_feedback
p_bg = 0
user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
part2 = p_bg * user_feedback['p_notcomp']
p_same_list = part1 + part2
else:
feedback_lookup = {}
infr.user_feedback
needs_review_list = []
num_top = 4
for cm, row in zip(cm_list, prob_names):
# Find top scoring names for this chip match in the posterior distribution
idxs = row.argsort()[::-1]
top_idxs = idxs[:num_top]
nids = ut.take(unique_nids, top_idxs)
# Find the matched annotations in the pairwise prior distributions
nidxs = ut.dict_take(cm.nid2_nidx, nids, None)
name_groupxs = ut.take(cm.name_groupxs, ut.filter_Nones(nidxs))
daids_list = ut.take(cm.daid_list, name_groupxs)
for daids in daids_list:
ut.take(cm.score_list, ut.take(cm.daid2_idx, daids))
scores_all = cm.annot_score_list / cm.annot_score_list.sum()
idxs = ut.take(cm.daid2_idx, daids)
scores = scores_all.take(idxs)
raw_scores = cm.score_list.take(idxs)
scorex = scores.argmax()
raw_score = raw_scores[scorex]
daid = daids[scorex]
import scipy.special
# SUPER HACK: these are not probabilities
# TODO: set a and b based on dbsize and param configuration
# python -m plottool.draw_func2 --exec-plot_func --show --range=0,3 --func="lambda x: scipy.special.expit(2 * x - 2)"
#a = 2.0
a = 1.5
b = 2
p_same = scipy.special.expit(b * raw_score - a)
#confidence = scores[scorex]
#p_diff = 1 - p_same
#decision = 'same' if confidence > thresh else 'diff'
#confidence = p_same if confidence > thresh else p_diff
#tup = (cm.qaid, daid, decision, confidence, raw_score)
confidence = (2 * np.abs(0.5 - p_same))**2
#if infr.user_feedback is not None:
# import utool
# utool.embed(
key = (cm.qaid, daid)
fb_idx = feedback_lookup.get(key)
if fb_idx is not None:
confidence = p_same_list[fb_idx]
tup = (cm.qaid, daid, p_same, confidence, raw_score)
needs_review_list.append(tup)
# Sort resulting list by confidence
sortx = ut.argsort(ut.take_column(needs_review_list, 3))
needs_review_list = ut.take(needs_review_list, sortx)
infr.needs_review_list = needs_review_list
infr.cluster_tuples = cluster_tuples
def show_name_matches(ibs, qaid, name_daid_list, name_fm_list, name_fs_list,
name_H1_list, name_featflag_list, qreq_=None, **kwargs):
"""
Called from chip_match.py
Args:
ibs (IBEISController): ibeis controller object
qaid (int): query annotation id
name_daid_list (list):
name_fm_list (list):
name_fs_list (list):
name_H1_list (list):
name_featflag_list (list):
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
show_nid, show_aid, show_annot_score, show_truth, name_score,
show_name_score, show_name_rank, show_timedelta
CommandLine:
python -m ibeis.viz.viz_matches --exec-show_name_matches
python -m ibeis.viz.viz_matches --test-show_name_matches --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_matches import * # NOQA
>>> from ibeis.algo.hots import chip_match
>>> from ibeis.algo.hots import name_scoring
>>> import vtool as vt
>>> from ibeis.algo.hots import _pipeline_helpers as plh # NOQA
>>> import numpy as np
>>> func = chip_match.ChipMatch.show_single_namematch
>>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
>>> strip_docstr=True)
>>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
>>> homog = False
>>> print(ut.indent(setup, '>>> '))
>>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
>>> cm = cm_list[0]
>>> cm.score_nsum(qreq_)
>>> dnid = ibs.get_annot_nids(cm.qaid)
>>> # +--- COPIED SECTION
>>> locals_ = locals()
>>> var_list = ut.exec_func_src(
>>> func, locals_=locals_,
>>> sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
>>> exec(ut.execstr_dict(var_list))
>>> # L___ COPIED SECTION
>>> kwargs = {}
>>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
>>> name_fs_list, name_h1_list, name_featflag_list,
>>> qreq_=qreq_, **kwargs)
>>> ut.quit_if_noshow()
>>> ut.show_if_requested()
"""
#print("SHOW NAME MATCHES")
#print(ut.repr2(kwargs, nl=True))
#from ibeis import constants as const
from ibeis import tag_funcs
draw_fmatches = kwargs.pop('draw_fmatches', True)
rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
qreq_, draw_fmatches)
fm_list = name_fm_list
fs_list = name_fs_list
featflag_list = name_featflag_list
offset_list, sf_list, bbox_list = show_multichip_match(rchip1, rchip2_list,
kpts1, kpts2_list,
fm_list, fs_list,
featflag_list,
**kwargs)
aid_list = [qaid] + name_daid_list
annotate_matches3(ibs, aid_list, bbox_list, offset_list, name_fm_list,
name_fs_list, qreq_=None, **kwargs)
ax = pt.gca()
title = vh.get_query_text(ibs, None, name_daid_list, False, qaid=qaid,
**kwargs)
pt.set_title(title, ax)
# Case tags
annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
[qaid] * len(name_daid_list), name_daid_list)
annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
if not ut.get_argflag('--show'): # False:
tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
tag_list = ut.unique_ordered(ut.flatten(tags_list))
name_rank = kwargs.get('name_rank', None)
truth = get_multitruth(ibs, aid_list)
xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]
if False:
if name_rank is None:
xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
#xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
else:
if name_rank == 0:
xlabel = {
1: 'True Positive', 0: 'False Positive', 2: 'Unknown'}[truth]
else:
xlabel = {
1: 'False Negative', 0: 'True Negative', 2: 'Unknown'}[truth]
if len(tag_list) > 0:
xlabel += '\n' + ', '.join(tag_list)
pt.set_xlabel(xlabel)
return ax
def make_inference(infr):
cm_list = infr.cm_list
unique_nids, prob_names = infr.make_prob_names()
cluster_tuples = infr.make_clusters()
# Make pair list for output
if infr.user_feedback is not None:
keys = list(zip(infr.user_feedback['aid1'], infr.user_feedback['aid2']))
feedback_lookup = ut.make_index_lookup(keys)
user_feedback = infr.user_feedback
p_bg = 0
user_feedback = ut.map_dict_vals(np.array, infr.user_feedback)
part1 = user_feedback['p_match'] * (1 - user_feedback['p_notcomp'])
part2 = p_bg * user_feedback['p_notcomp']
p_same_list = part1 + part2
else:
feedback_lookup = {}
infr.user_feedback
needs_review_list = []
num_top = 4
for cm, row in zip(cm_list, prob_names):
# Find top scoring names for this chip match in the posterior distribution
idxs = row.argsort()[::-1]
top_idxs = idxs[:num_top]
nids = ut.take(unique_nids, top_idxs)
# Find the matched annotations in the pairwise prior distributions
nidxs = ut.dict_take(cm.nid2_nidx, nids, None)
name_groupxs = ut.take(cm.name_groupxs, ut.filter_Nones(nidxs))
daids_list = ut.take(cm.daid_list, name_groupxs)
for daids in daids_list:
ut.take(cm.score_list, ut.take(cm.daid2_idx, daids))
scores_all = cm.annot_score_list / cm.annot_score_list.sum()
idxs = ut.take(cm.daid2_idx, daids)
scores = scores_all.take(idxs)
raw_scores = cm.score_list.take(idxs)
scorex = scores.argmax()
raw_score = raw_scores[scorex]
daid = daids[scorex]
import scipy.special
# SUPER HACK: these are not probabilities
# TODO: set a and b based on dbsize and param configuration
# python -m plottool.draw_func2 --exec-plot_func --show --range=0,3 --func="lambda x: scipy.special.expit(2 * x - 2)"
#a = 2.0
a = 1.5
b = 2
p_same = scipy.special.expit(b * raw_score - a)
#confidence = scores[scorex]
#p_diff = 1 - p_same
#decision = 'same' if confidence > thresh else 'diff'
#confidence = p_same if confidence > thresh else p_diff
#tup = (cm.qaid, daid, decision, confidence, raw_score)
confidence = (2 * np.abs(0.5 - p_same)) ** 2
#if infr.user_feedback is not None:
# import utool
# utool.embed(
key = (cm.qaid, daid)
fb_idx = feedback_lookup.get(key)
if fb_idx is not None:
confidence = p_same_list[fb_idx]
tup = (cm.qaid, daid, p_same, confidence, raw_score)
needs_review_list.append(tup)
# Sort resulting list by confidence
sortx = ut.argsort(ut.take_column(needs_review_list, 3))
needs_review_list = ut.take(needs_review_list, sortx)
infr.needs_review_list = needs_review_list
infr.cluster_tuples = cluster_tuples
def get_dbinfo(
ibs,
verbose=True,
with_imgsize=False,
with_bytes=False,
with_contrib=False,
with_agesex=False,
with_header=True,
short=False,
tag='dbinfo',
aid_list=None,
aids=None,
):
"""
Returns dictionary of digestable database information
Infostr is a string summary of all the stats. Prints infostr in addition to
returning locals
Args:
ibs (IBEISController):
verbose (bool):
with_imgsize (bool):
with_bytes (bool):
Returns:
dict:
SeeAlso:
python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --use-hist=True --old=False --per_name_vpedge=False
python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --all
CommandLine:
python -m wbia.other.dbinfo --exec-get_dbinfo:0
python -m wbia.other.dbinfo --test-get_dbinfo:1
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db LF_Bajo_bonito -a default
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db DETECT_SEATURTLES -a default --readonly
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a ctrl
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0
Example1:
>>> # SCRIPT
>>> from wbia.other.dbinfo import * # NOQA
>>> import wbia
>>> defaultdb = 'testdb1'
>>> ibs, aid_list = wbia.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
>>> kwargs = ut.get_kwdefaults(get_dbinfo)
>>> kwargs['verbose'] = False
>>> kwargs['aid_list'] = aid_list
>>> kwargs = ut.parse_dict_from_argv(kwargs)
>>> output = get_dbinfo(ibs, **kwargs)
>>> result = (output['info_str'])
>>> print(result)
>>> #ibs = wbia.opendb(defaultdb='testdb1')
>>> # <HACK FOR FILTERING>
>>> #from wbia.expt import cfghelpers
>>> #from wbia.expt import annotation_configs
>>> #from wbia.init import filter_annots
>>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
>>> # annotation_configs.TEST_NAMES)
>>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
>>> # ut.get_list_column(named_defaults_dict, 'qcfg')))
>>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
>>> #aid_list = ibs.get_valid_aids()
>>> # </HACK FOR FILTERING>
Example1:
>>> # ENABLE_DOCTEST
>>> from wbia.other.dbinfo import * # NOQA
>>> import wbia
>>> verbose = True
>>> short = True
>>> #ibs = wbia.opendb(db='GZ_ALL')
>>> #ibs = wbia.opendb(db='PZ_Master0')
>>> ibs = wbia.opendb('testdb1')
>>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
>>> ibs.delete_contributors(ibs.get_valid_contributor_rowids())
>>> ibs.delete_empty_nids()
>>> #ibs = wbia.opendb(db='PZ_MTEST')
>>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
>>> result = (output['info_str'])
>>> print(result)
+============================
DB Info: testdb1
DB Notes: None
DB NumContrib: 0
----------
# Names = 7
# Names (unassociated) = 0
# Names (singleton) = 5
# Names (multiton) = 2
----------
# Annots = 13
# Annots (unknown) = 4
# Annots (singleton) = 5
# Annots (multiton) = 4
----------
# Img = 13
L============================
"""
# TODO Database size in bytes
# TODO: occurrence, contributors, etc...
if aids is not None:
aid_list = aids
# Basic variables
request_annot_subset = False
_input_aid_list = aid_list # NOQA
if aid_list is None:
valid_aids = ibs.get_valid_aids()
valid_nids = ibs.get_valid_nids()
valid_gids = ibs.get_valid_gids()
else:
if isinstance(aid_list, str):
# Hack to get experiment stats on aids
acfg_name_list = [aid_list]
logger.info('Specified custom aids via acfgname %s' % (acfg_name_list,))
from wbia.expt import experiment_helpers
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs, acfg_name_list
)
aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
# aid_list =
if verbose:
logger.info('Specified %d custom aids' % (len(aid_list,)))
request_annot_subset = True
valid_aids = aid_list
valid_nids = list(
set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False))
- {const.UNKNOWN_NAME_ROWID}
)
valid_gids = list(set(ibs.get_annot_gids(aid_list)))
# associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation
valid_images = ibs.images(valid_gids)
valid_annots = ibs.annots(valid_aids)
# Image info
if verbose:
logger.info('Checking Image Info')
gx2_aids = valid_images.aids
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
gx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in gx2_aids]
gx2_nAnnots = np.array(list(map(len, gx2_aids)))
image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
gx2_nAnnots_stats = ut.repr4(
ut.get_stats(gx2_nAnnots, use_median=True), nl=0, precision=2, si=True
)
image_reviewed_list = ibs.get_image_reviewed(valid_gids)
# Name stats
if verbose:
logger.info('Checking Name Info')
nx2_aids = ibs.get_name_aids(valid_nids)
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
nx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in nx2_aids]
associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))
ibs.check_name_mapping_consistency(nx2_aids)
if False:
# Occurrence Info
def compute_annot_occurrence_ids(ibs, aid_list):
from wbia.algo.preproc import preproc_occurrence
gid_list = ibs.get_annot_gids(aid_list)
gid2_aids = ut.group_items(aid_list, gid_list)
config = {'seconds_thresh': 4 * 60 * 60}
flat_imgsetids, flat_gids = preproc_occurrence.wbia_compute_occurrences(
ibs, gid_list, config=config, verbose=False
)
occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
occurid2_aids = {
oid: ut.flatten(ut.take(gid2_aids, gids))
for oid, gids in occurid2_gids.items()
}
return occurid2_aids
import utool
with utool.embed_on_exception_context:
occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
nid2_occurxs = ut.ddict(list)
for occurx, nids in enumerate(occur_unique_nids):
for nid in nids:
nid2_occurxs[nid].append(occurx)
nid2_occurx_single = {
nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1
}
nid2_occurx_resight = {
nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1
}
singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())
singlesight_annot_stats = ut.get_stats(
list(map(len, singlesight_encounters)), use_median=True, use_sum=True
)
resight_name_stats = ut.get_stats(
list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True
)
# Encounter Info
def break_annots_into_encounters(aids):
from wbia.algo.preproc import occurrence_blackbox
import datetime
thresh_sec = datetime.timedelta(minutes=30).seconds
posixtimes = np.array(ibs.get_annot_image_unixtimes_asfloat(aids))
# latlons = ibs.get_annot_image_gps(aids)
labels = occurrence_blackbox.cluster_timespace2(
posixtimes, None, thresh_sec=thresh_sec
)
return labels
# ave_enc_time = [np.mean(times) for lbl, times in ut.group_items(posixtimes, labels).items()]
# ut.square_pdist(ave_enc_time)
try:
am_rowids = ibs.get_annotmatch_rowids_between_groups([valid_aids], [valid_aids])[
0
]
aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0, am_rowids=am_rowids)
undirected_tags = ibs.get_aidpair_tags(
aid_pairs.T[0], aid_pairs.T[1], directed=False
)
tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
pair_tag_info = ut.map_dict_vals(len, tag_dict)
except Exception:
pair_tag_info = {}
# logger.info(ut.repr2(pair_tag_info))
# Annot Stats
# TODO: number of images where chips cover entire image
# TODO: total image coverage of annotation
# TODO: total annotation overlap
"""
ax2_unknown = ibs.is_aid_unknown(valid_aids)
ax2_nid = ibs.get_annot_name_rowids(valid_aids)
assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
"""
#
if verbose:
logger.info('Checking Annot Species')
unknown_annots = valid_annots.compress(ibs.is_aid_unknown(valid_annots))
species_list = valid_annots.species_texts
species2_annots = valid_annots.group_items(valid_annots.species_texts)
species2_nAids = {key: len(val) for key, val in species2_annots.items()}
if verbose:
logger.info('Checking Multiton/Singleton Species')
nx2_nAnnots = np.array(list(map(len, nx2_aids)))
# Seperate singleton / multitons
multiton_nxs = np.where(nx2_nAnnots > 1)[0]
singleton_nxs = np.where(nx2_nAnnots == 1)[0]
unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
valid_nxs = np.hstack([multiton_nxs, singleton_nxs])
num_names_with_gt = len(multiton_nxs)
# Annot Info
if verbose:
logger.info('Checking Annot Info')
multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
assert len(set(multiton_nxs)) == len(multiton_nxs)
if len(multiton_aids_list) == 0:
multiton_aids = np.array([], dtype=np.int)
else:
multiton_aids = np.hstack(multiton_aids_list)
assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
singleton_aids = ut.take(nx2_aids, singleton_nxs)
multiton_nid2_nannots = list(map(len, multiton_aids_list))
# Image size stats
if with_imgsize:
if verbose:
logger.info('Checking ImageSize Info')
gpath_list = ibs.get_image_paths(valid_gids)
def wh_print_stats(wh_list):
if len(wh_list) == 0:
return '{empty}'
wh_list = np.asarray(wh_list)
stat_dict = collections.OrderedDict(
[
('max', wh_list.max(0)),
('min', wh_list.min(0)),
('mean', wh_list.mean(0)),
('std', wh_list.std(0)),
]
)
def arr2str(var):
return '[' + (', '.join(list(map(lambda x: '%.1f' % x, var)))) + ']'
ret = ',\n '.join(
['%s:%s' % (key, arr2str(val)) for key, val in stat_dict.items()]
)
return '{\n ' + ret + '\n}'
logger.info('reading image sizes')
# Image size stats
img_size_list = ibs.get_image_sizes(valid_gids)
img_size_stats = wh_print_stats(img_size_list)
# Chip size stats
annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
annotation_bbox_arr = np.array(annotation_bbox_list)
if len(annotation_bbox_arr) == 0:
annotation_size_list = []
else:
annotation_size_list = annotation_bbox_arr[:, 2:4]
chip_size_stats = wh_print_stats(annotation_size_list)
imgsize_stat_lines = [
(' # Img in dir = %d' % len(gpath_list)),
(' Image Size Stats = %s' % (img_size_stats,)),
(' * Chip Size Stats = %s' % (chip_size_stats,)),
]
else:
imgsize_stat_lines = []
if verbose:
logger.info('Building Stats String')
multiton_stats = ut.repr3(
ut.get_stats(multiton_nid2_nannots, use_median=True), nl=0, precision=2, si=True
)
# Time stats
unixtime_list = valid_images.unixtime2
# valid_unixtime_list = [time for time in unixtime_list if time != -1]
# unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
if ut.get_argflag('--hackshow-unixtime'):
show_time_distributions(ibs, unixtime_list)
ut.show_if_requested()
unixtime_statstr = ut.repr3(ut.get_timestats_dict(unixtime_list, full=True), si=True)
# GPS stats
gps_list_ = ibs.get_image_gps(valid_gids)
gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
gps_list = ut.compress(gps_list_, gpsvalid_list)
def get_annot_age_stats(aid_list):
annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
age_dict = ut.ddict((lambda: 0))
for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
if max_age is None:
max_age = min_age
if min_age is None:
min_age = max_age
if max_age is None and min_age is None:
logger.info('Found UNKNOWN Age: %r, %r' % (min_age, max_age,))
age_dict['UNKNOWN'] += 1
elif (min_age is None or min_age < 12) and max_age < 12:
age_dict['Infant'] += 1
elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
age_dict['Juvenile'] += 1
elif 36 <= min_age and (max_age is None or 36 <= max_age):
age_dict['Adult'] += 1
return age_dict
def get_annot_sex_stats(aid_list):
annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(
set(annot_sextext_list) - set(sex_keys)
)
sextext2_nAnnots = ut.odict(
[(key, len(sextext2_aids.get(key, []))) for key in sex_keys]
)
# Filter 0's
sextext2_nAnnots = {
key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0
}
return sextext2_nAnnots
def get_annot_qual_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
qualtext2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.quality_texts)),
list(ibs.const.QUALITY_TEXT_TO_INT.keys()),
)
return qualtext2_nAnnots
def get_annot_viewpoint_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
viewcode2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.viewpoint_code)),
list(ibs.const.VIEW.CODE_TO_INT.keys()) + [None],
)
return viewcode2_nAnnots
if verbose:
logger.info('Checking Other Annot Stats')
qualtext2_nAnnots = get_annot_qual_stats(ibs, valid_aids)
viewcode2_nAnnots = get_annot_viewpoint_stats(ibs, valid_aids)
agetext2_nAnnots = get_annot_age_stats(valid_aids)
sextext2_nAnnots = get_annot_sex_stats(valid_aids)
if verbose:
logger.info('Checking Contrib Stats')
# Contributor Statistics
# hack remove colon for image alignment
def fix_tag_list(tag_list):
return [None if tag is None else tag.replace(':', ';') for tag in tag_list]
image_contributor_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
annot_contributor_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
contributor_tag_to_gids = ut.group_items(valid_gids, image_contributor_tags)
contributor_tag_to_aids = ut.group_items(valid_aids, annot_contributor_tags)
contributor_tag_to_qualstats = {
key: get_annot_qual_stats(ibs, aids)
for key, aids in six.iteritems(contributor_tag_to_aids)
}
contributor_tag_to_viewstats = {
key: get_annot_viewpoint_stats(ibs, aids)
for key, aids in six.iteritems(contributor_tag_to_aids)
}
contributor_tag_to_nImages = {
key: len(val) for key, val in six.iteritems(contributor_tag_to_gids)
}
contributor_tag_to_nAnnots = {
key: len(val) for key, val in six.iteritems(contributor_tag_to_aids)
}
if verbose:
logger.info('Summarizing')
# Summarize stats
num_names = len(valid_nids)
num_names_unassociated = len(valid_nids) - len(associated_nids)
num_names_singleton = len(singleton_nxs)
num_names_multiton = len(multiton_nxs)
num_singleton_annots = len(singleton_aids)
num_multiton_annots = len(multiton_aids)
num_unknown_annots = len(unknown_annots)
num_annots = len(valid_aids)
if with_bytes:
if verbose:
logger.info('Checking Disk Space')
ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))
if True:
if verbose:
logger.info('Check asserts')
try:
bad_aids = np.intersect1d(multiton_aids, unknown_annots)
_num_names_total_check = (
num_names_singleton + num_names_unassociated + num_names_multiton
)
_num_annots_total_check = (
num_unknown_annots + num_singleton_annots + num_multiton_annots
)
assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
assert _num_names_total_check == num_names, 'inconsistent num names'
# if not request_annot_subset:
# dont check this if you have an annot subset
assert _num_annots_total_check == num_annots, 'inconsistent num annots'
except Exception as ex:
ut.printex(
ex,
keys=[
'_num_names_total_check',
'num_names',
'_num_annots_total_check',
'num_annots',
'num_names_singleton',
'num_names_multiton',
'num_unknown_annots',
'num_multiton_annots',
'num_singleton_annots',
],
)
raise
# Get contributor statistics
contributor_rowids = ibs.get_valid_contributor_rowids()
num_contributors = len(contributor_rowids)
# print
num_tabs = 5
def align2(str_):
return ut.align(str_, ':', ' :')
def align_dict2(dict_):
str_ = ut.repr2(dict_, si=True)
return align2(str_)
header_block_lines = [('+============================')] + (
[
('+ singleton := single sighting'),
('+ multiton := multiple sightings'),
('--' * num_tabs),
]
if not short and with_header
else []
)
source_block_lines = [
('DB Info: ' + ibs.get_dbname()),
('DB Notes: ' + ibs.get_dbnotes()),
('DB NumContrib: %d' % num_contributors),
]
bytes_block_lines = (
[
('--' * num_tabs),
('DB Bytes: '),
(' +- dbdir nBytes: ' + dbdir_space),
(' | +- _ibsdb nBytes: ' + ibsdir_space),
(' | | +-imgdir nBytes: ' + imgdir_space),
(' | | +-cachedir nBytes: ' + cachedir_space),
]
if with_bytes
else []
)
name_block_lines = [
('--' * num_tabs),
('# Names = %d' % num_names),
('# Names (unassociated) = %d' % num_names_unassociated),
('# Names (singleton) = %d' % num_names_singleton),
('# Names (multiton) = %d' % num_names_multiton),
]
subset_str = ' ' if not request_annot_subset else '(SUBSET)'
annot_block_lines = [
('--' * num_tabs),
('# Annots %s = %d' % (subset_str, num_annots,)),
('# Annots (unknown) = %d' % num_unknown_annots),
('# Annots (singleton) = %d' % num_singleton_annots),
('# Annots (multiton) = %d' % num_multiton_annots),
]
annot_per_basic_block_lines = (
[
('--' * num_tabs),
('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)),
('# Annots per Species = %s' % (align_dict2(species2_nAids),)),
]
if not short
else []
)
occurrence_block_lines = (
[
('--' * num_tabs),
# ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
# ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
]
if not short
else []
)
annot_per_qualview_block_lines = [
None if short else '# Annots per Viewpoint = %s' % align_dict2(viewcode2_nAnnots),
None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
]
annot_per_agesex_block_lines = (
[
'# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
'# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
]
if not short and with_agesex
else []
)
contributor_block_lines = (
[
'# Images per contributor = ' + align_dict2(contributor_tag_to_nImages),
'# Annots per contributor = ' + align_dict2(contributor_tag_to_nAnnots),
'# Quality per contributor = '
+ ut.repr2(contributor_tag_to_qualstats, sorted_=True),
'# Viewpoint per contributor = '
+ ut.repr2(contributor_tag_to_viewstats, sorted_=True),
]
if with_contrib
else []
)
img_block_lines = [
('--' * num_tabs),
('# Img = %d' % len(valid_gids)),
None
if short
else ('# Img reviewed = %d' % sum(image_reviewed_list)),
None if short else ('# Img with gps = %d' % len(gps_list)),
# ('# Img with timestamp = %d' % len(valid_unixtime_list)),
None
if short
else ('Img Time Stats = %s' % (align2(unixtime_statstr),)),
]
info_str_lines = (
header_block_lines
+ bytes_block_lines
+ source_block_lines
+ name_block_lines
+ annot_block_lines
+ annot_per_basic_block_lines
+ occurrence_block_lines
+ annot_per_qualview_block_lines
+ annot_per_agesex_block_lines
+ img_block_lines
+ contributor_block_lines
+ imgsize_stat_lines
+ [('L============================')]
)
info_str = '\n'.join(ut.filter_Nones(info_str_lines))
info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
if verbose:
logger.info(info_str2)
locals_ = locals()
return locals_
def stagger_group(list_):
return ut.filter_Nones(ut.iflatten(zip_longest(*list_)))
def make_netx_graph_from_aid_groups(ibs, aids_list, only_reviewed_matches=True,
invis_edges=None, ensure_edges=None,
temp_nids=None, allow_directed=False):
r"""
Args:
ibs (ibeis.IBEISController): image analysis api
aids_list (list):
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_graph import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> aids_list = [[1, 2, 3, 4], [5, 6, 7]]
>>> invis_edges = [(1, 5)]
>>> only_reviewed_matches = True
>>> graph = make_netx_graph_from_aid_groups(ibs, aids_list,
>>> only_reviewed_matches,
>>> invis_edges)
>>> list(nx.connected_components(graph.to_undirected()))
"""
#aids_list, nid_list = ibs.group_annots_by_name(aid_list)
unique_aids = list(ut.flatten(aids_list))
# grouped version
unflat_edges = (list(itertools.product(aids, aids)) for aids in aids_list)
aid_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
aids1 = ut.get_list_column(aid_pairs, 0)
aids2 = ut.get_list_column(aid_pairs, 1)
if only_reviewed_matches:
annotmatch_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2)
annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)
graph = make_netx_graph_from_aidpairs(ibs, aids1, aids2, unique_aids=unique_aids)
if ensure_edges is not None:
if ensure_edges == 'all':
ensure_edges = list(ut.upper_diag_self_prodx(list(graph.nodes())))
ensure_edges_ = []
for edge in ensure_edges:
edge = tuple(edge)
redge = tuple(edge[::-1]) # HACK
if graph.has_edge(*edge):
ensure_edges_.append(edge)
pass
#nx.set_edge_attributes(graph, 'weight', {edge: .001})
elif (not allow_directed) and graph.has_edge(*redge):
ensure_edges_.append(redge)
#nx.set_edge_attributes(graph, 'weight', {redge: .001})
pass
else:
ensure_edges_.append(edge)
#graph.add_edge(*edge, weight=.001)
graph.add_edge(*edge)
if temp_nids is None:
unique_nids = ibs.get_annot_nids(list(graph.nodes()))
else:
# HACK
unique_nids = [1] * len(list(graph.nodes()))
#unique_nids = temp_nids
nx.set_node_attributes(graph, 'nid', dict(zip(graph.nodes(), unique_nids)))
import plottool as pt
ensure_names_are_connected(graph, aids_list)
# Color edges by nid
color_by_nids(graph, unique_nids=unique_nids)
if invis_edges:
for edge in invis_edges:
if graph.has_edge(*edge):
nx.set_edge_attributes(graph, 'style', {edge: 'invis'})
nx.set_edge_attributes(graph, 'invisible', {edge: True})
else:
graph.add_edge(*edge, style='invis', invisible=True)
# Hack color images orange
if ensure_edges:
nx.set_edge_attributes(graph, 'color',
{tuple(edge): pt.ORANGE for edge in ensure_edges_})
return graph
def show_name_matches(ibs,
qaid,
name_daid_list,
name_fm_list,
name_fs_list,
name_H1_list,
name_featflag_list,
qreq_=None,
**kwargs):
"""
Called from chip_match.py
Args:
ibs (IBEISController): ibeis controller object
qaid (int): query annotation id
name_daid_list (list):
name_fm_list (list):
name_fs_list (list):
name_H1_list (list):
name_featflag_list (list):
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
show_nid, show_aid, show_annot_score, show_truth, name_score,
show_name_score, show_name_rank, show_timedelta
CommandLine:
python -m ibeis.viz.viz_matches --exec-show_name_matches
python -m ibeis.viz.viz_matches --test-show_name_matches --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_matches import * # NOQA
>>> from ibeis.algo.hots import chip_match
>>> from ibeis.algo.hots import name_scoring
>>> import vtool as vt
>>> from ibeis.algo.hots import _pipeline_helpers as plh # NOQA
>>> import numpy as np
>>> func = chip_match.ChipMatch.show_single_namematch
>>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
>>> strip_docstr=True)
>>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
>>> homog = False
>>> print(ut.indent(setup, '>>> '))
>>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
>>> cm = cm_list[0]
>>> cm.score_nsum(qreq_)
>>> dnid = ibs.get_annot_nids(cm.qaid)
>>> # +--- COPIED SECTION
>>> locals_ = locals()
>>> var_list = ut.exec_func_src(
>>> func, locals_=locals_,
>>> sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
>>> exec(ut.execstr_dict(var_list))
>>> # L___ COPIED SECTION
>>> kwargs = {}
>>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
>>> name_fs_list, name_h1_list, name_featflag_list,
>>> qreq_=qreq_, **kwargs)
>>> ut.quit_if_noshow()
>>> ut.show_if_requested()
"""
#print("SHOW NAME MATCHES")
#print(ut.repr2(kwargs, nl=True))
#from ibeis import constants as const
from ibeis import tag_funcs
draw_fmatches = kwargs.pop('draw_fmatches', True)
rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
qreq_, draw_fmatches)
fm_list = name_fm_list
fs_list = name_fs_list
featflag_list = name_featflag_list
offset_list, sf_list, bbox_list = show_multichip_match(
rchip1, rchip2_list, kpts1, kpts2_list, fm_list, fs_list,
featflag_list, **kwargs)
aid_list = [qaid] + name_daid_list
annotate_matches3(ibs,
aid_list,
bbox_list,
offset_list,
name_fm_list,
name_fs_list,
qreq_=None,
**kwargs)
ax = pt.gca()
title = vh.get_query_text(ibs,
None,
name_daid_list,
False,
qaid=qaid,
**kwargs)
pt.set_title(title, ax)
# Case tags
annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
[qaid] * len(name_daid_list), name_daid_list)
annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
if not ut.get_argflag('--show'): # False:
tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
tag_list = ut.unique_ordered(ut.flatten(tags_list))
name_rank = kwargs.get('name_rank', None)
truth = get_multitruth(ibs, aid_list)
xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]
if False:
if name_rank is None:
xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
#xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
else:
if name_rank == 0:
xlabel = {
1: 'True Positive',
0: 'False Positive',
2: 'Unknown'
}[truth]
else:
xlabel = {
1: 'False Negative',
0: 'True Negative',
2: 'Unknown'
}[truth]
if len(tag_list) > 0:
xlabel += '\n' + ', '.join(tag_list)
pt.set_xlabel(xlabel)
return ax
def make_score_tabular(
row_lbls, col_lbls, values, title=None, out_of=None, bold_best=False,
flip=False, bigger_is_better=True, multicol_lbls=None, FORCE_INT=False,
precision=None, SHORTEN_ROW_LBLS=False, col_align='l', col_sep='|',
multicol_sep='|', centerline=True, astable=False, table_position='',
AUTOFIX_LATEX=True, **kwargs):
r"""
makes a LaTeX tabular for displaying scores or errors
Args:
row_lbls (list of str):
col_lbls (list of str):
values (ndarray):
title (str): (default = None)
out_of (None): (default = None)
bold_best (bool): (default = True)
flip (bool): (default = False)
table_position (str) : eg '[h]'
Returns:
str: tabular_str
CommandLine:
python -m utool.util_latex --test-make_score_tabular:0 --show
python -m utool.util_latex --test-make_score_tabular:1 --show
python -m utool.util_latex --test-make_score_tabular:2 --show
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2], [3.2, 4]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1']
>>> col_lbls = ['score \leq 1', 'metric2']
>>> values = np.array([[1.2, 2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_latex import * # NOQA
>>> import utool as ut
>>> row_lbls = ['config1', 'config2']
>>> col_lbls = ['score \leq 1', 'metric2', 'foobar']
>>> multicol_lbls = [('spam', 1), ('eggs', 2)]
>>> values = np.array([[1.2, 2, -3], [3.2, 4, -2]])
>>> title = 'title'
>>> out_of = 10
>>> bold_best = True
>>> flip = False
>>> tabular_str = make_score_tabular(row_lbls, col_lbls, values, title, out_of, bold_best, flip, multicol_lbls=multicol_lbls)
>>> result = tabular_str
>>> print(result)
>>> ut.quit_if_noshow()
>>> render_latex_text(tabular_str)
"""
import utool as ut
if flip:
bigger_is_better = not bigger_is_better
flip_repltups = [
('<=', '>'),
('>', '<='),
('\\leq', '\\gt'),
('\\geq', '\\lt'),
('score', 'error')
]
col_lbls = [replace_all(lbl, flip_repltups) for lbl in col_lbls]
if title is not None:
title = replace_all(title, flip_repltups)
if out_of is not None:
values = out_of - values
# Abbreviate based on common substrings
common_rowlbl = None
if SHORTEN_ROW_LBLS:
if isinstance(row_lbls, list):
row_lbl_list = row_lbls
else:
row_lbl_list = row_lbls.flatten().tolist()
# Split the rob labels into the alg components
#algcomp_list = [lbl.split(')_') for lbl in row_lbl_list]
longest = long_substr(row_lbl_list)
common_strs = []
while len(longest) > 10:
common_strs += [longest]
row_lbl_list = [row.replace(longest, '...') for row in row_lbl_list]
longest = long_substr(row_lbl_list)
common_rowlbl = ('...'.join(common_strs)).replace(')_', ')_\n')
row_lbls = row_lbl_list
if len(row_lbl_list) == 1:
common_rowlbl = row_lbl_list[0]
row_lbls = ['0']
# Stack values into a tabular body
# TODO: need ability to specify datatypes
def ensurelist(row_values):
try:
return row_values.tolist()
except AttributeError:
return row_values
if False:
# Numpy formatting
def padvec(shape=(1, 1)):
pad = np.array([[' ' for c in range(shape[1])] for r in range(shape[0])])
return pad
col_lbls = ensure_rowvec(col_lbls)
row_lbls = ensure_colvec(row_lbls)
_0 = np.vstack([padvec(), row_lbls])
_1 = np.vstack([col_lbls, values])
body = np.hstack([_0, _1])
body = [[str_ for str_ in row] for row in body]
else:
assert len(row_lbls) == len(values)
body = [[' '] + col_lbls]
body += [[row_lbl] + ensurelist(row_values) for row_lbl, row_values in zip(row_lbls, values)]
#import utool as ut
# Fix things in each body cell
DO_PERCENT = True
try:
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if precision is not None:
# Hack
if ut.is_float(body[r][c]):
fmtstr = '%.' + str(precision) + 'f'
body[r][c] = fmtstr % (float(body[r][c]),)
# Force integer
if FORCE_INT:
body[r][c] = str(int(float(body[r][c])))
body[r][c] = str(body[r][c])
# Remove bad formatting;
if AUTOFIX_LATEX:
body[r][c] = escape_latex(body[r][c])
except Exception as ex:
import utool as ut
print('len(row_lbls) = %r' % (len(row_lbls),))
print('len(col_lbls) = %r' % (len(col_lbls),))
print('len(values) = %r' % (values,))
print('ut.depth_profile(values) = %r' % (ut.depth_profile(values),))
util_dbg.printex(ex, keys=['r', 'c'])
raise
# Bold the best values
if bold_best:
best_col_scores = values.max(0) if bigger_is_better else values.min(0)
rows_to_bold = [np.where(values[:, colx] == best_col_scores[colx])[0]
for colx in range(len(values.T))]
for colx, rowx_list in enumerate(rows_to_bold):
for rowx in rowx_list:
body[rowx + 1][colx + 1] = '\\txtbf{' + body[rowx + 1][colx + 1] + '}'
# More fixing after the bold is in place
for r in range(len(body)):
for c in range(len(body[0])):
# In data land
if r > 0 and c > 0:
if out_of is not None:
body[r][c] = body[r][c] + '/' + str(out_of)
if DO_PERCENT:
percent = ' = %.1f%%' % float(100 * values[r - 1, c - 1] / out_of)
body[r][c] += escape_latex(percent)
# Align columns for pretty printing
body = np.array(body)
ALIGN_BODY = True
if ALIGN_BODY:
new_body_cols = []
for col in body.T:
colstrs = list(map(str, ensurelist(col)))
collens = list(map(len, colstrs))
maxlen = max(collens)
newcols = [str_ + (' ' * (maxlen - len(str_))) for str_ in colstrs]
new_body_cols += [newcols]
body = np.array(new_body_cols).T
# Build Body (and row layout)
HLINE_SEP = True
rowvalsep = ''
colvalsep = ' & '
endl = '\\\\\n'
hline = r'\hline'
#extra_rowsep_pos_list = [1] # rows to insert an extra hline after
extra_rowsep_pos_list = [] # rows to insert an extra hline after
if HLINE_SEP:
rowvalsep = hline + '\n'
# rowstr list holds blocks of rows
rowstr_list = [colvalsep.join(row) + endl for row in body]
#rowstr_list = [row[0] + rowlbl_sep + colvalsep.join(row[1:]) + endl for row in body]
#rowstr_list = [(
# ('' if len(row) == 0 else row[0])
# if len(row) <= 1 else
# row[0] + rowlblcol_sep + colvalsep.join(row[1:]) + endl)
# for row in body]
rowsep_list = [rowvalsep for row in rowstr_list[0:-1]] # should be len 1 less than rowstr_list
# Insert multicolumn names
if multicol_lbls is not None:
# TODO: label of the row labels
multicol_sep
multicols = [latex_multicolumn(multicol, size, 'c' + multicol_sep) for multicol, size in multicol_lbls]
multicol_str = latex_multirow('', 2) + colvalsep + colvalsep.join(multicols) + endl
ncols = sum([tup[1] for tup in multicol_lbls])
mcol_sep = '\\cline{2-%d}\n' % (ncols + 1,)
rowstr_list = [multicol_str] + rowstr_list
rowsep_list = [mcol_sep] + rowsep_list
#extra_rowsep_pos_list += [1]
# Insert title
if title is not None and not astable:
tex_title = latex_multicolumn(title, len(body[0])) + endl
rowstr_list = [tex_title] + rowstr_list
rowsep_list = [rowvalsep] + rowsep_list
#extra_rowsep_pos_list += [2]
# Apply an extra hline (for label)
#extra_rowsep_pos_list = []
for pos in sorted(extra_rowsep_pos_list)[::-1]:
rowstr_list.insert(pos, '')
rowsep_list.insert(pos, rowvalsep)
#tabular_body = rowvalsep.join(rowstr_list)
from six.moves import zip_longest
tabular_body = ''.join([row if sep is None else row + sep for row, sep in zip_longest(rowstr_list, rowsep_list)])
# Build Column Layout
col_align_list = [col_align] * len(body[0])
#extra_collayoutsep_pos_list = [1]
extra_collayoutsep_pos_list = []
for pos in sorted(extra_collayoutsep_pos_list)[::-1]:
col_align_list.insert(pos, '')
#col_layaout_sep_list = rowlblcol_sep # TODO
rowlblcol_sep = '|'
# Build build internal seprations between column alignments
# Defaults to just the normal col_sep
col_align_sep_list = [col_sep] * (len(col_align_list) - 1)
# Adjust for the separations between row labels and the actual row data
if len(col_align_sep_list) > 0:
col_align_sep_list[0] = rowlblcol_sep
# Continue multicolumn sepratation
if multicol_lbls is not None:
multicol_offsets = ut.cumsum(ut.get_list_column(multicol_lbls, 1))
for offset in multicol_offsets:
if offset < len(col_align_sep_list):
col_align_sep_list[offset] = multicol_sep
from six.moves import zip_longest
_tmp = [ut.filter_Nones(tup) for tup in zip_longest(col_align_list, col_align_sep_list)]
col_layout = ''.join(ut.flatten(_tmp))
#if len(col_align_list) > 1:
# col_layout = col_align_list[0] + rowlblcol_sep + col_sep.join(col_align_list[1:])
#else:
# col_layout = col_sep.join(col_align_list)
tabular_head = (r'\begin{tabular}{|%s|}' % col_layout) + '\n'
tabular_tail = r'\end{tabular}'
if centerline:
tabular_head = r'\centerline{' + '\n' + tabular_head
tabular_tail = tabular_tail + '}'
if astable:
#tabular_head = r'\begin{centering}' + '\n' + tabular_head
tabular_head = r'\centering' + '\n' + tabular_head
tabular_head = r'\begin{table}' + table_position + '\n' + tabular_head
lblstr = latex_sanitize_command_name(kwargs.get('label', title))
caption = title
if AUTOFIX_LATEX:
caption = escape_latex(caption)
caption = '\n% ---\n' + caption + '\n% ---\n'
#tabular_head = r'\end{centering}' + '\n' + tabular_head
tabular_tail = tabular_tail + '\n\caption[%s]{%s}\n\label{tbl:%s}\n\end{table}' % (lblstr, caption, lblstr)
tabular_str = rowvalsep.join([tabular_head, tabular_body, tabular_tail])
topsep = '\\hline\n' if True else '\\toprule\n'
botsep = '\\hline\n' if True else '\\bottomrule\n'
tabular_str = tabular_head + topsep + tabular_body + botsep + tabular_tail
if common_rowlbl is not None:
#tabular_str += escape_latex('\n\nThe following parameters were held fixed:\n' + common_rowlbl)
pass
return tabular_str