def get_allkeys(dict_):
import utool as ut
if not isinstance(dict_, dict):
return []
subkeys = [[key] + get_allkeys(val)
for key, val in dict_.items()]
return ut.unique_ordered(ut.flatten(subkeys))
def add_to_win32_PATH(script_fpath, *add_path_list):
r"""
Writes a registery script to update the PATH variable into the sync registry
CommandLine:
python -m utool.util_win32 --test-add_to_win32_PATH --newpath "C:\Program Files (x86)\Graphviz2.38\bin"
Example:
>>> # SCRIPT
>>> from utool.util_win32 import * # NOQA
>>> script_fpath = join(ut.truepath('~'), 'Sync/win7/registry', 'UPDATE_PATH.reg')
>>> new_path = ut.get_argval('--newpath', str, default=None)
>>> result = add_to_win32_PATH(script_fpath, new_path)
>>> print(result)
"""
import utool as ut
write_dir = dirname(script_fpath)
key = '[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\Environment]'
rtype = 'REG_EXPAND_SZ'
# Read current PATH values
win_pathlist = list(os.environ['PATH'].split(os.path.pathsep))
new_path_list = ut.unique_ordered(win_pathlist + list(add_path_list))
#new_path_list = unique_ordered(win_pathlist, rob_pathlist)
print('\n'.join(new_path_list))
pathtxt = pathsep.join(new_path_list)
varval_list = [('Path', pathtxt)]
regfile_str = make_regfile_str(key, varval_list, rtype)
ut.view_directory(write_dir)
print(regfile_str)
ut.writeto(script_fpath, regfile_str, mode='wb')
print('Please have an admin run the script. You may need to restart')
def get_column_keys(metadata):
unflat_colname_list = [[
cols.keys() for cols in qaid2_cols.values()
] for qaid2_cols in six.itervalues(metadata.dictstore)]
colname_list = ut.unique_ordered(
ut.flatten(ut.flatten(unflat_colname_list)))
return colname_list
def write_path(r):
"""
Writes a script to update the PATH variable into the sync registry
The PATH update mirrors the current RobSettings
SeeAlso:
utool.util_win32.add_to_win32_PATH
"""
import utool
write_dir = join(r.d.HOME, 'Sync/win7/registry')
path_fpath = normpath(join(write_dir, 'UPDATE_PATH.reg'))
key = '[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\Environment]'
rtype = 'REG_EXPAND_SZ'
pathsep = os.path.pathsep
# Read current PATH values
win_pathlist = list(os.environ['PATH'].split(os.path.pathsep))
rob_pathlist = list(map(normpath, r.path_vars_list))
new_path_list = utool.unique_ordered(win_pathlist + rob_pathlist)
#new_path_list = unique_ordered(win_pathlist, rob_pathlist)
print('\n'.join(new_path_list))
pathtxt = pathsep.join(new_path_list)
varval_list = [('Path', pathtxt)]
write_regfile(path_fpath, key, varval_list, rtype)
rob_helpers.view_directory(write_dir)
def add_to_win32_PATH(script_fpath, *add_path_list):
r"""
Writes a registery script to update the PATH variable into the sync registry
CommandLine:
python -m utool.util_win32 --test-add_to_win32_PATH --newpath "C:\Program Files (x86)\Graphviz2.38\bin"
Example:
>>> # SCRIPT
>>> from utool.util_win32 import * # NOQA
>>> script_fpath = join(ut.truepath('~'), 'Sync/win7/registry', 'UPDATE_PATH.reg')
>>> new_path = ut.get_argval('--newpath', str, default=None)
>>> result = add_to_win32_PATH(script_fpath, new_path)
>>> print(result)
"""
import utool as ut
write_dir = dirname(script_fpath)
key = '[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\Environment]'
rtype = 'REG_EXPAND_SZ'
# Read current PATH values
win_pathlist = list(os.environ['PATH'].split(os.path.pathsep))
new_path_list = ut.unique_ordered(win_pathlist + list(add_path_list))
#new_path_list = unique_ordered(win_pathlist, rob_pathlist)
print('\n'.join(new_path_list))
pathtxt = pathsep.join(new_path_list)
varval_list = [('Path', pathtxt)]
regfile_str = make_regfile_str(key, varval_list, rtype)
ut.view_directory(write_dir)
print(regfile_str)
ut.writeto(script_fpath, regfile_str, mode='wb')
print('Please have an admin run the script. You may need to restart')
def get_column_keys(metadata):
unflat_colname_list = [
[cols.keys() for cols in qaid2_cols.values()]
for qaid2_cols in six.itervalues(metadata.dictstore)
]
colname_list = ut.unique_ordered(ut.flatten(ut.flatten(unflat_colname_list)))
return colname_list
def get_other_nids(self):
ibs = self.ibs
all_nid_list = ibs.get_annot_name_rowids(self.all_aid_list)
unique_nid_list = ut.unique_ordered(all_nid_list)
is_unknown = ibs.is_nid_unknown(unique_nid_list)
is_name1 = [nid == self.nid1 for nid in unique_nid_list]
is_name2 = [nid == self.nid2 for nid in unique_nid_list]
is_other = ut.and_lists(*tuple(map(ut.not_list, (is_name1, is_name2, is_unknown))))
other_nid_list = ut.compress(unique_nid_list, is_other)
return other_nid_list
def get_other_nids(self):
ibs = self.ibs
all_nid_list = ibs.get_annot_name_rowids(self.all_aid_list)
unique_nid_list = ut.unique_ordered(all_nid_list)
is_unknown = ibs.is_nid_unknown(unique_nid_list)
is_name1 = [nid == self.nid1 for nid in unique_nid_list]
is_name2 = [nid == self.nid2 for nid in unique_nid_list]
is_other = ut.and_lists(
*tuple(map(ut.not_list, (is_name1, is_name2, is_unknown))))
other_nid_list = ut.compress(unique_nid_list, is_other)
return other_nid_list
def parse_all_fname_tags(fname):
_tags = [splitext(fname)[0]]
_tags = ut.flatten([t.split('_') for t in _tags])
_tags = ut.flatten([t.split('.') for t in _tags])
_tags = [t.lower() for t in _tags]
_tags = [tag_alias_map.get(t, t) for t in _tags]
for key, vals in regex_alias_map.items():
pat = ut.regex_or(vals)
_tags = [key if re.match(pat, t) else t for t in _tags]
pat = ut.regex_or(invalid_tag_patterns)
_tags = [t for t in _tags if not re.match(pat, t)]
_tags = ut.unique_ordered(_tags)
return _tags
def parse_all_fname_tags(fname):
_tags = [splitext(fname)[0]]
_tags = ut.flatten([t.split('_') for t in _tags])
_tags = ut.flatten([t.split('.') for t in _tags])
_tags = [t.lower() for t in _tags]
_tags = [tag_alias_map.get(t, t) for t in _tags]
for key, vals in regex_alias_map.items():
pat = ut.regex_or(vals)
_tags = [key if re.match(pat, t) else t for t in _tags]
pat = ut.regex_or(invalid_tag_patterns)
_tags = [t for t in _tags if not re.match(pat, t)]
_tags = ut.unique_ordered(_tags)
return _tags
def get_system_python_library():
"""
FIXME; hacky way of finding python library. Not cross platform yet.
"""
import os
import utool as ut
from os.path import basename, realpath
pyname = basename(realpath(sys.executable))
ld_library_path = os.environ['LD_LIBRARY_PATH']
libdirs = [x for x in ld_library_path.split(os.pathsep) if x] + ['/usr/lib']
libfiles = ut.flatten([ut.glob(d, '*' + ut.get_lib_ext(), recursive=True) for d in libdirs])
python_libs = [realpath(f) for f in libfiles if 'lib' + pyname in basename(f)]
python_libs = ut.unique_ordered(python_libs)
assert len(python_libs) == 1, str(python_libs)
return python_libs[0]
def get_square_data(metadata, cfgstr=None):
# can only support one config at a time right now
if cfgstr is None:
cfgstr = metadata.get_cfgstr_list()[0]
qaid2_cols = metadata.dictstore[cfgstr]
qaids = list(qaid2_cols.keys())
col_name_list = ut.unique_ordered(ut.flatten([cols.keys() for cols in qaid2_cols.values()]))
#col_name_list = ['qx2_scoreexpdiff', 'qx2_gt_aid']
#colname2_colvals = [None for colname in col_name_list]
column_list = [
[colvals.get(colname, None) for qaid, colvals in six.iteritems(qaid2_cols)]
for colname in col_name_list]
col_name_list = ['qaids'] + col_name_list
column_list = [qaids] + column_list
print('depth_profile(column_list) = %r' % (ut.depth_profile(column_list),))
return col_name_list, column_list
def get_square_data(metadata, cfgstr=None):
# can only support one config at a time right now
if cfgstr is None:
cfgstr = metadata.get_cfgstr_list()[0]
qaid2_cols = metadata.dictstore[cfgstr]
qaids = list(qaid2_cols.keys())
col_name_list = ut.unique_ordered(ut.flatten([cols.keys() for cols in qaid2_cols.values()]))
#col_name_list = ['qx2_scoreexpdiff', 'qx2_gt_aid']
#colname2_colvals = [None for colname in col_name_list]
column_list = [
[colvals.get(colname, None) for qaid, colvals in six.iteritems(qaid2_cols)]
for colname in col_name_list]
col_name_list = ['qaids'] + col_name_list
column_list = [qaids] + column_list
print('depth_profile(column_list) = %r' % (ut.depth_profile(column_list),))
return col_name_list, column_list
def find_module_callers():
"""
TODO:
attempt to build a call graph between module functions to make it easy to see
what can be removed and what cannot.
"""
import utool as ut
from os.path import normpath
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_analyzer.py')
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_all.py')
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_organizer.py')
module = ut.import_module_from_fpath(mod_fpath)
user_profile = ut.ensure_user_profile()
doctestables = list(
ut.iter_module_doctestable(module, include_builtin=False))
grepkw = {}
grepkw['exclude_dirs'] = user_profile.project_exclude_dirs
grepkw['dpath_list'] = user_profile.project_dpaths
grepkw['verbose'] = True
usage_map = {}
for funcname, func in doctestables:
print('Searching for funcname = %r' % (funcname, ))
found_fpath_list, found_lines_list, found_lxs_list = ut.grep(
[funcname], **grepkw)
used_in = (found_fpath_list, found_lines_list, found_lxs_list)
usage_map[funcname] = used_in
external_usage_map = {}
for funcname, used_in in usage_map.items():
(found_fpath_list, found_lines_list, found_lxs_list) = used_in
isexternal_flag = [
normpath(fpath) != normpath(mod_fpath)
for fpath in found_fpath_list
]
ext_used_in = (ut.compress(found_fpath_list, isexternal_flag),
ut.compress(found_lines_list, isexternal_flag),
ut.compress(found_lxs_list, isexternal_flag))
external_usage_map[funcname] = ext_used_in
for funcname, used_in in external_usage_map.items():
(found_fpath_list, found_lines_list, found_lxs_list) = used_in
print('Calling modules: \n' + ut.repr2(ut.unique_ordered(
ut.flatten([used_in[0]
for used_in in external_usage_map.values()])),
nl=True))
def infer_data(self):
""" Initialize data related to the input aids
"""
ibs = self.ibs
# The two matching aids
self.aid_pair = (self.aid1, self.aid2)
(aid1, aid2) = self.aid_pair
self.match_text = ibs.get_match_text(self.aid1, self.aid2)
# The names of the matching annotations
self.nid1, self.nid2 = ibs.get_annot_name_rowids((aid1, aid2))
self.name1, self.name2 = ibs.get_annot_names((aid1, aid2))
self.other_valid_nids = []
# The other annotations that belong to these two names
self.gts_list = ibs.get_annot_groundtruth((aid1, aid2))
self.gt1, self.gt2 = self.gts_list
# A flat list of all the aids we are looking at
self.is_split_case = self.nid1 == self.nid2
self.all_aid_list = ut.unique_ordered([aid1, aid2] + self.gt1 +
self.gt2)
self.all_nid_list_orig = ibs.get_annot_name_rowids(self.all_aid_list)
self.other_aids = list(
set(self.all_aid_list) - set([self.aid1, self.aid2]))
if self.is_split_case:
# Split case
self.nCols = max(2, len(self.other_aids))
self.nRows = 2 if len(self.other_aids) > 0 else 1
else:
# Merge/New Match case
self.nCols = max(len(self.gt1) + 1, len(self.gt2) + 1)
self.nRows = 2
self.nCols = min(self.max_cols, self.nCols)
# Grab not just the exemplars
if ut.VERBOSE or ut.is_developer():
logger.info('[matchver] __init__ nid1=%r, nid2=%r ' %
(self.nid1, self.nid2))
logger.info('[matchver] __init__ self.gts_list=%r ' %
(self.gts_list))
if ut.VERBOSE or ut.is_developer():
logger.info('[matchver] __init__ nid1=%r, nid2=%r ' %
(self.nid1, self.nid2))
logger.info('[matchver] __init__ self.gts_list=%r ' %
(self.gts_list))
def get_system_python_library():
"""
FIXME; hacky way of finding python library. Not cross platform yet.
"""
import os
import utool as ut
from os.path import basename, realpath
pyname = basename(realpath(sys.executable))
ld_library_path = os.environ['LD_LIBRARY_PATH']
libdirs = [x
for x in ld_library_path.split(os.pathsep) if x] + ['/usr/lib']
libfiles = ut.flatten(
[ut.glob(d, '*' + ut.get_lib_ext(), recursive=True) for d in libdirs])
python_libs = [
realpath(f) for f in libfiles if 'lib' + pyname in basename(f)
]
python_libs = ut.unique_ordered(python_libs)
assert len(python_libs) == 1, str(python_libs)
return python_libs[0]
def find_module_callers():
"""
TODO:
attempt to build a call graph between module functions to make it easy to see
what can be removed and what cannot.
"""
import utool as ut
from os.path import normpath
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_analyzer.py')
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_all.py')
mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_organizer.py')
module = ut.import_module_from_fpath(mod_fpath)
user_profile = ut.ensure_user_profile()
doctestables = list(ut.iter_module_doctestable(module, include_builtin=False))
grepkw = {}
grepkw['exclude_dirs'] = user_profile.project_exclude_dirs
grepkw['dpath_list'] = user_profile.project_dpaths
grepkw['verbose'] = True
usage_map = {}
for funcname, func in doctestables:
print('Searching for funcname = %r' % (funcname,))
found_fpath_list, found_lines_list, found_lxs_list = ut.grep([funcname], **grepkw)
used_in = (found_fpath_list, found_lines_list, found_lxs_list)
usage_map[funcname] = used_in
external_usage_map = {}
for funcname, used_in in usage_map.items():
(found_fpath_list, found_lines_list, found_lxs_list) = used_in
isexternal_flag = [normpath(fpath) != normpath(mod_fpath) for fpath in found_fpath_list]
ext_used_in = (ut.compress(found_fpath_list, isexternal_flag),
ut.compress(found_lines_list, isexternal_flag),
ut.compress(found_lxs_list, isexternal_flag))
external_usage_map[funcname] = ext_used_in
for funcname, used_in in external_usage_map.items():
(found_fpath_list, found_lines_list, found_lxs_list) = used_in
print('Calling modules: \n' +
ut.repr2(ut.unique_ordered(ut.flatten([used_in[0] for used_in in external_usage_map.values()])), nl=True))
def infer_data(self):
""" Initialize data related to the input aids
"""
ibs = self.ibs
# The two matching aids
self.aid_pair = (self.aid1, self.aid2)
(aid1, aid2) = self.aid_pair
self.match_text = ibs.get_match_text(self.aid1, self.aid2)
# The names of the matching annotations
self.nid1, self.nid2 = ibs.get_annot_name_rowids((aid1, aid2))
self.name1, self.name2 = ibs.get_annot_names((aid1, aid2))
self.other_valid_nids = []
# The other annotations that belong to these two names
self.gts_list = ibs.get_annot_groundtruth((aid1, aid2))
self.gt1, self.gt2 = self.gts_list
# A flat list of all the aids we are looking at
self.is_split_case = self.nid1 == self.nid2
self.all_aid_list = ut.unique_ordered([aid1, aid2] + self.gt1 + self.gt2)
self.all_nid_list_orig = ibs.get_annot_name_rowids(self.all_aid_list)
self.other_aids = list(set(self.all_aid_list) - set([self.aid1, self.aid2]))
if self.is_split_case:
# Split case
self.nCols = max(2, len(self.other_aids))
self.nRows = 2 if len(self.other_aids) > 0 else 1
else:
# Merge/New Match case
self.nCols = max(len(self.gt1) + 1, len(self.gt2) + 1)
self.nRows = 2
self.nCols = min(self.max_cols, self.nCols)
# Grab not just the exemplars
if ut.VERBOSE or ut.is_developer():
print('[matchver] __init__ nid1=%r, nid2=%r ' % (self.nid1, self.nid2))
print('[matchver] __init__ self.gts_list=%r ' % (self.gts_list))
if ut.VERBOSE or ut.is_developer():
print('[matchver] __init__ nid1=%r, nid2=%r ' % (self.nid1, self.nid2))
print('[matchver] __init__ self.gts_list=%r ' % (self.gts_list))
def download_sharks(XMLdata, number):
"""
cd ~/work/WS_ALL
python -m ibeis.scripts.getshark
>>> from ibeis.scripts.getshark import * # NOQA
>>> url = 'www.whaleshark.org/listImages.jsp'
>>> XMLdata = ut.url_read(url)
>>> number = None
"""
# Prepare the output directory for writing, if it doesn't exist
output_dir = 'sharkimages'
ut.ensuredir(output_dir)
dom = parseString(XMLdata)
# Download files
if number:
maxCount = min(number, len(dom.getElementsByTagName('img')))
else:
maxCount = len(dom.getElementsByTagName('img'))
parsed_info = dict(
img_url_list=[],
localid_list=[],
nameid_list=[],
orig_fname_list=[],
new_fname_list=[],
)
print('Preparing to fetch %i files...' % maxCount)
for shark in dom.getElementsByTagName('shark'):
localCount = 0
for imageset in shark.getElementsByTagName('imageset'):
for img in imageset.getElementsByTagName('img'):
localCount += 1
img_url = img.getAttribute('href')
orig_fname = split(img_url)[1]
ext = splitext(orig_fname)[1].lower()
nameid = shark.getAttribute('number')
new_fname = '%s-%i%s' % (
nameid, localCount, ext)
parsed_info['img_url_list'].append(img_url)
parsed_info['nameid_list'].append(nameid)
parsed_info['localid_list'].append(localCount)
parsed_info['orig_fname_list'].append(orig_fname)
parsed_info['new_fname_list'].append(new_fname)
print('Parsed %i / %i files.' % (len(parsed_info['orig_fname_list']), maxCount))
if number is not None and len(parsed_info['orig_fname_list']) == number:
break
parsed_info['new_fpath_list'] = [join(output_dir, _fname)
for _fname in parsed_info['new_fname_list']]
print('Filtering parsed images')
# Filter based on image type (keep only jpgs)
ext_flags = [_fname.endswith('.jpg') or _fname.endswith('.jpg')
for _fname in parsed_info['new_fname_list']]
parsed_info = {key: ut.compress(list_, ext_flags) for key, list_ in parsed_info.items()}
# Filter to only images matching the appropriate tags
from ibeis import tag_funcs
parsed_info['tags_list'] = parse_shark_tags(parsed_info['orig_fname_list'])
tag_flags = tag_funcs.filterflags_general_tags(
parsed_info['tags_list'],
has_any=['view-left'],
none_match=['qual.*', 'view-top', 'part-.*', 'cropped'],
)
parsed_info = {key: ut.compress(list_, tag_flags) for key, list_ in parsed_info.items()}
print('Tags in chosen images:')
print(ut.dict_hist(ut.flatten(parsed_info['tags_list'] )))
# Download selected subset
print('Downloading selected subset')
_iter = list(zip(parsed_info['img_url_list'],
parsed_info['new_fpath_list']))
_iter = ut.ProgressIter(_iter, lbl='downloading sharks')
for img_url, new_fpath in _iter:
if not exists(new_fpath):
ut.download_url(img_url, new_fpath)
# Remove corrupted or ill-formatted images
print('Checking for corrupted images')
import vtool as vt
noncorrupt_flags = vt.filterflags_valid_images(parsed_info['new_fpath_list'])
parsed_info = {
key: ut.compress(list_, noncorrupt_flags)
for key, list_ in parsed_info.items()
}
print('Removing small images')
import numpy as np
imgsize_list = np.array([vt.open_image_size(gpath) for gpath in parsed_info['new_fpath_list']])
sqrt_area_list = np.sqrt(np.prod(imgsize_list, axis=1))
areq_flags_list = sqrt_area_list >= 750
parsed_info = {key: ut.compress(list_, areq_flags_list)
for key, list_ in parsed_info.items()}
grouped_idxs = ut.group_items(list(range(len(parsed_info['nameid_list']))),
parsed_info['nameid_list'])
keep_idxs = sorted(ut.flatten([idxs for key, idxs in grouped_idxs.items() if len(idxs) >= 2]))
parsed_info = {key: ut.take(list_, keep_idxs) for key, list_ in parsed_info.items()}
print('Moving imagse to secondary directory')
named_outputdir = 'named-left-sharkimages'
# Build names
parsed_info['namedir_fpath_list'] = [
join(named_outputdir, _nameid, _fname)
for _fname, _nameid in zip(parsed_info['new_fname_list'],
parsed_info['nameid_list'])]
# Create directories
ut.ensuredir(named_outputdir)
named_dirs = ut.unique_ordered(list(map(dirname, parsed_info['namedir_fpath_list'])))
for dir_ in named_dirs:
ut.ensuredir(dir_)
# Copy
ut.copy_files_to(src_fpath_list=parsed_info['new_fpath_list'],
dst_fpath_list=parsed_info['namedir_fpath_list'])
def show_page(self, bring_to_front=False, onlyrows=None, fulldraw=True):
""" Plots all subaxes on a page
onlyrows is a hack to only draw a subset of the data again
"""
if ut.VERBOSE:
if not fulldraw:
print('[matchver] show_page(fulldraw=%r, onlyrows=%r)' % (fulldraw, onlyrows))
else:
print('[matchver] show_page(fulldraw=%r)' % (fulldraw))
self.prepare_page(fulldraw=fulldraw)
# Variables we will work with to paint a pretty picture
ibs = self.ibs
nRows = self.nRows
colpad = 1 if self.cm is not None else 0
nCols = self.nCols + colpad
# Distinct color for every unique name
unique_nids = ut.unique_ordered(ibs.get_annot_name_rowids(self.all_aid_list, distinguish_unknowns=False))
unique_colors = pt.distinct_colors(len(unique_nids), brightness=.7, hue_range=(.05, .95))
self.nid2_color = dict(zip(unique_nids, unique_colors))
row_aids_list = self.get_row_aids_list()
if self.cm is not None:
print("DRAWING QRES")
pnum = (1, nCols, 1)
if not fulldraw:
# not doing full draw so we have to clear any axes
# that are here already manually
ax = self.fig.add_subplot(*pnum)
self.clear_parent_axes(ax)
self.cm.show_single_annotmatch(self.qreq_, self.aid2, fnum=self.fnum, pnum=pnum, draw_fmatch=True, colorbar_=False)
# For each row
for rowx, aid_list in enumerate(row_aids_list):
offset = rowx * nCols + 1
if onlyrows is not None and rowx not in onlyrows:
continue
#ibsfuncs.assert_valid_aids(ibs, groundtruth)
# For each column
for colx, aid in enumerate(aid_list, start=colpad):
if colx >= nCols:
break
try:
nid = ibs.get_annot_name_rowids(aid)
if ibsfuncs.is_nid_unknown(ibs, [nid])[0]:
color = const.UNKNOWN_PURPLE_RGBA01
else:
color = self.nid2_color[nid]
except Exception as ex:
ut.printex(ex)
print('nid = %r' % (nid,))
print('self.nid2_color = %s' % (ut.dict_str(self.nid2_color),))
raise
px = colx + offset
ax = self.plot_chip(int(aid), nRows, nCols, px, color=color, fulldraw=fulldraw)
# If there are still more in this row to display
if colx + 1 < len(aid_list) and colx + 1 >= nCols:
total_indices = len(aid_list)
current_index = self.col_offset_list[rowx] + 1
next_text = 'next\n%d/%d' % (current_index, total_indices)
next_func = functools.partial(self.rotate_row, rowx=rowx)
self.append_button(next_text, callback=next_func,
location='right', size='33%', ax=ax)
if fulldraw:
self.show_hud()
#pt.adjust_subplots_safe(top=0.85, hspace=0.03)
hspace = .05 if (self.nCols) > 1 else .1
pt.adjust_subplots_safe(top=0.85, hspace=hspace)
self.draw()
self.show()
if bring_to_front:
self.bring_to_front()
def get_test_qaids(ibs, default_qaids=None, return_annot_info=False, aidcfg=None):
"""
Gets test annot_rowids based on command line arguments
DEPRICATE
Args:
ibs (IBEISController): ibeis controller object
default_qaids (None): if list then used only if no other aids are available (default = [1])
as a string it mimics the command line
Returns:
list: available_qaids
CommandLine:
python -m ibeis.init.main_helpers --test-get_test_qaids
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0 --qaid 1
python -m ibeis.init.main_helpers --test-get_test_qaids --allgt --db PZ_MTEST
python -m ibeis.init.main_helpers --test-get_test_qaids --qaid 4 5 8 --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST --qaid 2 --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST --qaid 2
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0 --qindex 0:10 --verbmhelp
python -m ibeis.init.main_helpers --exec-get_test_qaids --controlled --db PZ_Master0 --exec-mode
python -m ibeis.init.main_helpers --exec-get_test_qaids --db testdb1 --allgt --qindex 0:256
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> default_qaids = None
>>> available_qaids = get_test_qaids(ibs, default_qaids)
>>> ibeis.other.dbinfo.get_dbinfo(ibs, aid_list=available_qaids, with_contrib=False, short=True)
>>> result = 'available_qaids = ' + ut.obj_str(available_qaids, truncate=True, nl=False)
>>> print('len(available_qaids) = %d' % len(available_qaids))
>>> print(result)
available_qaids = [1]
"""
qaid_request_info = {}
if VERB_MAIN_HELPERS:
print("[get_test_qaids] + --- GET_TEST_QAIDS ---")
# Old version of this function
if VERB_MAIN_HELPERS:
print("[get_test_qaids] + --- GET_TEST_QAIDS ---")
print("[get_test_qaids] * default_qaids = %s" % (ut.obj_str(default_qaids, truncate=True, nl=False)))
valid_aids = ibs.get_valid_aids()
if len(valid_aids) == 0:
print("[get_test_qaids] WARNING no annotations available")
# ---- INCLUDING STEP
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * include step")
available_qaids = []
# ut.get_argflag(('--all-cases', '--all'))
# ut.get_argflag(('--all-gt-cases', '--allgt'))
# ut.get_argflag(('--all-hard-cases', '--allhard'))
# ut.get_argflag(('--qaid', '--qaids'))
# ut.get_argflag('--controlled') or ut.get_argflag('--controlled_qaids')
# not ut.get_argflag('--junk')
ALL_CASES = params.args.all_cases or default_qaids == "all"
GT_CASES = params.args.all_gt_cases or default_qaids == "gt"
HARD_CASES = params.args.all_hard_cases or ut.get_argflag(("--all-hard-cases", "--allhard", "--hard"))
NO_JUNK = not ut.get_argflag("--junk")
CONTROLLED_CASES = ut.get_argflag("--controlled") or ut.get_argflag("--controlled_qaids")
NO_REVIEWED = ut.get_argflag("--unreviewed")
species = ut.get_argval("--species")
# QAID = params.args.qaid
QAID = ut.get_argval("--qaid", type_="fuzzy_subset", default=None)
QINDEX = params.args.qindex
QSHUFFLE = ut.get_argval("--qshuffle")
if QAID is not None:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Including cmdline specified qaids")
try:
args_qaid = ensure_flatlistlike(QAID)
except Exception:
args_qaid = QAID
available_qaids.extend(args_qaid)
qaid_request_info["custom_commandline"] = args_qaid
if ALL_CASES:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Including all qaids")
available_qaids.extend(valid_aids)
qaid_request_info["all_cases"] = True
if HARD_CASES:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Including hard qaids")
is_hard_list = ibs.get_annot_is_hard(valid_aids)
hard_aids = ut.compress(valid_aids, is_hard_list)
available_qaids.extend(hard_aids)
qaid_request_info["hard_cases"] = True
if GT_CASES:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Including groundtruth qaids")
has_gt_list = ibs.get_annot_has_groundtruth(valid_aids)
hasgt_aids = ut.compress(valid_aids, has_gt_list)
print("[get_test_qaids] Adding all %d/%d ground-truthed test cases" % (len(hasgt_aids), len(valid_aids)))
available_qaids.extend(hasgt_aids)
qaid_request_info["gt_cases"] = True
if CONTROLLED_CASES:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Including controlled qaids")
from ibeis.other import ibsfuncs
# Override all other gts with controlled
controlled_qaids = ibsfuncs.get_two_annots_per_name_and_singletons(ibs, onlygt=True)
available_qaids.extend(controlled_qaids)
qaid_request_info["controlled"] = True
else:
qaid_request_info["controlled"] = False
# ---- CHECK_DEFAULTS QUERY
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * len(available_qaids) = %r" % (len(available_qaids)))
if len(available_qaids) == 0:
print("[get_test_qaids] * ... defaulting, no available qaids on command line.")
if default_qaids is None:
default_qaids = valid_aids[0:1]
qaid_request_info["default_one"] = True
elif isinstance(default_qaids, six.string_types):
if default_qaids == "gt" or default_qaids == "allgt":
default_qaids = ibs.get_valid_aids(hasgt=True)
qaid_request_info["default_gt"] = True
available_qaids = default_qaids
else:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * ... not defaulting")
available_qaids = ut.unique_ordered(available_qaids)
# ---- EXCLUSION STEP
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * len(available_qaids) = %r" % (len(available_qaids)))
print("[get_test_qaids] * exclude step")
if NO_JUNK:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Filtering junk")
available_qaids = ibs.filter_junk_annotations(available_qaids)
qaid_request_info["has_junk"] = False
if NO_REVIEWED:
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Filtering unreviewed")
isreviewed_list = ibs.get_annot_has_reviewed_matching_aids(available_qaids)
available_qaids = ut.filterfalse_items(available_qaids, isreviewed_list)
qaid_request_info["has_unreviewed"] = False
if species is not None:
if species == "primary":
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Finiding primary species")
# species = ibs.get_primary_database_species(available_qaids)
species = ibs.get_primary_database_species()
qaid_request_info["primary_species"] = True
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Filtering to species=%r" % (species,))
isvalid_list = np.array(ibs.get_annot_species(available_qaids)) == species
available_qaids = ut.compress(available_qaids, isvalid_list)
qaid_request_info["species_filter"] = species
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * len(available_qaids) = %r" % (len(available_qaids)))
print("[get_test_qaids] * subindex step")
# ---- INDEX SUBSET
# ut.get_argval('--qshuffle')
if QSHUFFLE:
# Determenistic shuffling
available_qaids = ut.take(available_qaids, ut.random_indexes(len(available_qaids), seed=42))
qaid_request_info["shuffled"] = True
# Sample a large pool of chosen query qindexes
if QINDEX is not None:
# FIXME: should use a slice of the list or a sublist
qindexes = ensure_flatlistlike(QINDEX)
_test_qaids = [available_qaids[qx] for qx in qindexes if qx < len(available_qaids)]
print("[get_test_qaids] Chose subset of size %d/%d" % (len(_test_qaids), len(available_qaids)))
available_qaids = _test_qaids
qaid_request_info["subset"] = qindexes
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * len(available_qaids) = %r" % (len(available_qaids)))
print("[get_test_qaids] L ___ GET_TEST_QAIDS ___")
if return_annot_info:
return available_qaids, qaid_request_info
else:
return available_qaids
def get_test_daids(ibs, default_daids='all', qaid_list=None, return_annot_info=False, aidcfg=None):
""" Gets database annot_rowids based on command line arguments
DEPRICATE
CommandLine:
python dev.py --db PZ_MTEST -t best --exclude-query --qaid 72 -r 0 -c 0 --show --va --vf --dump-extra
Args:
ibs (IBEISController): ibeis controller object
default_daids (str): (default = 'all')
qaid_list (list): list of chosen qaids that may affect daids (default = None)
Returns:
list: available_daids
CommandLine:
python -m ibeis.init.main_helpers --test-get_test_daids
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --exclude-query
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --daid-exclude 2 3 4
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --species=zebra_grevys
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_Master0 --species=zebra_grevys
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_Master0 --controlled --verbmhelp
python -m ibeis.init.main_helpers --exec-get_test_daids --controlled --db PZ_Master0 --exec-mode
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> default_daids = 'all'
>>> qaid_list = [1]
>>> available_daids = get_test_daids(ibs, default_daids, qaid_list)
>>> ibeis.other.dbinfo.get_dbinfo(ibs, aid_list=available_daids, with_contrib=False, short=True)
>>> result = 'available_daids = ' + ut.obj_str(available_daids, truncate=True, nl=False)
>>> print('len(available_daids) %d' % len(available_daids))
>>> print(result)
available_daids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
"""
daid_request_info = {}
if VERB_MAIN_HELPERS:
print('[get_test_daids] + --- GET_TEST_DAIDS ---')
print('[get_test_daids] * default_daids = %s' % (ut.obj_str(default_daids, truncate=True, nl=False)))
print('[get_test_daids] * qaid_list = %s' % (ut.obj_str(qaid_list, truncate=True, nl=False)))
# ---- INCLUDING STEP
if VERB_MAIN_HELPERS:
print('[get_test_daids] * include step')
available_daids = []
CONTROLLED_CASES = ut.get_argflag('--controlled') or ut.get_argflag('--controlled_daids')
DSHUFFLE = ut.get_argval('--dshuffle')
DINDEX = params.args.dindex
NO_JUNK = not ut.get_argflag('--junk')
EXCLUDE_QUERY = ut.get_argflag('--exclude-query')
#daids_exclude = params.args.daid_exclude
daids_exclude = None
if CONTROLLED_CASES:
print('[get_test_daids] * Including controlled daids')
from ibeis.other import ibsfuncs
controlled_daids = ibsfuncs.get_two_annots_per_name_and_singletons(ibs, onlygt=False)
available_daids.extend(controlled_daids)
daid_request_info['controlled'] = True
else:
daid_request_info['controlled'] = False
# ---- CHECK_DEFAULTS DATA
if VERB_MAIN_HELPERS:
print('[get_test_daids] * len(available_daids) = %r' % (len(available_daids)))
if len(available_daids) == 0:
print('[get_test_daids] * ... defaulting, no available daids on command line.')
if isinstance(default_daids, six.string_types):
if default_daids == 'all':
default_daids = ibs.get_valid_aids()
daid_request_info['default_daids'] = 'all'
elif default_daids == 'gt':
default_daids = ut.flatten(ibs.get_annot_groundtruth(qaid_list))
daid_request_info['default_daids'] = 'gt'
#available_qaids = valid_aids[0:1]
assert not isinstance(available_daids, six.string_types)
available_daids = default_daids
else:
if VERB_MAIN_HELPERS:
print('[get_test_daids] * ... not defaulting')
available_daids = ut.unique_ordered(available_daids)
# ---- EXCLUSION STEP
if VERB_MAIN_HELPERS:
print('[get_test_daids] * len(available_daids) = %r' % (len(available_daids)))
print('[get_test_daids] * exclude step')
species = ut.get_argval('--species', type_=str, default=None)
if NO_JUNK:
if VERB_MAIN_HELPERS:
print('[get_test_daids] * Filtering junk')
available_daids = ibs.filter_junk_annotations(available_daids)
if EXCLUDE_QUERY:
if VERB_MAIN_HELPERS:
print('[get_test_daids] * Excluding query qaids')
assert qaid_list is not None, 'must specify qaids to exclude'
available_daids = ut.setdiff_ordered(available_daids, qaid_list)
if daids_exclude is not None:
if VERB_MAIN_HELPERS:
print('[get_test_daids] * Excluding specified daids')
available_daids = ut.setdiff_ordered(available_daids, daids_exclude)
if species is not None:
if species == 'primary':
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Finiding primary species')
#species = ibs.get_primary_database_species(available_daids)
species = ibs.get_primary_database_species()
if VERB_MAIN_HELPERS:
print('[get_test_daids] * Filtering to species=%r' % (species,))
import numpy as np
isvalid_list = np.array(ibs.get_annot_species(available_daids)) == species
available_daids = ut.compress(available_daids, isvalid_list)
# ---- SUBINDEXING STEP
if VERB_MAIN_HELPERS:
print('[get_test_daids] * len(available_daids) = %r' % (len(available_daids)))
print('[get_test_daids] * subindex step')
#ut.get_argval('--qshuffle')
if DSHUFFLE:
# Determenistic shuffling
available_daids = ut.take(available_daids, ut.random_indexes(len(available_daids), seed=43))
daid_request_info['shuffled'] = True
if DINDEX is not None:
dindexes = ensure_flatlistlike(DINDEX)
_test_daids = [available_daids[dx] for dx in dindexes if dx < len(available_daids)]
print('[get_test_daids] Chose subset of size %d/%d' % (len(_test_daids), len(available_daids)))
available_daids = _test_daids
if VERB_MAIN_HELPERS:
print('[get_test_daids] * len(available_daids) = %r' % (len(available_daids)))
print('[get_test_daids] L ___ GET_TEST_DAIDS ___')
if return_annot_info:
return available_daids, daid_request_info
else:
return available_daids
def query_vsone_pairs(ibs, vsone_query_pairs, use_cache=False, save_qcache=False):
"""
does vsone queries to rerank the top few vsmany querys
Returns:
tuple: qaid2_qres_vsone, qreq_vsone_
CommandLine:
python -m ibeis.algo.hots.special_query --test-query_vsone_pairs
Example:
>>> # SLOW_DOCTEST
>>> from ibeis.algo.hots.special_query import * # NOQA
>>> ibs, valid_aids = testdata_special_query()
>>> qaids = valid_aids[0:1]
>>> daids = valid_aids[1:]
>>> qaid = qaids[0]
>>> filtkey = hstypes.FiltKeys.DISTINCTIVENESS
>>> use_cache = False
>>> save_qcache = False
>>> # execute function
>>> qaid2_qres_vsmany, qreq_vsmany_ = query_vsmany_initial(ibs, qaids, daids)
>>> vsone_query_pairs = build_vsone_shortlist(ibs, qaid2_qres_vsmany)
>>> qaid2_qres_vsone, qreq_vsone_ = query_vsone_pairs(ibs, vsone_query_pairs)
>>> qres_vsone = qaid2_qres_vsone[qaid]
>>> top_namescore_aids = qres_vsone.get_top_aids().tolist()
>>> result = str(top_namescore_aids)
>>> top_namescore_names = ibs.get_annot_names(top_namescore_aids)
>>> assert top_namescore_names[0] == 'easy', 'top_namescore_names[0]=%r' % (top_namescore_names[0],)
"""
#vsone_cfgdict = dict(codename='vsone_unnorm')
#codename = 'vsone_unnorm_dist_ratio_extern_distinctiveness',
codename = 'vsone_unnorm_dist_ratio'
vsone_cfgdict = dict(
index_method='single',
codename=codename,
)
#------------------------
qaid2_qres_vsone = {}
for qaid, top_aids in vsone_query_pairs:
# Perform a query request for each
cm_list_vsone_, __qreq_vsone_ = ibs.query_chips(
[qaid], top_aids, cfgdict=vsone_cfgdict, return_request=True,
use_cache=use_cache, save_qcache=save_qcache)
qaid2_qres_vsone_ = {cm.qaid: cm for cm in cm_list_vsone_}
qaid2_qres_vsone.update(qaid2_qres_vsone_)
#------------------------
# Create pseudo query request because there is no good way to
# represent the vsone reranking as a single query request and
# we need one for the score normalizer
#pseudo_codename_ = codename.replace('unnorm', 'norm') + '_extern_distinctiveness'
pseudo_codename_ = codename.replace('unnorm', 'norm') # + '_extern_distinctiveness'
pseudo_vsone_cfgdict = dict(codename=pseudo_codename_)
pseudo_qaids = ut.get_list_column(vsone_query_pairs, 0)
pseudo_daids = ut.unique_ordered(ut.flatten(ut.get_list_column(vsone_query_pairs, 1)))
# FIXME: making the pseudo qreq_ takes a nontrivial amount of time for what
# should be a trivial task.
pseudo_qreq_vsone_ = ibs.new_query_request(pseudo_qaids, pseudo_daids,
cfgdict=pseudo_vsone_cfgdict,
verbose=ut.VERBOSE)
#pseudo_qreq_vsone_.load_distinctiveness_normalizer()
qreq_vsone_ = pseudo_qreq_vsone_
# Hack in a special config name
qreq_vsone_.qparams.query_cfgstr = '_special' + qreq_vsone_.qparams.query_cfgstr
return qaid2_qres_vsone, qreq_vsone_
def get_test_qaids(ibs, default_qaids=None, return_annot_info=False, aidcfg=None):
"""
Gets test annot_rowids based on command line arguments
DEPRICATE
Args:
ibs (IBEISController): ibeis controller object
default_qaids (None): if list then used only if no other aids are available (default = [1])
as a string it mimics the command line
Returns:
list: available_qaids
CommandLine:
python -m ibeis.init.main_helpers --test-get_test_qaids
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0 --qaid 1
python -m ibeis.init.main_helpers --test-get_test_qaids --allgt --db PZ_MTEST
python -m ibeis.init.main_helpers --test-get_test_qaids --qaid 4 5 8 --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST --qaid 2 --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_MTEST --qaid 2
python -m ibeis.init.main_helpers --test-get_test_qaids --controlled --db PZ_Master0 --qindex 0:10 --verbmhelp
python -m ibeis.init.main_helpers --exec-get_test_qaids --controlled --db PZ_Master0 --exec-mode
python -m ibeis.init.main_helpers --exec-get_test_qaids --db testdb1 --allgt --qindex 0:256
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> default_qaids = None
>>> available_qaids = get_test_qaids(ibs, default_qaids)
>>> ibeis.other.dbinfo.get_dbinfo(ibs, aid_list=available_qaids, with_contrib=False, short=True)
>>> result = 'available_qaids = ' + ut.obj_str(available_qaids, truncate=True, nl=False)
>>> print('len(available_qaids) = %d' % len(available_qaids))
>>> print(result)
available_qaids = [1]
"""
qaid_request_info = {}
if VERB_MAIN_HELPERS:
print('[get_test_qaids] + --- GET_TEST_QAIDS ---')
# Old version of this function
if VERB_MAIN_HELPERS:
print('[get_test_qaids] + --- GET_TEST_QAIDS ---')
print('[get_test_qaids] * default_qaids = %s' % (ut.obj_str(default_qaids, truncate=True, nl=False)))
valid_aids = ibs.get_valid_aids()
if len(valid_aids) == 0:
print('[get_test_qaids] WARNING no annotations available')
# ---- INCLUDING STEP
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * include step')
available_qaids = []
#ut.get_argflag(('--all-cases', '--all'))
#ut.get_argflag(('--all-gt-cases', '--allgt'))
#ut.get_argflag(('--all-hard-cases', '--allhard'))
#ut.get_argflag(('--qaid', '--qaids'))
#ut.get_argflag('--controlled') or ut.get_argflag('--controlled_qaids')
#not ut.get_argflag('--junk')
ALL_CASES = params.args.all_cases or default_qaids == 'all'
GT_CASES = params.args.all_gt_cases or default_qaids == 'gt'
HARD_CASES = params.args.all_hard_cases or ut.get_argflag(('--all-hard-cases', '--allhard', '--hard'))
NO_JUNK = not ut.get_argflag('--junk')
CONTROLLED_CASES = ut.get_argflag('--controlled') or ut.get_argflag('--controlled_qaids')
NO_REVIEWED = ut.get_argflag('--unreviewed')
species = ut.get_argval('--species')
#QAID = params.args.qaid
QAID = ut.get_argval('--qaid', type_='fuzzy_subset', default=None)
QINDEX = params.args.qindex
QSHUFFLE = ut.get_argval('--qshuffle')
if QAID is not None:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Including cmdline specified qaids')
try:
args_qaid = ensure_flatlistlike(QAID)
except Exception:
args_qaid = QAID
available_qaids.extend(args_qaid)
qaid_request_info['custom_commandline'] = args_qaid
if ALL_CASES:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Including all qaids')
available_qaids.extend(valid_aids)
qaid_request_info['all_cases'] = True
if HARD_CASES:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Including hard qaids')
is_hard_list = ibs.get_annot_is_hard(valid_aids)
hard_aids = ut.compress(valid_aids, is_hard_list)
available_qaids.extend(hard_aids)
qaid_request_info['hard_cases'] = True
if GT_CASES:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Including groundtruth qaids')
has_gt_list = ibs.get_annot_has_groundtruth(valid_aids)
hasgt_aids = ut.compress(valid_aids, has_gt_list)
print('[get_test_qaids] Adding all %d/%d ground-truthed test cases' % (len(hasgt_aids), len(valid_aids)))
available_qaids.extend(hasgt_aids)
qaid_request_info['gt_cases'] = True
if CONTROLLED_CASES:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Including controlled qaids')
from ibeis.other import ibsfuncs
# Override all other gts with controlled
controlled_qaids = ibsfuncs.get_two_annots_per_name_and_singletons(ibs, onlygt=True)
available_qaids.extend(controlled_qaids)
qaid_request_info['controlled'] = True
else:
qaid_request_info['controlled'] = False
# ---- CHECK_DEFAULTS QUERY
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * len(available_qaids) = %r' % (len(available_qaids)))
if len(available_qaids) == 0:
print('[get_test_qaids] * ... defaulting, no available qaids on command line.')
if default_qaids is None:
default_qaids = valid_aids[0:1]
qaid_request_info['default_one'] = True
elif isinstance(default_qaids, six.string_types):
if default_qaids == 'gt' or default_qaids == 'allgt':
default_qaids = ibs.get_valid_aids(hasgt=True)
qaid_request_info['default_gt'] = True
available_qaids = default_qaids
else:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * ... not defaulting')
available_qaids = ut.unique_ordered(available_qaids)
# ---- EXCLUSION STEP
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * len(available_qaids) = %r' % (len(available_qaids)))
print('[get_test_qaids] * exclude step')
if NO_JUNK:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Filtering junk')
available_qaids = ibs.filter_junk_annotations(available_qaids)
qaid_request_info['has_junk'] = False
if NO_REVIEWED:
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Filtering unreviewed')
isreviewed_list = ibs.get_annot_has_reviewed_matching_aids(available_qaids)
available_qaids = ut.filterfalse_items(available_qaids, isreviewed_list)
qaid_request_info['has_unreviewed'] = False
if species is not None:
if species == 'primary':
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Finiding primary species')
#species = ibs.get_primary_database_species(available_qaids)
species = ibs.get_primary_database_species()
qaid_request_info['primary_species'] = True
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * Filtering to species=%r' % (species,))
isvalid_list = np.array(ibs.get_annot_species(available_qaids)) == species
available_qaids = ut.compress(available_qaids, isvalid_list)
qaid_request_info['species_filter'] = species
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * len(available_qaids) = %r' % (len(available_qaids)))
print('[get_test_qaids] * subindex step')
# ---- INDEX SUBSET
#ut.get_argval('--qshuffle')
if QSHUFFLE:
# Determenistic shuffling
available_qaids = ut.take(available_qaids, ut.random_indexes(len(available_qaids), seed=42))
qaid_request_info['shuffled'] = True
# Sample a large pool of chosen query qindexes
if QINDEX is not None:
# FIXME: should use a slice of the list or a sublist
qindexes = ensure_flatlistlike(QINDEX)
_test_qaids = [available_qaids[qx] for qx in qindexes if qx < len(available_qaids)]
print('[get_test_qaids] Chose subset of size %d/%d' % (len(_test_qaids), len(available_qaids)))
available_qaids = _test_qaids
qaid_request_info['subset'] = qindexes
if VERB_MAIN_HELPERS:
print('[get_test_qaids] * len(available_qaids) = %r' % (len(available_qaids)))
print('[get_test_qaids] L ___ GET_TEST_QAIDS ___')
if return_annot_info:
return available_qaids, qaid_request_info
else:
return available_qaids
def mana_potential2(card, deck=None, recurse=True):
r"""Returns a list of mana sets or mana producers
CommandLine:
python -m mtgmonte.mtgobjs --exec-mana_potential2:1
Example:
>>> # ENABLE_DOCTEST
>>> from mtgmonte.mtgobjs import * # NOQA
>>> from mtgmonte import mtgobjs
>>> deck = mtgobjs.Deck(mtgobjs.load_cards(['Tropical Island', 'Sunken Hollow', 'Island']))
>>> cards = mtgobjs.load_cards(['Tundra', 'Ancient Tomb', 'Black Lotus'])
>>> card = cards[-1]
>>> result = ut.repr2([card.mana_potential2(deck) for card in cards])
>>> print(str(result))
Example:
>>> # ENABLE_DOCTEST
>>> from mtgmonte.mtgobjs import * # NOQA
>>> from mtgmonte import mtgobjs
>>> deck = mtgobjs.Deck(mtgobjs.load_cards(['Tropical Island', 'Sunken Hollow', 'Island']))
>>> cards = mtgobjs.load_cards(['Flooded Strand', 'Tundra', 'Island', 'Shivan Reef', 'Ancient Tomb'])
>>> card = cards[-1]
>>> result = ut.repr2([card.mana_potential2(deck, recurse=True)
>>> for card in cards], nl=1, strvals=1, nobr=1)
>>> print(result)
[{G}, {U}, {B}],
[{W}, {U}],
[{U}],
[{C}, {U}, {R}],
[{CC}],
Example:
>>> # ENABLE_DOCTEST
>>> from mtgmonte.mtgobjs import * # NOQA
>>> from mtgmonte import mtgobjs
>>> deck = mtgobjs.Deck(mtgobjs.load_cards(['Tropical Island', 'Sunken Hollow', 'Island']))
>>> cards = mtgobjs.load_cards(['Flooded Strand', 'Tundra', 'Island', 'Shivan Reef', 'Ancient Tomb'])
>>> card = cards[-1]
>>> result = ut.repr2([card.mana_potential2(deck, recurse=False)
>>> for card in cards], nl=1, strvals=True, nobr=1)
>>> print(result)
[Tropical Island, Sunken Hollow, Island],
[{W}, {U}],
[{U}],
[{C}, {U}, {R}],
[{CC}],
"""
from mtgmonte import mtgrules
potential = ManaOption()
#ManaOption()
for block in card.ability_blocks:
mana_generated = mtgrules.mana_generated(block, card)
if mana_generated is not None:
potential.extend(mana_generated)
else:
if mtgrules.is_fetchland(block, card):
fetch_targets = mtgrules.get_fetch_search_targets(block, card, deck)
if recurse:
mana_generated = [t.mana_potential2(deck) for t in fetch_targets]
mana_generated = ut.flatten(mana_generated)
mana_generated = ut.unique_ordered(mana_generated)
potential.extend(ManaOption(mana_generated))
else:
potential.extend(fetch_targets)
return potential
def get_test_daids(ibs, default_daids="all", qaid_list=None, return_annot_info=False, aidcfg=None):
""" Gets database annot_rowids based on command line arguments
DEPRICATE
CommandLine:
python dev.py --db PZ_MTEST -t best --exclude-query --qaid 72 -r 0 -c 0 --show --va --vf --dump-extra
Args:
ibs (IBEISController): ibeis controller object
default_daids (str): (default = 'all')
qaid_list (list): list of chosen qaids that may affect daids (default = None)
Returns:
list: available_daids
CommandLine:
python -m ibeis.init.main_helpers --test-get_test_daids
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --verbmhelp
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --exclude-query
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --daid-exclude 2 3 4
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_MTEST --species=zebra_grevys
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_Master0 --species=zebra_grevys
python -m ibeis.init.main_helpers --test-get_test_daids --db PZ_Master0 --controlled --verbmhelp
python -m ibeis.init.main_helpers --exec-get_test_daids --controlled --db PZ_Master0 --exec-mode
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.init.main_helpers import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='testdb1')
>>> default_daids = 'all'
>>> qaid_list = [1]
>>> available_daids = get_test_daids(ibs, default_daids, qaid_list)
>>> ibeis.other.dbinfo.get_dbinfo(ibs, aid_list=available_daids, with_contrib=False, short=True)
>>> result = 'available_daids = ' + ut.obj_str(available_daids, truncate=True, nl=False)
>>> print('len(available_daids) %d' % len(available_daids))
>>> print(result)
available_daids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
"""
daid_request_info = {}
if VERB_MAIN_HELPERS:
print("[get_test_daids] + --- GET_TEST_DAIDS ---")
print("[get_test_daids] * default_daids = %s" % (ut.obj_str(default_daids, truncate=True, nl=False)))
print("[get_test_daids] * qaid_list = %s" % (ut.obj_str(qaid_list, truncate=True, nl=False)))
# ---- INCLUDING STEP
if VERB_MAIN_HELPERS:
print("[get_test_daids] * include step")
available_daids = []
CONTROLLED_CASES = ut.get_argflag("--controlled") or ut.get_argflag("--controlled_daids")
DSHUFFLE = ut.get_argval("--dshuffle")
DINDEX = params.args.dindex
NO_JUNK = not ut.get_argflag("--junk")
EXCLUDE_QUERY = ut.get_argflag("--exclude-query")
# daids_exclude = params.args.daid_exclude
daids_exclude = None
if CONTROLLED_CASES:
print("[get_test_daids] * Including controlled daids")
from ibeis.other import ibsfuncs
controlled_daids = ibsfuncs.get_two_annots_per_name_and_singletons(ibs, onlygt=False)
available_daids.extend(controlled_daids)
daid_request_info["controlled"] = True
else:
daid_request_info["controlled"] = False
# ---- CHECK_DEFAULTS DATA
if VERB_MAIN_HELPERS:
print("[get_test_daids] * len(available_daids) = %r" % (len(available_daids)))
if len(available_daids) == 0:
print("[get_test_daids] * ... defaulting, no available daids on command line.")
if isinstance(default_daids, six.string_types):
if default_daids == "all":
default_daids = ibs.get_valid_aids()
daid_request_info["default_daids"] = "all"
elif default_daids == "gt":
default_daids = ut.flatten(ibs.get_annot_groundtruth(qaid_list))
daid_request_info["default_daids"] = "gt"
# available_qaids = valid_aids[0:1]
assert not isinstance(available_daids, six.string_types)
available_daids = default_daids
else:
if VERB_MAIN_HELPERS:
print("[get_test_daids] * ... not defaulting")
available_daids = ut.unique_ordered(available_daids)
# ---- EXCLUSION STEP
if VERB_MAIN_HELPERS:
print("[get_test_daids] * len(available_daids) = %r" % (len(available_daids)))
print("[get_test_daids] * exclude step")
species = ut.get_argval("--species", type_=str, default=None)
if NO_JUNK:
if VERB_MAIN_HELPERS:
print("[get_test_daids] * Filtering junk")
available_daids = ibs.filter_junk_annotations(available_daids)
if EXCLUDE_QUERY:
if VERB_MAIN_HELPERS:
print("[get_test_daids] * Excluding query qaids")
assert qaid_list is not None, "must specify qaids to exclude"
available_daids = ut.setdiff_ordered(available_daids, qaid_list)
if daids_exclude is not None:
if VERB_MAIN_HELPERS:
print("[get_test_daids] * Excluding specified daids")
available_daids = ut.setdiff_ordered(available_daids, daids_exclude)
if species is not None:
if species == "primary":
if VERB_MAIN_HELPERS:
print("[get_test_qaids] * Finiding primary species")
# species = ibs.get_primary_database_species(available_daids)
species = ibs.get_primary_database_species()
if VERB_MAIN_HELPERS:
print("[get_test_daids] * Filtering to species=%r" % (species,))
import numpy as np
isvalid_list = np.array(ibs.get_annot_species(available_daids)) == species
available_daids = ut.compress(available_daids, isvalid_list)
# ---- SUBINDEXING STEP
if VERB_MAIN_HELPERS:
print("[get_test_daids] * len(available_daids) = %r" % (len(available_daids)))
print("[get_test_daids] * subindex step")
# ut.get_argval('--qshuffle')
if DSHUFFLE:
# Determenistic shuffling
available_daids = ut.take(available_daids, ut.random_indexes(len(available_daids), seed=43))
daid_request_info["shuffled"] = True
if DINDEX is not None:
dindexes = ensure_flatlistlike(DINDEX)
_test_daids = [available_daids[dx] for dx in dindexes if dx < len(available_daids)]
print("[get_test_daids] Chose subset of size %d/%d" % (len(_test_daids), len(available_daids)))
available_daids = _test_daids
if VERB_MAIN_HELPERS:
print("[get_test_daids] * len(available_daids) = %r" % (len(available_daids)))
print("[get_test_daids] L ___ GET_TEST_DAIDS ___")
if return_annot_info:
return available_daids, daid_request_info
else:
return available_daids
def show_name_matches(ibs, qaid, name_daid_list, name_fm_list, name_fs_list,
name_H1_list, name_featflag_list, qreq_=None, **kwargs):
"""
Called from chip_match.py
Args:
ibs (IBEISController): ibeis controller object
qaid (int): query annotation id
name_daid_list (list):
name_fm_list (list):
name_fs_list (list):
name_H1_list (list):
name_featflag_list (list):
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
show_nid, show_aid, show_annot_score, show_truth, name_score,
show_name_score, show_name_rank, show_timedelta
CommandLine:
python -m ibeis.viz.viz_matches --exec-show_name_matches
python -m ibeis.viz.viz_matches --test-show_name_matches --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_matches import * # NOQA
>>> from ibeis.algo.hots import chip_match
>>> from ibeis.algo.hots import name_scoring
>>> import vtool as vt
>>> from ibeis.algo.hots import _pipeline_helpers as plh # NOQA
>>> import numpy as np
>>> func = chip_match.ChipMatch.show_single_namematch
>>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
>>> strip_docstr=True)
>>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
>>> homog = False
>>> print(ut.indent(setup, '>>> '))
>>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
>>> cm = cm_list[0]
>>> cm.score_nsum(qreq_)
>>> dnid = ibs.get_annot_nids(cm.qaid)
>>> # +--- COPIED SECTION
>>> locals_ = locals()
>>> var_list = ut.exec_func_src(
>>> func, locals_=locals_,
>>> sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
>>> exec(ut.execstr_dict(var_list))
>>> # L___ COPIED SECTION
>>> kwargs = {}
>>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
>>> name_fs_list, name_h1_list, name_featflag_list,
>>> qreq_=qreq_, **kwargs)
>>> ut.quit_if_noshow()
>>> ut.show_if_requested()
"""
#print("SHOW NAME MATCHES")
#print(ut.repr2(kwargs, nl=True))
#from ibeis import constants as const
from ibeis import tag_funcs
draw_fmatches = kwargs.pop('draw_fmatches', True)
rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
qreq_, draw_fmatches)
fm_list = name_fm_list
fs_list = name_fs_list
featflag_list = name_featflag_list
offset_list, sf_list, bbox_list = show_multichip_match(rchip1, rchip2_list,
kpts1, kpts2_list,
fm_list, fs_list,
featflag_list,
**kwargs)
aid_list = [qaid] + name_daid_list
annotate_matches3(ibs, aid_list, bbox_list, offset_list, name_fm_list,
name_fs_list, qreq_=None, **kwargs)
ax = pt.gca()
title = vh.get_query_text(ibs, None, name_daid_list, False, qaid=qaid,
**kwargs)
pt.set_title(title, ax)
# Case tags
annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
[qaid] * len(name_daid_list), name_daid_list)
annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
if not ut.get_argflag('--show'): # False:
tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
tag_list = ut.unique_ordered(ut.flatten(tags_list))
name_rank = kwargs.get('name_rank', None)
truth = get_multitruth(ibs, aid_list)
xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]
if False:
if name_rank is None:
xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
#xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
else:
if name_rank == 0:
xlabel = {
1: 'True Positive', 0: 'False Positive', 2: 'Unknown'}[truth]
else:
xlabel = {
1: 'False Negative', 0: 'True Negative', 2: 'Unknown'}[truth]
if len(tag_list) > 0:
xlabel += '\n' + ', '.join(tag_list)
pt.set_xlabel(xlabel)
return ax
def show_page(self, bring_to_front=False, onlyrows=None, fulldraw=True):
""" Plots all subaxes on a page
onlyrows is a hack to only draw a subset of the data again
"""
if ut.VERBOSE:
if not fulldraw:
print('[matchver] show_page(fulldraw=%r, onlyrows=%r)' %
(fulldraw, onlyrows))
else:
print('[matchver] show_page(fulldraw=%r)' % (fulldraw))
self.prepare_page(fulldraw=fulldraw)
# Variables we will work with to paint a pretty picture
ibs = self.ibs
nRows = self.nRows
colpad = 1 if self.cm is not None else 0
nCols = self.nCols + colpad
# Distinct color for every unique name
unique_nids = ut.unique_ordered(
ibs.get_annot_name_rowids(self.all_aid_list,
distinguish_unknowns=False))
unique_colors = pt.distinct_colors(len(unique_nids),
brightness=.7,
hue_range=(.05, .95))
self.nid2_color = dict(zip(unique_nids, unique_colors))
row_aids_list = self.get_row_aids_list()
if self.cm is not None:
print("DRAWING QRES")
pnum = (1, nCols, 1)
if not fulldraw:
# not doing full draw so we have to clear any axes
# that are here already manually
ax = self.fig.add_subplot(*pnum)
self.clear_parent_axes(ax)
self.cm.show_single_annotmatch(self.qreq_,
self.aid2,
fnum=self.fnum,
pnum=pnum,
draw_fmatch=True,
colorbar_=False)
# For each row
for rowx, aid_list in enumerate(row_aids_list):
offset = rowx * nCols + 1
if onlyrows is not None and rowx not in onlyrows:
continue
#ibsfuncs.assert_valid_aids(ibs, groundtruth)
# For each column
for colx, aid in enumerate(aid_list, start=colpad):
if colx >= nCols:
break
try:
nid = ibs.get_annot_name_rowids(aid)
if ibsfuncs.is_nid_unknown(ibs, [nid])[0]:
color = const.UNKNOWN_PURPLE_RGBA01
else:
color = self.nid2_color[nid]
except Exception as ex:
ut.printex(ex)
print('nid = %r' % (nid, ))
print('self.nid2_color = %s' %
(ut.repr2(self.nid2_color), ))
raise
px = colx + offset
ax = self.plot_chip(int(aid),
nRows,
nCols,
px,
color=color,
fulldraw=fulldraw)
# If there are still more in this row to display
if colx + 1 < len(aid_list) and colx + 1 >= nCols:
total_indices = len(aid_list)
current_index = self.col_offset_list[rowx] + 1
next_text = 'next\n%d/%d' % (current_index, total_indices)
next_func = functools.partial(self.rotate_row, rowx=rowx)
self.append_button(next_text,
callback=next_func,
location='right',
size='33%',
ax=ax)
if fulldraw:
self.show_hud()
hspace = .05 if (self.nCols) > 1 else .1
subplotspar = {
'left': .1,
'right': .9,
'top': .85,
'bottom': .1,
'wspace': .3,
'hspace': hspace,
}
pt.adjust_subplots(**subplotspar)
self.draw()
self.show()
if bring_to_front:
self.bring_to_front()
def latex_dbstats(ibs_list, **kwargs):
r"""
Args:
ibs (IBEISController): ibeis controller object
CommandLine:
python -m ibeis.other.dbinfo --exec-latex_dbstats --dblist testdb1
python -m ibeis.other.dbinfo --exec-latex_dbstats --dblist testdb1 --show
python -m ibeis.other.dbinfo --exec-latex_dbstats --dblist PZ_Master0 testdb1 --show
python -m ibeis.other.dbinfo --exec-latex_dbstats --dblist PZ_Master0 PZ_MTEST GZ_ALL --show
python -m ibeis.other.dbinfo --test-latex_dbstats --dblist GZ_ALL NNP_MasterGIRM_core --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> db_list = ut.get_argval('--dblist', type_=list, default=['testdb1'])
>>> ibs_list = [ibeis.opendb(db=db) for db in db_list]
>>> tabular_str = latex_dbstats(ibs_list)
>>> tabular_cmd = ut.latex_newcommand(ut.latex_sanitize_command_name('DatabaseInfo'), tabular_str)
>>> ut.copy_text_to_clipboard(tabular_cmd)
>>> write_fpath = ut.get_argval('--write', type_=str, default=None)
>>> if write_fpath is not None:
>>> fpath = ut.truepath(write_fpath)
>>> text = ut.readfrom(fpath)
>>> new_text = ut.replace_between_tags(text, tabular_cmd, '% <DBINFO>', '% </DBINFO>')
>>> ut.writeto(fpath, new_text)
>>> ut.print_code(tabular_cmd, 'latex')
>>> ut.quit_if_noshow()
>>> ut.render_latex_text('\\noindent \n' + tabular_str)
"""
import ibeis
# Parse for aids test data
aids_list = [ibeis.testdata_aids(ibs=ibs) for ibs in ibs_list]
#dbinfo_list = [get_dbinfo(ibs, with_contrib=False, verbose=False) for ibs in ibs_list]
dbinfo_list = [get_dbinfo(ibs, with_contrib=False, verbose=False, aid_list=aids)
for ibs, aids in zip(ibs_list, aids_list)]
#title = db_name + ' database statistics'
title = 'Database statistics'
stat_title = '# Annotations per name (multiton)'
#col_lbls = [
# 'multiton',
# #'singleton',
# 'total',
# 'multiton',
# 'singleton',
# 'total',
#]
key_to_col_lbls = {
'num_names_multiton': 'multiton',
'num_names_singleton': 'singleton',
'num_names': 'total',
'num_multiton_annots': 'multiton',
'num_singleton_annots': 'singleton',
'num_unknown_annots': 'unknown',
'num_annots': 'total',
}
# Structure of columns / multicolumns
multi_col_keys = [
('# Names', (
'num_names_multiton',
#'num_names_singleton',
'num_names',
)),
('# Annots', (
'num_multiton_annots',
'num_singleton_annots',
#'num_unknown_annots',
'num_annots')),
]
#multicol_lbls = [('# Names', 3), ('# Annots', 3)]
multicol_lbls = [(mcolname, len(mcols)) for mcolname, mcols in multi_col_keys]
# Flatten column labels
col_keys = ut.flatten(ut.get_list_column(multi_col_keys, 1))
col_lbls = ut.dict_take(key_to_col_lbls, col_keys)
row_lbls = []
row_values = []
#stat_col_lbls = ['max', 'min', 'mean', 'std', 'nMin', 'nMax']
stat_col_lbls = ['max', 'min', 'mean', 'std', 'med']
#stat_row_lbls = ['# Annot per Name (multiton)']
stat_row_lbls = []
stat_row_values = []
SINGLE_TABLE = False
EXTRA = True
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
row_ = ut.dict_take(dbinfo_locals, col_keys)
dbname = ibs.get_dbname_alias()
row_lbls.append(dbname)
multiton_annot_stats = ut.get_stats(dbinfo_locals['multiton_nid2_nannots'], use_median=True)
stat_rows = ut.dict_take(multiton_annot_stats, stat_col_lbls)
if SINGLE_TABLE:
row_.extend(stat_rows)
else:
stat_row_lbls.append(dbname)
stat_row_values.append(stat_rows)
row_values.append(row_)
CENTERLINE = False
AS_TABLE = True
tablekw = dict(
astable=AS_TABLE, centerline=CENTERLINE, FORCE_INT=False, precision=2,
col_sep='', multicol_sep='|',
**kwargs)
if EXTRA:
extra_keys = [
#'species2_nAids',
'qualtext2_nAnnots',
'yawtext2_nAnnots',
]
extra_titles = {
'species2_nAids': 'Annotations per species.',
'qualtext2_nAnnots': 'Annotations per quality.',
'yawtext2_nAnnots': 'Annotations per viewpoint.',
}
extra_collbls = ut.ddict(list)
extra_rowvalues = ut.ddict(list)
extra_tables = ut.ddict(list)
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
for key in extra_keys:
extra_collbls[key] = ut.unique_ordered(extra_collbls[key] + list(dbinfo_locals[key].keys()))
extra_collbls['qualtext2_nAnnots'] = ['excellent', 'good', 'ok', 'poor', 'junk', 'UNKNOWN']
#extra_collbls['yawtext2_nAnnots'] = ['backleft', 'left', 'frontleft', 'front', 'frontright', 'right', 'backright', 'back', None]
extra_collbls['yawtext2_nAnnots'] = ['BL', 'L', 'FL', 'F', 'FR', 'R', 'BR', 'B', None]
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
for key in extra_keys:
extra_rowvalues[key].append(ut.dict_take(dbinfo_locals[key], extra_collbls[key], 0))
qualalias = {'UNKNOWN': None}
extra_collbls['yawtext2_nAnnots'] = [ibs.const.YAWALIAS.get(val, val) for val in extra_collbls['yawtext2_nAnnots']]
extra_collbls['qualtext2_nAnnots'] = [qualalias.get(val, val) for val in extra_collbls['qualtext2_nAnnots']]
for key in extra_keys:
extra_tables[key] = ut.util_latex.make_score_tabular(
row_lbls, extra_collbls[key], extra_rowvalues[key],
title=extra_titles[key], col_align='r', table_position='[h!]', **tablekw)
#tabular_str = util_latex.tabular_join(tabular_body_list)
if SINGLE_TABLE:
col_lbls += stat_col_lbls
multicol_lbls += [(stat_title, len(stat_col_lbls))]
count_tabular_str = ut.util_latex.make_score_tabular(
row_lbls, col_lbls, row_values, title=title, multicol_lbls=multicol_lbls, table_position='[ht!]', **tablekw)
#print(row_lbls)
if SINGLE_TABLE:
tabular_str = count_tabular_str
else:
stat_tabular_str = ut.util_latex.make_score_tabular(
stat_row_lbls, stat_col_lbls, stat_row_values, title=stat_title,
col_align='r', table_position='[h!]', **tablekw)
# Make a table of statistics
if tablekw['astable']:
tablesep = '\n%--\n'
else:
tablesep = '\\\\\n%--\n'
if EXTRA:
tabular_str = tablesep.join([count_tabular_str, stat_tabular_str] + ut.dict_take(extra_tables, extra_keys))
else:
tabular_str = tablesep.join([count_tabular_str, stat_tabular_str])
return tabular_str
def latex_dbstats(ibs_list, **kwargs):
r"""
Args:
ibs (IBEISController): wbia controller object
CommandLine:
python -m wbia.other.dbinfo --exec-latex_dbstats --dblist testdb1
python -m wbia.other.dbinfo --exec-latex_dbstats --dblist testdb1 --show
python -m wbia.other.dbinfo --exec-latex_dbstats --dblist PZ_Master0 testdb1 --show
python -m wbia.other.dbinfo --exec-latex_dbstats --dblist PZ_Master0 PZ_MTEST GZ_ALL --show
python -m wbia.other.dbinfo --test-latex_dbstats --dblist GZ_ALL NNP_MasterGIRM_core --show
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.other.dbinfo import * # NOQA
>>> import wbia
>>> db_list = ut.get_argval('--dblist', type_=list, default=['testdb1'])
>>> ibs_list = [wbia.opendb(db=db) for db in db_list]
>>> tabular_str = latex_dbstats(ibs_list)
>>> tabular_cmd = ut.latex_newcommand(ut.latex_sanitize_command_name('DatabaseInfo'), tabular_str)
>>> ut.copy_text_to_clipboard(tabular_cmd)
>>> write_fpath = ut.get_argval('--write', type_=str, default=None)
>>> if write_fpath is not None:
>>> fpath = ut.truepath(write_fpath)
>>> text = ut.readfrom(fpath)
>>> new_text = ut.replace_between_tags(text, tabular_cmd, '% <DBINFO>', '% </DBINFO>')
>>> ut.writeto(fpath, new_text)
>>> ut.print_code(tabular_cmd, 'latex')
>>> ut.quit_if_noshow()
>>> ut.render_latex_text('\\noindent \n' + tabular_str)
"""
import wbia
# Parse for aids test data
aids_list = [wbia.testdata_aids(ibs=ibs) for ibs in ibs_list]
# dbinfo_list = [get_dbinfo(ibs, with_contrib=False, verbose=False) for ibs in ibs_list]
dbinfo_list = [
get_dbinfo(ibs, with_contrib=False, verbose=False, aid_list=aids)
for ibs, aids in zip(ibs_list, aids_list)
]
# title = db_name + ' database statistics'
title = 'Database statistics'
stat_title = '# Annotations per name (multiton)'
# col_lbls = [
# 'multiton',
# #'singleton',
# 'total',
# 'multiton',
# 'singleton',
# 'total',
# ]
key_to_col_lbls = {
'num_names_multiton': 'multiton',
'num_names_singleton': 'singleton',
'num_names': 'total',
'num_multiton_annots': 'multiton',
'num_singleton_annots': 'singleton',
'num_unknown_annots': 'unknown',
'num_annots': 'total',
}
# Structure of columns / multicolumns
multi_col_keys = [
(
'# Names',
(
'num_names_multiton',
# 'num_names_singleton',
'num_names',
),
),
(
'# Annots',
(
'num_multiton_annots',
'num_singleton_annots',
# 'num_unknown_annots',
'num_annots',
),
),
]
# multicol_lbls = [('# Names', 3), ('# Annots', 3)]
multicol_lbls = [(mcolname, len(mcols)) for mcolname, mcols in multi_col_keys]
# Flatten column labels
col_keys = ut.flatten(ut.get_list_column(multi_col_keys, 1))
col_lbls = ut.dict_take(key_to_col_lbls, col_keys)
row_lbls = []
row_values = []
# stat_col_lbls = ['max', 'min', 'mean', 'std', 'nMin', 'nMax']
stat_col_lbls = ['max', 'min', 'mean', 'std', 'med']
# stat_row_lbls = ['# Annot per Name (multiton)']
stat_row_lbls = []
stat_row_values = []
SINGLE_TABLE = False
EXTRA = True
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
row_ = ut.dict_take(dbinfo_locals, col_keys)
dbname = ibs.get_dbname_alias()
row_lbls.append(dbname)
multiton_annot_stats = ut.get_stats(
dbinfo_locals['multiton_nid2_nannots'], use_median=True, nl=1
)
stat_rows = ut.dict_take(multiton_annot_stats, stat_col_lbls)
if SINGLE_TABLE:
row_.extend(stat_rows)
else:
stat_row_lbls.append(dbname)
stat_row_values.append(stat_rows)
row_values.append(row_)
CENTERLINE = False
AS_TABLE = True
tablekw = dict(
astable=AS_TABLE,
centerline=CENTERLINE,
FORCE_INT=False,
precision=2,
col_sep='',
multicol_sep='|',
**kwargs
)
if EXTRA:
extra_keys = [
# 'species2_nAids',
'qualtext2_nAnnots',
'viewcode2_nAnnots',
]
extra_titles = {
'species2_nAids': 'Annotations per species.',
'qualtext2_nAnnots': 'Annotations per quality.',
'viewcode2_nAnnots': 'Annotations per viewpoint.',
}
extra_collbls = ut.ddict(list)
extra_rowvalues = ut.ddict(list)
extra_tables = ut.ddict(list)
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
for key in extra_keys:
extra_collbls[key] = ut.unique_ordered(
extra_collbls[key] + list(dbinfo_locals[key].keys())
)
extra_collbls['qualtext2_nAnnots'] = [
'excellent',
'good',
'ok',
'poor',
'junk',
'UNKNOWN',
]
# extra_collbls['viewcode2_nAnnots'] = ['backleft', 'left', 'frontleft', 'front', 'frontright', 'right', 'backright', 'back', None]
extra_collbls['viewcode2_nAnnots'] = [
'BL',
'L',
'FL',
'F',
'FR',
'R',
'BR',
'B',
None,
]
for ibs, dbinfo_locals in zip(ibs_list, dbinfo_list):
for key in extra_keys:
extra_rowvalues[key].append(
ut.dict_take(dbinfo_locals[key], extra_collbls[key], 0)
)
qualalias = {'UNKNOWN': None}
extra_collbls['viewcode2_nAnnots'] = [
ibs.const.YAWALIAS.get(val, val) for val in extra_collbls['viewcode2_nAnnots']
]
extra_collbls['qualtext2_nAnnots'] = [
qualalias.get(val, val) for val in extra_collbls['qualtext2_nAnnots']
]
for key in extra_keys:
extra_tables[key] = ut.util_latex.make_score_tabular(
row_lbls,
extra_collbls[key],
extra_rowvalues[key],
title=extra_titles[key],
col_align='r',
table_position='[h!]',
**tablekw
)
# tabular_str = util_latex.tabular_join(tabular_body_list)
if SINGLE_TABLE:
col_lbls += stat_col_lbls
multicol_lbls += [(stat_title, len(stat_col_lbls))]
count_tabular_str = ut.util_latex.make_score_tabular(
row_lbls,
col_lbls,
row_values,
title=title,
multicol_lbls=multicol_lbls,
table_position='[ht!]',
**tablekw
)
# logger.info(row_lbls)
if SINGLE_TABLE:
tabular_str = count_tabular_str
else:
stat_tabular_str = ut.util_latex.make_score_tabular(
stat_row_lbls,
stat_col_lbls,
stat_row_values,
title=stat_title,
col_align='r',
table_position='[h!]',
**tablekw
)
# Make a table of statistics
if tablekw['astable']:
tablesep = '\n%--\n'
else:
tablesep = '\\\\\n%--\n'
if EXTRA:
tabular_str = tablesep.join(
[count_tabular_str, stat_tabular_str]
+ ut.dict_take(extra_tables, extra_keys)
)
else:
tabular_str = tablesep.join([count_tabular_str, stat_tabular_str])
return tabular_str
def make_example_docstr(funcname=None, modname=None, argname_list=None,
defaults=None, return_type=None, return_name=None,
ismethod=False):
"""
Creates skeleton code to build an example doctest
Args:
funcname (str): function name
modname (str): module name
argname_list (str): list of argument names
defaults (None):
return_type (None):
return_name (str): return variable name
ismethod (bool):
Returns:
str: examplecode
CommandLine:
python -m utool.util_autogen --test-make_example_docstr
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_autogen import * # NOQA
>>> # build test data
>>> funcname = 'make_example_docstr'
>>> modname = 'utool.util_autogen'
>>> argname_list = ['qaids', 'qreq_']
>>> defaults = None
>>> return_type = tuple
>>> return_name = 'foo'
>>> ismethod = False
>>> # execute function
>>> examplecode = make_example_docstr(funcname, modname, argname_list, defaults, return_type, return_name, ismethod)
>>> # verify results
>>> result = str(examplecode)
>>> print(result)
# DISABLE_DOCTEST
from utool.util_autogen import * # NOQA
import utool as ut
import ibeis
species = ibeis.const.TEST_SPECIES.ZEB_PLAIN
qaids = ibs.get_valid_aids(species=species)
qreq_ = ibeis.testdata_qreq_()
foo = make_example_docstr(qaids, qreq_)
result = ('foo = %s' % (ut.repr2(foo),))
print(result)
"""
import utool as ut
examplecode_lines = []
top_import_fmstr = 'from {modname} import * # NOQA'
top_import = top_import_fmstr.format(modname=modname)
import_lines = [top_import]
if modname.startswith('utool'):
import_lines += ['import utool as ut']
is_show_func = not modname.startswith('utool') and not modname.startswith('mtgmonte')
# TODO: Externally register these
default_argval_map = {
'ibs' : 'ibeis.opendb(defaultdb=\'testdb1\')',
'testres' : 'ibeis.testdata_expts(\'PZ_MTEST\')',
'qreq_' : 'ibeis.testdata_qreq_()',
'cm_list' : 'qreq_.execute()',
'cm' : 'qreq_.execute()[0]',
'aid_list' : 'ibs.get_valid_aids()',
'nid_list' : 'ibs._get_all_known_nids()',
'qaids' : 'ibs.get_valid_aids(species=species)',
'daids' : 'ibs.get_valid_aids(species=species)',
'species' : 'ibeis.const.TEST_SPECIES.ZEB_PLAIN',
'kpts' : 'vt.dummy.get_dummy_kpts()',
'dodraw' : 'ut.show_was_requested()',
'img_fpath' : 'ut.grab_test_imgpath(\'carl.jpg\')',
'gfpath' : 'ut.grab_test_imgpath(\'carl.jpg\')',
'img' : 'vt.imread(img_fpath)',
'img_in' : 'vt.imread(img_fpath)',
'bbox' : '(10, 10, 50, 50)',
'theta' : '0.0',
'rng' : 'np.random.RandomState(0)',
}
import_depends_map = {
'ibeis': 'import ibeis',
'vt': 'import vtool as vt',
#'img': 'import vtool as vt', # TODO: remove. fix dependency
#'species': 'import ibeis',
}
var_depends_map = {
'species': ['ibeis'],
'ibs': ['ibeis'],
'testres': ['ibeis'],
'kpts': ['vt'],
#'qreq_': ['ibs', 'species', 'daids', 'qaids'],
'qreq_': ['ibeis'],
'qaids': ['ibs'],
'daids': ['ibs'],
'qaids': ['species'],
'daids': ['species'],
'img': ['img_fpath', 'vt'],
}
def find_arg_defaultrepr(argname, val):
import types
if val == '?':
if argname in default_argval_map:
val = ut.PythonStatement(default_argval_map[argname])
if argname in import_depends_map:
import_lines.append(import_depends_map[argname])
elif isinstance(val, types.ModuleType):
return val.__name__
return repr(val)
# augment argname list with dependencies
dependant_argnames = [] # deque()
def append_dependant_argnames(argnames, dependant_argnames):
""" use hints to add known dependencies for certain argument inputs """
for argname in argnames:
# Check if argname just implies an import
if argname in import_depends_map:
import_lines.append(import_depends_map[argname])
# Check if argname was already added as dependency
if (argname not in dependant_argnames and argname not in
argname_list and argname not in import_depends_map):
dependant_argnames.append(argname)
# Check if argname has dependants
if argname in var_depends_map:
argdeps = var_depends_map[argname]
# RECURSIVE CALL
append_dependant_argnames(argdeps, dependant_argnames)
append_dependant_argnames(argname_list, dependant_argnames)
# Define argnames and dependencies in example code
# argnames prefixed with dependeancies
argname_list_ = list(dependant_argnames) + argname_list
# Default example values
defaults_ = [] if defaults is None else defaults
num_unknown = (len(argname_list_) - len(defaults_))
default_vals = ['?'] * num_unknown + list(defaults_)
arg_val_iter = zip(argname_list_, default_vals)
infered_defaults = [find_arg_defaultrepr(argname, val)
for argname, val in arg_val_iter]
argdef_lines = ['%s = %s' % (argname, inferrepr)
for argname, inferrepr in
zip(argname_list_, infered_defaults)]
import_lines = ut.unique_ordered(import_lines)
if any([inferrepr == repr('?') for inferrepr in infered_defaults]):
examplecode_lines.append('# DISABLE_DOCTEST')
else:
# Enable the test if it can be run immediately
examplecode_lines.append('# DISABLE_DOCTEST')
examplecode_lines.extend(import_lines)
#examplecode_lines.append('# build test data')
examplecode_lines.extend(argdef_lines)
# Default example result assignment
result_assign = ''
result_print = None
if 'return_name' in vars():
if return_type is not None:
if return_name is None:
return_name = 'result'
result_assign = return_name + ' = '
result_print = 'print(result)' # + return_name + ')'
# Default example call
if ismethod:
selfname = argname_list[0]
methodargs = ', '.join(argname_list[1:])
tup = (selfname, '.', funcname, '(', methodargs, ')')
example_call = ''.join(tup)
else:
funcargs = ', '.join(argname_list)
tup = (funcname, '(', funcargs, ')')
example_call = ''.join(tup)
# Append call line
#examplecode_lines.append('# execute function')
examplecode_lines.append(result_assign + example_call)
#examplecode_lines.append('# verify results')
if result_print is not None:
if return_name != 'result':
#examplecode_lines.append('result = str(' + return_name + ')')
result_line_fmt = 'result = (\'{return_name} = %s\' % (ut.repr2({return_name}),))'
result_line = result_line_fmt.format(return_name=return_name)
examplecode_lines.append(result_line)
examplecode_lines.append(result_print)
# TODO: infer this
if is_show_func:
examplecode_lines += [
'ut.quit_if_noshow()',
'import plottool as pt',
'ut.show_if_requested()',
]
examplecode = '\n'.join(examplecode_lines)
return examplecode
def download_sharks(XMLdata, number):
"""
cd ~/work/WS_ALL
python -m ibeis.scripts.getshark
>>> from ibeis.scripts.getshark import * # NOQA
>>> url = 'www.whaleshark.org/listImages.jsp'
>>> XMLdata = ut.url_read(url)
>>> number = None
"""
# Prepare the output directory for writing, if it doesn't exist
output_dir = 'sharkimages'
ut.ensuredir(output_dir)
dom = parseString(XMLdata)
# Download files
if number:
maxCount = min(number, len(dom.getElementsByTagName('img')))
else:
maxCount = len(dom.getElementsByTagName('img'))
parsed_info = dict(
img_url_list=[],
localid_list=[],
nameid_list=[],
orig_fname_list=[],
new_fname_list=[],
)
print('Preparing to fetch %i files...' % maxCount)
for shark in dom.getElementsByTagName('shark'):
localCount = 0
for imageset in shark.getElementsByTagName('imageset'):
for img in imageset.getElementsByTagName('img'):
localCount += 1
img_url = img.getAttribute('href')
orig_fname = split(img_url)[1]
ext = splitext(orig_fname)[1].lower()
nameid = shark.getAttribute('number')
new_fname = '%s-%i%s' % (nameid, localCount, ext)
parsed_info['img_url_list'].append(img_url)
parsed_info['nameid_list'].append(nameid)
parsed_info['localid_list'].append(localCount)
parsed_info['orig_fname_list'].append(orig_fname)
parsed_info['new_fname_list'].append(new_fname)
print('Parsed %i / %i files.' %
(len(parsed_info['orig_fname_list']), maxCount))
if number is not None and len(
parsed_info['orig_fname_list']) == number:
break
parsed_info['new_fpath_list'] = [
join(output_dir, _fname) for _fname in parsed_info['new_fname_list']
]
print('Filtering parsed images')
# Filter based on image type (keep only jpgs)
ext_flags = [
_fname.endswith('.jpg') or _fname.endswith('.jpg')
for _fname in parsed_info['new_fname_list']
]
parsed_info = {
key: ut.compress(list_, ext_flags)
for key, list_ in parsed_info.items()
}
# Filter to only images matching the appropriate tags
from ibeis import tag_funcs
parsed_info['tags_list'] = parse_shark_tags(parsed_info['orig_fname_list'])
tag_flags = tag_funcs.filterflags_general_tags(
parsed_info['tags_list'],
has_any=['view-left'],
none_match=['qual.*', 'view-top', 'part-.*', 'cropped'],
)
parsed_info = {
key: ut.compress(list_, tag_flags)
for key, list_ in parsed_info.items()
}
print('Tags in chosen images:')
print(ut.dict_hist(ut.flatten(parsed_info['tags_list'])))
# Download selected subset
print('Downloading selected subset')
_iter = list(
zip(parsed_info['img_url_list'], parsed_info['new_fpath_list']))
_iter = ut.ProgressIter(_iter, lbl='downloading sharks')
for img_url, new_fpath in _iter:
if not exists(new_fpath):
ut.download_url(img_url, new_fpath)
# Remove corrupted or ill-formatted images
print('Checking for corrupted images')
import vtool as vt
noncorrupt_flags = vt.filterflags_valid_images(
parsed_info['new_fpath_list'])
parsed_info = {
key: ut.compress(list_, noncorrupt_flags)
for key, list_ in parsed_info.items()
}
print('Removing small images')
import numpy as np
imgsize_list = np.array(
[vt.open_image_size(gpath) for gpath in parsed_info['new_fpath_list']])
sqrt_area_list = np.sqrt(np.prod(imgsize_list, axis=1))
areq_flags_list = sqrt_area_list >= 750
parsed_info = {
key: ut.compress(list_, areq_flags_list)
for key, list_ in parsed_info.items()
}
grouped_idxs = ut.group_items(list(range(len(parsed_info['nameid_list']))),
parsed_info['nameid_list'])
keep_idxs = sorted(
ut.flatten(
[idxs for key, idxs in grouped_idxs.items() if len(idxs) >= 2]))
parsed_info = {
key: ut.take(list_, keep_idxs)
for key, list_ in parsed_info.items()
}
print('Moving imagse to secondary directory')
named_outputdir = 'named-left-sharkimages'
# Build names
parsed_info['namedir_fpath_list'] = [
join(named_outputdir, _nameid, _fname) for _fname, _nameid in zip(
parsed_info['new_fname_list'], parsed_info['nameid_list'])
]
# Create directories
ut.ensuredir(named_outputdir)
named_dirs = ut.unique_ordered(
list(map(dirname, parsed_info['namedir_fpath_list'])))
for dir_ in named_dirs:
ut.ensuredir(dir_)
# Copy
ut.copy_files_to(src_fpath_list=parsed_info['new_fpath_list'],
dst_fpath_list=parsed_info['namedir_fpath_list'])
def query_vsone_pairs(ibs,
vsone_query_pairs,
use_cache=False,
save_qcache=False):
"""
does vsone queries to rerank the top few vsmany querys
Returns:
tuple: qaid2_qres_vsone, qreq_vsone_
CommandLine:
python -m ibeis.algo.hots.special_query --test-query_vsone_pairs
Example:
>>> # SLOW_DOCTEST
>>> from ibeis.algo.hots.special_query import * # NOQA
>>> ibs, valid_aids = testdata_special_query()
>>> qaids = valid_aids[0:1]
>>> daids = valid_aids[1:]
>>> qaid = qaids[0]
>>> filtkey = hstypes.FiltKeys.DISTINCTIVENESS
>>> use_cache = False
>>> save_qcache = False
>>> # execute function
>>> qaid2_qres_vsmany, qreq_vsmany_ = query_vsmany_initial(ibs, qaids, daids)
>>> vsone_query_pairs = build_vsone_shortlist(ibs, qaid2_qres_vsmany)
>>> qaid2_qres_vsone, qreq_vsone_ = query_vsone_pairs(ibs, vsone_query_pairs)
>>> qres_vsone = qaid2_qres_vsone[qaid]
>>> top_namescore_aids = qres_vsone.get_top_aids().tolist()
>>> result = str(top_namescore_aids)
>>> top_namescore_names = ibs.get_annot_names(top_namescore_aids)
>>> assert top_namescore_names[0] == 'easy', 'top_namescore_names[0]=%r' % (top_namescore_names[0],)
"""
#vsone_cfgdict = dict(codename='vsone_unnorm')
#codename = 'vsone_unnorm_dist_ratio_extern_distinctiveness',
codename = 'vsone_unnorm_dist_ratio'
vsone_cfgdict = dict(
index_method='single',
codename=codename,
)
#------------------------
qaid2_qres_vsone = {}
for qaid, top_aids in vsone_query_pairs:
# Perform a query request for each
cm_list_vsone_, __qreq_vsone_ = ibs.query_chips(
[qaid],
top_aids,
cfgdict=vsone_cfgdict,
return_request=True,
use_cache=use_cache,
save_qcache=save_qcache)
qaid2_qres_vsone_ = {cm.qaid: cm for cm in cm_list_vsone_}
qaid2_qres_vsone.update(qaid2_qres_vsone_)
#------------------------
# Create pseudo query request because there is no good way to
# represent the vsone reranking as a single query request and
# we need one for the score normalizer
#pseudo_codename_ = codename.replace('unnorm', 'norm') + '_extern_distinctiveness'
pseudo_codename_ = codename.replace('unnorm',
'norm') # + '_extern_distinctiveness'
pseudo_vsone_cfgdict = dict(codename=pseudo_codename_)
pseudo_qaids = ut.get_list_column(vsone_query_pairs, 0)
pseudo_daids = ut.unique_ordered(
ut.flatten(ut.get_list_column(vsone_query_pairs, 1)))
# FIXME: making the pseudo qreq_ takes a nontrivial amount of time for what
# should be a trivial task.
pseudo_qreq_vsone_ = ibs.new_query_request(pseudo_qaids,
pseudo_daids,
cfgdict=pseudo_vsone_cfgdict,
verbose=ut.VERBOSE)
#pseudo_qreq_vsone_.load_distinctiveness_normalizer()
qreq_vsone_ = pseudo_qreq_vsone_
# Hack in a special config name
qreq_vsone_.qparams.query_cfgstr = '_special' + qreq_vsone_.qparams.query_cfgstr
return qaid2_qres_vsone, qreq_vsone_
def show_name_matches(ibs,
qaid,
name_daid_list,
name_fm_list,
name_fs_list,
name_H1_list,
name_featflag_list,
qreq_=None,
**kwargs):
"""
Called from chip_match.py
Args:
ibs (IBEISController): ibeis controller object
qaid (int): query annotation id
name_daid_list (list):
name_fm_list (list):
name_fs_list (list):
name_H1_list (list):
name_featflag_list (list):
qreq_ (QueryRequest): query request object with hyper-parameters(default = None)
Kwargs:
draw_fmatches, name_rank, fnum, pnum, colorbar_, nonvote_mode,
fastmode, show_matches, fs, fm_norm, lbl1, lbl2, rect, draw_border,
cmap, H1, H2, scale_factor1, scale_factor2, draw_pts, draw_ell,
draw_lines, show_nMatches, all_kpts, in_image, show_query, draw_lbl,
name_annot_scores, score, rawscore, aid2_raw_rank, show_name,
show_nid, show_aid, show_annot_score, show_truth, name_score,
show_name_score, show_name_rank, show_timedelta
CommandLine:
python -m ibeis.viz.viz_matches --exec-show_name_matches
python -m ibeis.viz.viz_matches --test-show_name_matches --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_matches import * # NOQA
>>> from ibeis.algo.hots import chip_match
>>> from ibeis.algo.hots import name_scoring
>>> import vtool as vt
>>> from ibeis.algo.hots import _pipeline_helpers as plh # NOQA
>>> import numpy as np
>>> func = chip_match.ChipMatch.show_single_namematch
>>> sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True,
>>> strip_docstr=True)
>>> setup = ut.regex_replace('viz_matches.show_name_matches', '#', sourcecode)
>>> homog = False
>>> print(ut.indent(setup, '>>> '))
>>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[1])
>>> cm = cm_list[0]
>>> cm.score_nsum(qreq_)
>>> dnid = ibs.get_annot_nids(cm.qaid)
>>> # +--- COPIED SECTION
>>> locals_ = locals()
>>> var_list = ut.exec_func_src(
>>> func, locals_=locals_,
>>> sentinal='name_annot_scores = cm.annot_score_list.take(sorted_groupxs')
>>> exec(ut.execstr_dict(var_list))
>>> # L___ COPIED SECTION
>>> kwargs = {}
>>> show_name_matches(ibs, qaid, name_daid_list, name_fm_list,
>>> name_fs_list, name_h1_list, name_featflag_list,
>>> qreq_=qreq_, **kwargs)
>>> ut.quit_if_noshow()
>>> ut.show_if_requested()
"""
#print("SHOW NAME MATCHES")
#print(ut.repr2(kwargs, nl=True))
#from ibeis import constants as const
from ibeis import tag_funcs
draw_fmatches = kwargs.pop('draw_fmatches', True)
rchip1, kpts1 = get_query_annot_pair_info(ibs, qaid, qreq_, draw_fmatches)
rchip2_list, kpts2_list = get_data_annot_pair_info(ibs, name_daid_list,
qreq_, draw_fmatches)
fm_list = name_fm_list
fs_list = name_fs_list
featflag_list = name_featflag_list
offset_list, sf_list, bbox_list = show_multichip_match(
rchip1, rchip2_list, kpts1, kpts2_list, fm_list, fs_list,
featflag_list, **kwargs)
aid_list = [qaid] + name_daid_list
annotate_matches3(ibs,
aid_list,
bbox_list,
offset_list,
name_fm_list,
name_fs_list,
qreq_=None,
**kwargs)
ax = pt.gca()
title = vh.get_query_text(ibs,
None,
name_daid_list,
False,
qaid=qaid,
**kwargs)
pt.set_title(title, ax)
# Case tags
annotmatch_rowid_list = ibs.get_annotmatch_rowid_from_superkey(
[qaid] * len(name_daid_list), name_daid_list)
annotmatch_rowid_list = ut.filter_Nones(annotmatch_rowid_list)
tags_list = ibs.get_annotmatch_case_tags(annotmatch_rowid_list)
if not ut.get_argflag('--show'): # False:
tags_list = tag_funcs.consolodate_annotmatch_tags(tags_list)
tag_list = ut.unique_ordered(ut.flatten(tags_list))
name_rank = kwargs.get('name_rank', None)
truth = get_multitruth(ibs, aid_list)
xlabel = {1: 'Correct ID', 0: 'Incorrect ID', 2: 'Unknown ID'}[truth]
if False:
if name_rank is None:
xlabel = {1: 'Genuine', 0: 'Imposter', 2: 'Unknown'}[truth]
#xlabel = {1: 'True', 0: 'False', 2: 'Unknown'}[truth]
else:
if name_rank == 0:
xlabel = {
1: 'True Positive',
0: 'False Positive',
2: 'Unknown'
}[truth]
else:
xlabel = {
1: 'False Negative',
0: 'True Negative',
2: 'Unknown'
}[truth]
if len(tag_list) > 0:
xlabel += '\n' + ', '.join(tag_list)
pt.set_xlabel(xlabel)
return ax
def get_all_descendant_rowids(depc, tablename, root_rowids, config=None,
ensure=True, eager=True, nInput=None,
recompute=False, recompute_all=False,
levels_up=None, _debug=False):
r"""
Connects `root_rowids` to rowids in `tablename`, and computes all
values needed along the way. This is the main workhorse function for
dependency computations.
Args:
tablename (str): table to compute dependencies to
root_rowids (list): rowids for ``tablename``
config (dict): config applicable for all tables (default = None)
ensure (bool): eager evaluation if True(default = True)
eager (bool): (default = True)
nInput (None): (default = None)
recompute (bool): (default = False)
recompute_all (bool): (default = False)
levels_up (int): only partially compute dependencies (default = 0)
_debug (bool): (default = False)
CommandLine:
python -m dtool.depcache_control --exec-get_all_descendant_rowids:0
python -m dtool.depcache_control --exec-get_all_descendant_rowids:1
Example:
>>> # DISABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> tablename = 'spam'
>>> root_rowids = [1, 2]
>>> config1 = {'dim_size': 500}
>>> config2 = {'dim_size': 100}
>>> config3 = {'dim_size': 500, 'adapt_shape': False}
>>> ensure, eager, nInput = True, True, None
>>> _debug = True
>>> rowid_dict1 = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config1, ensure, eager, nInput, _debug=_debug)
>>> rowid_dict2 = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config2, ensure, eager, nInput, _debug=_debug)
>>> rowid_dict3 = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config3, ensure, eager, nInput, _debug=_debug)
>>> result1 = 'rowid_dict1 = ' + ut.repr3(rowid_dict1, nl=1)
>>> result2 = 'rowid_dict2 = ' + ut.repr3(rowid_dict2, nl=1)
>>> result3 = 'rowid_dict3 = ' + ut.repr3(rowid_dict3, nl=1)
>>> result = '\n'.join([result1, result2, result3])
>>> print(result)
rowid_dict1 = {
'chip': [1, 2],
'dummy_annot': [1, 2],
'fgweight': [1, 2],
'keypoint': [1, 2],
'probchip': [1, 2],
'spam': [1, 2],
}
rowid_dict2 = {
'chip': [3, 4],
'dummy_annot': [1, 2],
'fgweight': [3, 4],
'keypoint': [3, 4],
'probchip': [1, 2],
'spam': [3, 4],
}
rowid_dict3 = {
'chip': [1, 2],
'dummy_annot': [1, 2],
'fgweight': [5, 6],
'keypoint': [5, 6],
'probchip': [1, 2],
'spam': [5, 6],
}
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> _debug = True
>>> tablename = 'vsmany'
>>> config = depc.configclass_dict['vsmany']()
>>> root_rowids = [1, 2, 3]
>>> ensure, eager, nInput = False, True, None
>>> # Get rowids of algo ( should be None )
>>> rowid_dict = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config, ensure, eager, nInput,
>>> _debug=_debug)
>>> result = ut.repr3(rowid_dict, nl=1)
>>> print(result)
{
'dummy_annot': [1, 2, 3],
'vsmany': [None, None, None],
}
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> # Make sure algo config can correctly get properites
>>> depc = testdata_depc()
>>> tablename = 'chip'
>>> recompute = False
>>> recompute_all = False
>>> _debug = True
>>> root_rowids = [1, 2]
>>> configclass = depc.configclass_dict['chip']
>>> config_ = configclass()
>>> config1 = depc.configclass_dict['vsmany'](dim_size=500)
>>> config2 = depc.configclass_dict['vsmany'](dim_size=100)
>>> config = config2
>>> prop_dicts1 = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config=config1, _debug=_debug)
>>> prop_dicts2 = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config=config2, _debug=_debug)
>>> print(prop_dicts2)
>>> print(prop_dicts1)
>>> assert prop_dicts1 != prop_dicts2
Example:
>>> # ENABLE_DOCTEST
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> depc = testdata_depc()
>>> exec(ut.execstr_funckw(depc.get_all_descendant_rowids), globals())
>>> _debug = True
>>> qaids, daids = [1, 2, 4], [2, 3, 4]
>>> root_rowids = list(zip(*ut.product(qaids, daids)))
>>> request = depc.new_request('vsone', qaids, daids)
>>> results = request.execute()
>>> tablename = 'vsone'
>>> rowid_dict = depc.get_all_descendant_rowids(
>>> tablename, root_rowids, config=None, _debug=_debug)
"""
# TODO: Need to have a nice way of ensuring configs dont overlap
# via namespaces.
_debug = depc._debug if _debug is None else _debug
indenter = ut.Indenter('[Descend-to-%s]' % (tablename,), enabled=_debug)
if _debug:
indenter.start()
print(' * GET DESCENDANT ROWIDS %s ' % (tablename,))
print(' * config = %r' % (config,))
dependency_levels = depc.get_dependencies(tablename)
if levels_up is not None:
dependency_levels = dependency_levels[:-levels_up]
configclass_levels = [
[depc.configclass_dict.get(tablekey, None)
for tablekey in keys]
for keys in dependency_levels
]
if _debug:
print('[depc] dependency_levels = %s' %
ut.repr3(dependency_levels, nl=1))
print('[depc] config_levels = %s' %
ut.repr3(configclass_levels, nl=1))
# TODO: better support for multi-edges
if (len(root_rowids) > 0 and ut.isiterable(root_rowids[0]) and
not depc[tablename].ismulti):
rowid_dict = {}
for colx, col in enumerate(root_rowids):
rowid_dict[depc.root + '%d' % (colx + 1,)] = col
rowid_dict[depc.root] = ut.unique_ordered(ut.flatten(root_rowids))
else:
rowid_dict = {depc.root: root_rowids}
# Ensure that each level ``tablename``'s dependencies have been computed
for level_keys in dependency_levels[1:]:
if _debug:
print(' * level_keys %s ' % (level_keys,))
# For each table in the level
for tablekey in level_keys:
try:
child_rowids = depc._expand_level_rowids(
tablename, tablekey, rowid_dict, ensure, eager, nInput,
config, recompute, recompute_all, _debug)
except Exception as ex:
table = depc[tablekey] # NOQA
keys = ['tablename', 'tablekey', 'rowid_dict', 'config',
'table', 'dependency_levels']
ut.printex(ex, 'error expanding rowids', keys=keys)
raise
rowid_dict[tablekey] = child_rowids
if _debug:
print(' GOT DESCENDANT ROWIDS')
indenter.stop()
return rowid_dict
