def shallowcopy(qreq_, qaids=None):
"""
Creates a copy of qreq with the same qparams object and a subset of
the qx and dx objects. used to generate chunks of vsone and vsmany
queries
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.smk.match_chips5 import * # NOQA
>>> from wbia.algo.smk.smk_pipeline import testdata_smk
>>> import wbia
>>> wbia, smk, qreq_ = testdata_smk()
>>> qreq2_ = qreq_.shallowcopy(qaids=1)
>>> assert qreq_.daids is qreq2_.daids, 'should be the same'
>>> assert len(qreq_.qaids) != len(qreq2_.qaids), 'should be diff'
>>> #assert qreq_.metadata is not qreq2_.metadata
"""
# qreq2_ = qreq_.__class__()
cls = qreq_.__class__
qreq2_ = cls.__new__(cls)
qreq2_.__dict__.update(qreq_.__dict__)
qaids = ut.ensure_iterable(qaids)
assert ut.issubset(qaids, qreq_.qaids), 'not a subset'
qreq2_.qaids = qaids
return qreq2_
def __init__(repo, url=None, code_dir=None, dpath=None,
modname=None, pythoncmd=None):
# modname might need to be called egg?
import utool as ut
if url is not None and '.git@' in url:
# parse out specific branch
repo.default_branch = url.split('@')[-1]
url = '@'.join(url.split('@')[:-1])
else:
repo.default_branch = None
repo.url = url
repo._modname = None
if modname is None:
modname = []
repo._modname_hints = ut.ensure_iterable(modname)
repo.dpath = None
repo.scripts = {}
if pythoncmd is None:
import sys
pythoncmd = sys.executable
repo.pythoncmd = pythoncmd
if dpath is None and repo.url is not None and code_dir is not None:
dpath = join(code_dir, repo.reponame)
if dpath is not None:
repo.dpath = util_path.unixpath(dpath)
def view_file_in_directory(fpaths):
import utool as ut
fpaths = ut.ensure_iterable(fpaths)
fnames = [basename(f) for f in fpaths]
dpaths = [dirname(f) for f in fpaths]
dpath_to_fnames = ut.group_items(fnames, dpaths)
for dpath, fnames in dpath_to_fnames.items():
ut.view_directory(dpath, fnames[0], verbose=False)
def _exec_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
if ignore_case:
fields = [f.lower() for f in fields]
flags_ = _flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
def get_prefered_browser(pref_list=[], fallback=True):
r"""
Args:
browser_preferences (list): (default = [])
fallback (bool): uses default if non of preferences work (default = True)
CommandLine:
python -m utool.util_grabdata --test-get_prefered_browser
Ignore:
import webbrowser
webbrowser._tryorder
pref_list = ['chrome', 'firefox', 'google-chrome']
pref_list = ['firefox', 'google-chrome']
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_grabdata import * # NOQA
>>> browser_preferences = ['firefox', 'chrome', 'safari']
>>> fallback = True
>>> browser = get_prefered_browser(browser_preferences, fallback)
>>> result = ('browser = %s' % (str(browser),))
>>> print(result)
>>> ut.quit_if_noshow()
"""
import webbrowser
import utool as ut
pref_list = ut.ensure_iterable(pref_list)
error_list = []
# Hack for finding chrome on win32
if ut.WIN32:
# http://stackoverflow.com/questions/24873302/webbrowser-chrome-exe-does-not-work
win32_chrome_fpath = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
win32_chrome_browsername = win32_chrome_fpath + ' %s'
win32_map = {
'chrome': win32_chrome_browsername,
'google-chrome': win32_chrome_browsername,
}
for browsername, win32_browsername in win32_map.items():
index = ut.listfind(pref_list, browsername)
if index is not None and True: # ut.checkpath(win32_browsername):
pref_list.insert(index + 1, win32_browsername)
for browsername in pref_list:
try:
browser = webbrowser.get(browsername)
return browser
except webbrowser.Error as ex:
error_list.append(ex)
print(str(browsername) + ' failed. Reason: ' + str(ex))
if fallback:
browser = webbrowser
return browser
else:
raise AssertionError('No browser meets preferences=%r. error_list=%r' %
(pref_list, error_list,))
def imagesets(ibs, gsids=None, text=None):
if text is not None:
gsids = ibs.get_imageset_imgsetids_from_text(text)
if gsids is None:
gsids = ibs.get_valid_imgsetids()
elif gsids.__class__.__name__ == 'ImageSets':
return gsids
gsids = ut.ensure_iterable(gsids)
return ImageSets(gsids, ibs)
def remove_annot_case_tags(ibs, aid_list, tag_list):
if isinstance(tag_list, six.string_types):
# Apply single tag to everybody
tag_list = [tag_list] * len(aid_list)
tags_list = [ut.ensure_iterable(tag) for tag in tag_list]
text_list = ibs.get_annot_tag_text(aid_list)
orig_tags_list = [[] if note is None else _parse_tags(note) for note in text_list]
new_tags_list = [ut.setdiff(t2, t1) for t1, t2 in zip(tags_list, orig_tags_list)]
new_text_list = [';'.join(tags) for tags in new_tags_list]
ibs.set_annot_tag_text(aid_list, new_text_list)
def dummy_single_func(depc, *row_arg, **kw):
config = kw.get('config')
param_val = config[config_param]
data = []
for row, p in zip(row_arg, parents):
p = p.replace('*', '')
# print('p = %r' % (p,))
# print('row = %r' % (row,))
if not p.startswith(depc.root):
native_cols = depc.get_native(p, ut.ensure_iterable(row))
parent_data = '+'.join(
['#'.join(col) for col in native_cols])
else:
parent_data = ('root(' + ';'.join(
list(map(str, ut.ensure_iterable(row)))) + ')')
data += [parent_data]
d = '[' + '&'.join(data) + ']'
retstr = tablename + '(' + d + ':' + str(param_val) + ')'
return (retstr, )
def images(ibs, gids=None, uuids=None, **kwargs):
""" Makes an Images object """
if uuids is not None:
assert gids is None, 'specify one primary key'
gids = ibs.get_image_gids_from_uuid(uuids)
if gids is None:
gids = ibs.get_valid_gids()
elif gids.__class__.__name__ == 'Images':
return gids
gids = ut.ensure_iterable(gids)
return Images(gids, ibs, **kwargs)
def get_annotmatch_prop(ibs, prop, annotmatch_rowids):
r"""
hacky getter for dynamic properties of annotmatches using notes table
Args:
prop (str):
annotmatch_rowids (?):
Returns:
list: filtered_aid_list
CommandLine:
python -m wbia.tag_funcs --exec-get_annotmatch_prop
Example:
>>> # DISABLE_DOCTEST
>>> # Test setting and getting standard keys
>>> from wbia.tag_funcs import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> prop = 'hard'
>>> annotmatch_rowids = ibs._get_all_annotmatch_rowids()
>>> flag_list = get_annotmatch_prop(ibs, prop, annotmatch_rowids)
>>> flag_list = ('filtered_aid_list = %s' % (str(flag_list),))
>>> subset_rowids = annotmatch_rowids[::2]
>>> set_annotmatch_prop(ibs, prop, subset_rowids, [True] * len(subset_rowids))
>>> flag_list2 = get_annotmatch_prop(ibs, prop, annotmatch_rowids)
>>> print('flag_list2 = %r' % (flag_list2,))
Example:
>>> # DISABLE_DOCTEST
>>> # Test setting and getting non-standard keys
>>> from wbia.tag_funcs import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> prop = 'occlusion'
>>> annotmatch_rowids = ibs._get_all_annotmatch_rowids()
>>> flag_list = get_annotmatch_prop(ibs, prop, annotmatch_rowids)
>>> flag_list = ('filtered_aid_list = %s' % (str(flag_list),))
>>> subset_rowids = annotmatch_rowids[1::2]
>>> subset_rowids1 = annotmatch_rowids[::2]
>>> set_annotmatch_prop(ibs, prop, subset_rowids1, [True] * len(subset_rowids))
>>> set_annotmatch_prop(ibs, 'pose', subset_rowids1, [True] * len(subset_rowids))
>>> flag_list2 = get_annotmatch_prop(ibs, prop, annotmatch_rowids)
>>> print('flag_list2 = %r' % (flag_list2,))
"""
# if prop.lower() in ANNOTMATCH_PROPS_STANDARD_SET:
# return ibs.get_annotmatch_standard_prop(prop, annotmatch_rowids)
for prop_ in ut.ensure_iterable(prop):
flag1 = prop_.lower() not in ANNOTMATCH_PROPS_OTHER_SET
flag2 = prop_.lower() not in ANNOTMATCH_PROPS_OLD_SET
if flag1 and flag2:
raise NotImplementedError('Unknown prop_=%r' % (prop_, ))
return get_annotmatch_other_prop(ibs, prop, annotmatch_rowids)
def annots(ibs, aids=None, uuids=None, **kwargs):
""" Makes an Annots object """
if uuids is not None:
assert aids is None, 'specify one primary key'
aids = ibs.get_annot_aids_from_uuid(uuids)
if aids is None:
aids = ibs.get_valid_aids()
elif aids.__class__.__name__ == 'Annots':
return aids
aids = ut.ensure_iterable(aids)
return Annots(aids, ibs, **kwargs)
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False):
"""
issues a command on a repo
CommandLine:
python -m utool.util_git --exec-repocmd
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True))
>>> command = 'git status'
>>> sudo = False
>>> result = repocmd(repo, command, sudo)
>>> print(result)
"""
import utool as ut
if ut.WIN32:
assert not sudo, 'cant sudo on windows'
if command == 'short_status':
return repo.short_status()
command_list = ut.ensure_iterable(command)
cmdstr = '\n '.join([cmd_ for cmd_ in command_list])
if not dry:
print('+--- *** repocmd(%s) *** ' % (cmdstr,))
print('repo=%s' % ut.color_text(repo.dpath, 'yellow'))
verbose = True
with repo.chdir_context():
ret = None
for count, cmd in enumerate(command_list):
if dry:
print(cmd)
continue
if not sudo or ut.WIN32:
# ret = os.system(cmd)
cmdinfo = ut.cmd2(cmd, verbout=True)
out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret'])
else:
# cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1)
out, err, ret = ut.cmd(cmd, sudo=True)
if verbose > 1:
print('ret(%d) = %r' % (count, ret,))
if ret != 0:
if error == 'raise':
raise Exception('Failed command %r' % (cmd,))
elif error == 'return':
return out
else:
raise ValueError('unknown flag error=%r' % (error,))
if return_out:
return out
if not dry:
print('L____')
def make_standard_test_scripts(*args):
if len(args) == 1:
varydict, expt_name, media_name = args[0]
else:
varydict, expt_name, media_name = args
media_names = ut.ensure_iterable(media_name)
cmd_fmtstr_list = []
for media_name in media_names:
cmd_flaglist = get_results_command(expt_name, media_name)
cmd_fmtstr = ''.join(cmd_flaglist)
cmd_fmtstr = ' \\\n '.join(cmd_flaglist)
cmd_fmtstr_list.append(cmd_fmtstr)
fname = 'experiment_' + expt_name + '.sh'
return write_formatted_script_lines(cmd_fmtstr_list, [varydict], fname)
def make_standard_test_scripts(*args):
if len(args) == 1:
varydict, expt_name, media_name = args[0]
else:
varydict, expt_name, media_name = args
media_names = ut.ensure_iterable(media_name)
cmd_fmtstr_list = []
for media_name in media_names:
cmd_flaglist = get_results_command(expt_name, media_name)
cmd_fmtstr = ''.join(cmd_flaglist)
cmd_fmtstr = ' \\\n '.join(cmd_flaglist)
cmd_fmtstr_list.append(cmd_fmtstr)
fname = 'experiment_' + expt_name + '.sh'
return write_formatted_script_lines(cmd_fmtstr_list, [varydict], fname)
def generate_class_images(dream, target_labels):
"""
import plottool as pt
fnum = None
kw = dict(init='gauss', niters=500, update_rate=.05, weight_decay=1e-4)
target_labels = list(range(model.output_dims))
dream = draw_net.Dream(model, **kw)
target_labels = 8
images = list(dream.generate_class_images(target_labels))
vid = vt.make_video(images, 'dynimg.pimj', fps=1, is_color=False, format='PIM1')
vid = vt.make_video2(images, 'dynimg')
import matplotlib.pyplot as plt
ims = []
for img in imgs:
im = plt.imshow(img[:, :, 0], interpolation='nearest', cmap='gray')
ims.append([im])
import matplotlib.animation as animation
fig = plt.figure()
ani = animation.ArtistAnimation(fig, ims, interval=50, blit=True,
repeat_delay=1000)
ani.save('dynamic_images.mp4')
ut.startfile('dynamic_images.mp4')
plt.show()
"""
import ibeis_cnn.__THEANO__ as theano
from ibeis_cnn.__THEANO__ import tensor as T # NOQA
import utool as ut
input_shape = dream.model.input_shape
b, c, w, h = input_shape
was_scalar = not ut.isiterable(target_labels)
target_labels = ut.ensure_iterable(target_labels)
assert len(target_labels) <= b, 'batch size too small'
initial_state = dream._make_init_state()
shared_images = theano.shared(initial_state.astype(np.float32))
step_fn = dream._make_objective(shared_images, target_labels)
out = dream._postprocess_class_image(shared_images, target_labels,
was_scalar)
yield out
for _ in ut.ProgIter(range(dream.niters), lbl='class dream',
bs=True):
step_fn()
# objective = step_fn()
# print('objective = %r' % (objective,))
out = dream._postprocess_class_image(shared_images, target_labels,
was_scalar)
yield out
def __init__(repo, url=None, code_dir=None, dpath=None, modname=None, pythoncmd=None):
# modname might need to be called egg?
import utool as ut
repo.url = url
repo._modname = None
if modname is None:
modname = []
repo._modname_hints = ut.ensure_iterable(modname)
repo.dpath = None
repo.scripts = {}
repo.pythoncmd = pythoncmd
if dpath is None and repo.url is not None and code_dir is not None:
dpath = join(code_dir, repo.reponame)
if dpath is not None:
repo.dpath = util_path.unixpath(dpath)
def __init__(repo, url=None, code_dir=None, dpath=None,
modname=None, pythoncmd=None):
# modname might need to be called egg?
import utool as ut
repo.url = url
repo._modname = None
if modname is None:
modname = []
repo._modname_hints = ut.ensure_iterable(modname)
repo.dpath = None
repo.scripts = {}
repo.pythoncmd = pythoncmd
if dpath is None and repo.url is not None and code_dir is not None:
dpath = join(code_dir, repo.reponame)
if dpath is not None:
repo.dpath = util_path.unixpath(dpath)
def get_test_qaids(ibs):
""" Gets test annotation_rowids based on command line arguments """
#print('[main_helpers]')
test_qaids = []
valid_aids = ibs.get_valid_aids()
printDBG('1. valid_aids = %r' % valid_aids[0:5])
#print(utool.dict_str(vars(params.args)))
if params.args.qaid is not None:
printDBG('Testing qaid=%r' % params.args.qaid)
test_qaids.extend(params.args.qaid)
if params.args.all_cases:
printDBG('Testing all %d cases' % (len(valid_aids),))
printDBG('1. test_qaids = %r' % test_qaids[0:5])
test_qaids.extend(valid_aids)
printDBG('2. test_qaids = %r' % test_qaids[0:5])
else:
is_hard_list = ibsfuncs.get_annot_is_hard(ibs, valid_aids)
hard_aids = utool.filter_items(valid_aids, is_hard_list)
printDBG('Testing %d known hard cases' % len(hard_aids))
test_qaids.extend(hard_aids)
if params.args.all_gt_cases:
has_gt_list = ibs.get_annot_has_groundtruth(valid_aids)
hasgt_aids = utool.filter_items(valid_aids, has_gt_list)
print('Testing all %d ground-truthed cases' % len(hasgt_aids))
test_qaids.extend(hasgt_aids)
# Sample a large pool of query indexes
# Filter only the ones you want from the large pool
if params.args.index is not None:
indexes = utool.ensure_iterable(params.args.index)
#printDBG('Chosen indexes=%r' % (indexes,))
#printDBG('test_qaids = %r' % test_qaids[0:5])
_test_qaids = [test_qaids[xx] for xx in indexes]
test_qaids = _test_qaids
#printDBG('test_qaids = %r' % test_qaids)
elif len(test_qaids) == 0 and len(valid_aids) > 0:
#printDBG('no hard or gt aids. Defaulting to the first ANNOTATION')
test_qaids = valid_aids[0:1]
#print('test_qaids = %r' % test_qaids)
test_qaids = utool.unique_keep_order2(test_qaids)
return test_qaids
def append_annot_case_tags(ibs, aid_list, tag_list):
"""
Generally appends tags to annotations. Careful not to introduce too many
random tags. Maybe we should just let that happen and introduce tag-aliases
Note: this is more of a set add rather than a list append
TODO: remove
"""
# Ensure each item is a list
#tags_list = [tag if isinstance(tag, list) else [tag] for tag in tag_list]
if isinstance(tag_list, six.string_types):
# Apply single tag to everybody
tag_list = [tag_list] * len(aid_list)
tags_list = [ut.ensure_iterable(tag) for tag in tag_list]
text_list = ibs.get_annot_tag_text(aid_list)
orig_tags_list = [[] if note is None else _parse_tags(note) for note in text_list]
new_tags_list = [ut.unique(t1 + t2) for t1, t2 in zip(tags_list, orig_tags_list)]
ibs.set_annot_case_tags(aid_list, new_tags_list)
def matches(ibs, ams=None, edges=None, uuid_edges=None, **kwargs):
""" Makes an Annots object """
if uuid_edges is not None:
assert ams is None, 'specify one primary key'
assert edges is None, 'specify one primary key'
uuids1, uuids2 = list(zip(*uuid_edges))
aids1 = ibs.get_annot_aids_from_uuid(uuids1)
aids2 = ibs.get_annot_aids_from_uuid(uuids2)
ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2)
if edges is not None:
assert ams is None, 'specify one primary key'
assert uuid_edges is None, 'specify one primary key'
aids1, aids2 = list(zip(*edges))
ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2)
if ams is None:
ams = ibs._get_all_annotmatch_rowids()
elif ams.__class__.__name__ == 'AnnotMatches':
return ams
ams = ut.ensure_iterable(ams)
return AnnotMatches(ams, ibs, **kwargs)
def issue(repo, command, sudo=False):
"""
issues a command on a repo
CommandLine:
python -m utool.util_git --exec-repocmd
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True))
>>> command = 'git status'
>>> sudo = False
>>> result = repocmd(repo, command, sudo)
>>> print(result)
"""
import utool as ut
if ut.WIN32:
assert not sudo, "cant sudo on windows"
command_list = ut.ensure_iterable(command)
cmdstr = "\n ".join([cmd_ for cmd_ in command_list])
print("+--- *** repocmd(%s) *** " % (cmdstr,))
print("repo=%s" % ut.color_text(repo.dpath, "yellow"))
with ut.ChdirContext(repo.dpath, verbose=False):
ret = None
for count, cmd in enumerate(command_list):
if not sudo or ut.WIN32:
ret = os.system(cmd)
else:
out, err, ret = ut.cmd(cmd, sudo=True)
verbose = True
if verbose > 1:
print("ret(%d) = %r" % (count, ret))
if ret != 0:
raise Exception("Failed command %r" % (cmd,))
print("L____")
def issue(repo, command, sudo=False):
"""
issues a command on a repo
CommandLine:
python -m utool.util_git --exec-repocmd
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True))
>>> command = 'git status'
>>> sudo = False
>>> result = repocmd(repo, command, sudo)
>>> print(result)
"""
import utool as ut
if ut.WIN32:
assert not sudo, 'cant sudo on windows'
command_list = ut.ensure_iterable(command)
cmdstr = '\n '.join([cmd_ for cmd_ in command_list])
print('+--- *** repocmd(%s) *** ' % (cmdstr,))
print('repo=%s' % ut.color_text(repo.dpath, 'yellow'))
with ut.ChdirContext(repo.dpath, verbose=False):
ret = None
for count, cmd in enumerate(command_list):
if not sudo or ut.WIN32:
ret = os.system(cmd)
else:
out, err, ret = ut.cmd(cmd, sudo=True)
verbose = True
if verbose > 1:
print('ret(%d) = %r' % (count, ret,))
if ret != 0:
raise Exception('Failed command %r' % (cmd,))
print('L____')
def build_alias_map(regex_map, tag_vocab):
"""
Constructs explicit mapping. Order of items in regex map matters.
Items at top are given preference.
Example:
>>> tags_list = [['t1', 't2'], [], ['t3'], ['t4', 't5']]
>>> tag_vocab = ut.flat_unique(*tags_list)
>>> regex_map = [('t[3-4]', 'A9'), ('t0', 'a0')]
>>> unmapped = list(set(tag_vocab) - set(alias_map.keys()))
"""
import utool as ut
import re
alias_map = ut.odict([])
for pats, new_tag in reversed(regex_map):
pats = ut.ensure_iterable(pats)
for pat in pats:
flags = [re.match(pat, t) for t in tag_vocab]
for old_tag in ut.compress(tag_vocab, flags):
alias_map[old_tag] = new_tag
identity_map = ut.take_column(regex_map, 1)
for tag in ut.filter_Nones(identity_map):
alias_map[tag] = tag
return alias_map
def make_class_images(dream, target_labels):
import ibeis_cnn.__THEANO__ as theano
from ibeis_cnn.__THEANO__ import tensor as T # NOQA
import utool as ut
was_scalar = not ut.isiterable(target_labels)
target_labels = ut.ensure_iterable(target_labels)
if True:
# We are forcing a batch size for this visualization
input_shape = (len(target_labels),) + dream.model.input_shape[1:]
else:
# Maybe some cnn layers cant take variable batches?
input_shape = dream.model.input_shape
b, c, w, h = input_shape
assert len(target_labels) <= b, 'batch size too small'
initial_state = dream._make_init_state()
# make image a shared variable that you can update
if dream.shared_images is None:
dream.shared_images = theano.shared(initial_state)
else:
dream.shared_images.set_value(initial_state)
if dream.step_fn is None:
dream.step_fn = dream._make_objective(dream.shared_images, target_labels)
# Optimize objective via backpropogation for a few iterations
for _ in ut.ProgIter(range(dream.niters), lbl='making class model img',
bs=True):
dream.step_fn()
#print('objective = %r' % (objective,))
out = dream._postprocess_class_image(dream.shared_images, target_labels, was_scalar)
return out
def export(ibs, aid_pairs=None):
"""
3 - 4 different animals
2 views of each
matching keypoint coordinates on each annotation
"""
if aid_pairs is None:
if ibs.get_dbname() == 'PZ_MOTHERS':
aid_pair_list = MOTHERS_VIEWPOINT_EXPORT_PAIRS
if ibs.get_dbname() == 'GZ_ALL':
aid_pair_list = GZ_VIEWPOINT_EXPORT_PAIRS
ibs.update_query_cfg(ratio_thresh=1.6)
export_path = expanduser('~/Dropbox/Assignments/dataset')
#utool.view_directory(export_path)
# MOTHERS EG:
for aid_pair in aid_pair_list:
cm_list, qreq_ = ibs.query_chips(aid_pair, aid_pair)
#ibeis.viz.show_qres(ibs, qaid2_qres.values()[1]); df2.iup()
mrids_list = []
mkpts_list = []
for cm in cm_list:
qaid = cm.qaid
print('Getting kpts from %r' % qaid)
#cm.show_top(ibs)
posrid_list = utool.ensure_iterable(cm.get_classified_pos())
mrids_list.extend([(qaid, posrid) for posrid in posrid_list])
mkpts_list.extend(cm.get_matching_keypoints(ibs, posrid_list))
mkey2_kpts = {}
for mrids_tup, mkpts_tup in zip(mrids_list, mkpts_list):
assert len(mrids_tup) == 2, 'must be a match tuple'
mrids_ = np.array(mrids_tup)
sortx = mrids_.argsort()
mrids_ = mrids_[sortx]
mkpts_ = np.array(mkpts_tup)[sortx]
if sortx[0] == 0:
pass
mkey = tuple(mrids_.tolist())
try:
kpts_list = mkey2_kpts[mkey]
print('append to mkey=%r' % (mkey, ))
except KeyError:
print('new mkey=%r' % (mkey, ))
kpts_list = []
kpts_list.append(mkpts_)
mkey2_kpts[mkey] = kpts_list
mkeys_list = mkey2_kpts.keys()
mkeys_keypoints = mkey2_kpts.values()
for mkeys, mkpts_list in zip(mkeys_list, mkeys_keypoints):
print(mkeys)
print(len(kpts_list))
kpts1_m = np.vstack([mkpts[0] for mkpts in mkpts_list])
kpts2_m = np.vstack([mkpts[1] for mkpts in mkpts_list])
match_lines = [
repr((
tuple(kp1[ktool.LOC_DIMS].tolist()),
tuple(kp2[ktool.LOC_DIMS].tolist()),
)) + ', ' for kp1, kp2 in zip(kpts1_m, kpts2_m)
]
mcpaths_list = ibs.get_annot_chip_fpath(mkeys)
fnames_list = list(map(lambda x: split(x)[1], mcpaths_list))
for path in mcpaths_list:
utool.copy(path, export_path)
header_lines = [
'# Exported keypoint matches (might be duplicates matches)',
'# matching_aids = %r' % (mkey, )
]
header_lines += [
'# img%d = %r' % (count, fname)
for count, fname in enumerate(fnames_list)
]
header_lines += [
'# LINE FORMAT: match_pts = [(img1_xy, img2_xy) ... ]'
]
header_text = '\n'.join(header_lines)
match_text = '\n'.join(['match_pts = ['] + match_lines + [']'])
matchfile_text = '\n'.join([header_text, match_text])
matchfile_name = ('match_aids(%d,%d).txt' % mkey)
matchfile_path = join(export_path, matchfile_name)
utool.write_to(matchfile_path, matchfile_text)
print(header_text)
print(utool.truncate_str(match_text, maxlen=500))
def execute_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
flags_ = flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
def filterflags_general_tags(
tags_list,
has_any=None,
has_all=None,
has_none=None,
min_num=None,
max_num=None,
any_startswith=None,
any_endswith=None,
any_match=None,
none_match=None,
logic='and',
):
r"""
maybe integrate into utool? Seems pretty general
Args:
tags_list (list):
has_any (None): (default = None)
has_all (None): (default = None)
min_num (None): (default = None)
max_num (None): (default = None)
CommandLine:
python -m wbia.tag_funcs --exec-filterflags_general_tags
python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --helpx
python -m wbia.tag_funcs --exec-filterflags_general_tags:0
python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --none_match n
python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --has_none=n,o
python -m wbia.tag_funcs --exec-filterflags_general_tags:1
python -m wbia.tag_funcs --exec-filterflags_general_tags:2
Example0:
>>> # DISABLE_DOCTEST
>>> from wbia.tag_funcs import * # NOQA
>>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']]
>>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list)
>>> print('kwargs = %r' % (kwargs,))
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> print(flags)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example1:
>>> # ENABLE_DOCTEST
>>> from wbia.tag_funcs import * # NOQA
>>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']]
>>> has_all = 'n'
>>> min_num = 1
>>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example2:
>>> # ENABLE_DOCTEST
>>> from wbia.tag_funcs import * # NOQA
>>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']]
>>> kwargs = {
>>> 'any_endswith': 'n',
>>> 'any_match': None,
>>> 'any_startswith': 'n',
>>> 'has_all': None,
>>> 'has_any': None,
>>> 'has_none': None,
>>> 'max_num': 3,
>>> 'min_num': 1,
>>> 'none_match': ['P'],
>>> }
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> filtered = ut.compress(tags_list, flags)
>>> result = ('result = %s' % (ut.repr2(filtered),))
result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']]
"""
import re
import operator
def fix_tags(tags):
return {six.text_type(t.lower()) for t in tags}
if logic is None:
logic = 'and'
logic_func = {'and': np.logical_and, 'or': np.logical_or}[logic]
default_func = {'and': np.ones, 'or': np.zeros}[logic]
tags_list_ = [fix_tags(tags_) for tags_ in tags_list]
flags = default_func(len(tags_list_), dtype=np.bool)
if min_num is not None:
flags_ = [len(tags_) >= min_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if max_num is not None:
flags_ = [len(tags_) <= max_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_any is not None:
has_any = fix_tags(set(ut.ensure_iterable(has_any)))
flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_none is not None:
has_none = fix_tags(set(ut.ensure_iterable(has_none)))
flags_ = [
len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_
]
logic_func(flags, flags_, out=flags)
if has_all is not None:
has_all = fix_tags(set(ut.ensure_iterable(has_all)))
flags_ = [
len(has_all.intersection(tags_)) == len(has_all)
for tags_ in tags_list_
]
logic_func(flags, flags_, out=flags)
def check_item(tags_, fields, op, compare):
t_flags = [any([compare(t, f) for f in fields]) for t in tags_]
num_passed = sum(t_flags)
flag = op(num_passed, 0)
return flag
def flag_tags(tags_list, fields, op, compare):
flags = [
check_item(tags_, fields, op, compare) for tags_ in tags_list_
]
return flags
def execute_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
flags_ = flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
flags = execute_filter(flags, tags_list, any_startswith, operator.gt,
six.text_type.startswith)
flags = execute_filter(flags, tags_list, any_endswith, operator.gt,
six.text_type.endswith)
flags = execute_filter(flags, tags_list, any_match, operator.gt,
lambda t, f: re.match(f, t))
flags = execute_filter(flags, tags_list, none_match, operator.eq,
lambda t, f: re.match(f, t))
return flags
def execute_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
flags_ = flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
def filterflags_general_tags(tags_list,
has_any=None,
has_all=None,
has_none=None,
min_num=None,
max_num=None,
any_startswith=None,
any_endswith=None,
any_match=None,
none_match=None,
logic='and'):
r"""
maybe integrate into utool? Seems pretty general
Args:
tags_list (list):
has_any (None): (default = None)
has_all (None): (default = None)
min_num (None): (default = None)
max_num (None): (default = None)
CommandLine:
python -m ibeis.tag_funcs --exec-filterflags_general_tags
python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --helpx
python -m ibeis.tag_funcs --exec-filterflags_general_tags:0
python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --none_match n
python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --has_none=n,o
python -m ibeis.tag_funcs --exec-filterflags_general_tags:1
python -m ibeis.tag_funcs --exec-filterflags_general_tags:2
Example0:
>>> # DISABLE_DOCTEST
>>> from ibeis.tag_funcs import * # NOQA
>>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']]
>>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list)
>>> print('kwargs = %r' % (kwargs,))
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> print(flags)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example1:
>>> # ENABLE_DOCTEST
>>> from ibeis.tag_funcs import * # NOQA
>>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']]
>>> has_all = 'n'
>>> min_num = 1
>>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example2:
>>> # ENABLE_DOCTEST
>>> from ibeis.tag_funcs import * # NOQA
>>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']]
>>> kwargs = {
>>> 'any_endswith': 'n',
>>> 'any_match': None,
>>> 'any_startswith': 'n',
>>> 'has_all': None,
>>> 'has_any': None,
>>> 'has_none': None,
>>> 'max_num': 3,
>>> 'min_num': 1,
>>> 'none_match': ['P'],
>>> }
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> filtered = ut.compress(tags_list, flags)
>>> result = ('result = %s' % (ut.repr2(filtered),))
result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']]
"""
import re
import operator
from ibeis import constants as const
def fix_tags(tags):
from ibeis import constants as const
return {const.__STR__(t.lower()) for t in tags}
if logic is None:
logic = 'and'
logic_func = {
'and': np.logical_and,
'or': np.logical_or,
}[logic]
default_func = {
'and': np.ones,
'or': np.zeros,
}[logic]
tags_list_ = [fix_tags(tags_) for tags_ in tags_list]
flags = default_func(len(tags_list_), dtype=np.bool)
if min_num is not None:
flags_ = [len(tags_) >= min_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if max_num is not None:
flags_ = [len(tags_) <= max_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_any is not None:
has_any = fix_tags(set(ut.ensure_iterable(has_any)))
flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_none is not None:
has_none = fix_tags(set(ut.ensure_iterable(has_none)))
flags_ = [len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_all is not None:
has_all = fix_tags(set(ut.ensure_iterable(has_all)))
flags_ = [len(has_all.intersection(tags_)) == len(has_all) for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
def test_item(tags_, fields, op, compare):
t_flags = [any([compare(t, f) for f in fields]) for t in tags_]
num_passed = sum(t_flags)
flag = op(num_passed, 0)
return flag
def flag_tags(tags_list, fields, op, compare):
flags = [test_item(tags_, fields, op, compare) for tags_ in tags_list_]
return flags
def execute_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
flags_ = flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
flags = execute_filter(
flags, tags_list, any_startswith,
operator.gt, const.__STR__.startswith)
flags = execute_filter(
flags, tags_list, any_endswith,
operator.gt, const.__STR__.endswith)
flags = execute_filter(
flags, tags_list, any_match,
operator.gt, lambda t, f: re.match(f, t))
flags = execute_filter(
flags, tags_list, none_match,
operator.eq, lambda t, f: re.match(f, t))
return flags
def demodata_infr(**kwargs):
"""
kwargs = {}
CommandLine:
python -m wbia.algo.graph.demo demodata_infr --show
python -m wbia.algo.graph.demo demodata_infr --num_pccs=25
python -m wbia.algo.graph.demo demodata_infr --profile --num_pccs=100
Ignore:
>>> from wbia.algo.graph.demo import * # NOQA
>>> from wbia.algo.graph import demo
>>> import networkx as nx
>>> kwargs = dict(num_pccs=6, p_incon=.5, size_std=2)
>>> kwargs = ut.argparse_dict(kwargs)
>>> infr = demo.demodata_infr(**kwargs)
>>> pccs = list(infr.positive_components())
>>> assert len(pccs) == kwargs['num_pccs']
>>> nonfull_pccs = [cc for cc in pccs if len(cc) > 1 and nx.is_empty(nx.complement(infr.pos_graph.subgraph(cc)))]
>>> expected_n_incon = len(nonfull_pccs) * kwargs['p_incon']
>>> n_incon = len(list(infr.inconsistent_components()))
>>> # TODO can test that we our sample num incon agrees with pop mean
>>> #sample_mean = n_incon / len(nonfull_pccs)
>>> #pop_mean = kwargs['p_incon']
>>> print('status = ' + ut.repr4(infr.status(extended=True)))
>>> ut.quit_if_noshow()
>>> infr.show(pickable=True, groupby='name_label')
>>> ut.show_if_requested()
Ignore:
kwargs = {
'ccs': [[1, 2, 3], [4, 5]]
}
"""
import networkx as nx
import vtool as vt
from wbia.algo.graph import nx_utils
def kwalias(*args):
params = args[0:-1]
default = args[-1]
for key in params:
if key in kwargs:
return kwargs[key]
return default
num_pccs = kwalias('num_pccs', 16)
size_mean = kwalias('pcc_size_mean', 'pcc_size', 'size', 5)
size_std = kwalias('pcc_size_std', 'size_std', 0)
# p_pcc_incon = kwargs.get('p_incon', .1)
p_pcc_incon = kwargs.get('p_incon', 0)
p_pcc_incomp = kwargs.get('p_incomp', 0)
pcc_sizes = kwalias('pcc_sizes', None)
pos_redun = kwalias('pos_redun', [1, 2, 3])
pos_redun = ut.ensure_iterable(pos_redun)
# number of maximum inconsistent edges per pcc
max_n_incon = kwargs.get('n_incon', 3)
rng = np.random.RandomState(0)
counter = 1
if pcc_sizes is None:
pcc_sizes = [
int(randn(size_mean, size_std, rng=rng, a_min=1))
for _ in range(num_pccs)
]
else:
num_pccs = len(pcc_sizes)
if 'ccs' in kwargs:
# Overwrites other options
pcc_sizes = list(map(len, kwargs['ccs']))
num_pccs = len(pcc_sizes)
size_mean = None
size_std = 0
new_ccs = []
pcc_iter = list(enumerate(pcc_sizes))
pcc_iter = ut.ProgIter(pcc_iter,
enabled=num_pccs > 20,
label='make pos-demo')
for i, size in pcc_iter:
p = 0.1
want_connectivity = rng.choice(pos_redun)
want_connectivity = min(size - 1, want_connectivity)
# Create basic graph of positive edges with desired connectivity
g = nx_utils.random_k_edge_connected_graph(size,
k=want_connectivity,
p=p,
rng=rng)
nx.set_edge_attributes(g, name='evidence_decision', values=POSTV)
nx.set_edge_attributes(g, name='truth', values=POSTV)
# nx.set_node_attributes(g, name='orig_name_label', values=i)
assert nx.is_connected(g)
# Relabel graph with non-conflicting names
if 'ccs' in kwargs:
g = nx.relabel_nodes(g, dict(enumerate(kwargs['ccs'][i])))
else:
# Make sure nodes do not conflict with others
g = nx.relabel_nodes(
g, dict(enumerate(range(counter,
len(g) + counter + 1))))
counter += len(g)
# The probability any edge is inconsistent is `p_incon`
# This is 1 - P(all edges consistent)
# which means p(edge is consistent) = (1 - p_incon) / N
complement_edges = ut.estarmap(nx_utils.e_,
nx_utils.complement_edges(g))
if len(complement_edges) > 0:
# compute probability that any particular edge is inconsistent
# to achieve probability the PCC is inconsistent
p_edge_inconn = 1 - (1 - p_pcc_incon)**(1 / len(complement_edges))
p_edge_unrev = 0.1
p_edge_notcomp = 1 - (1 - p_pcc_incomp)**(1 /
len(complement_edges))
probs = np.array([p_edge_inconn, p_edge_unrev, p_edge_notcomp])
# if the total probability is greater than 1 the parameters
# are invalid, so we renormalize to "fix" it.
# if probs.sum() > 1:
# warnings.warn('probabilities sum to more than 1')
# probs = probs / probs.sum()
pcumsum = probs.cumsum()
# Determine which mutually exclusive state each complement edge is in
# logger.info('pcumsum = %r' % (pcumsum,))
states = np.searchsorted(pcumsum, rng.rand(len(complement_edges)))
incon_idxs = np.where(states == 0)[0]
if len(incon_idxs) > max_n_incon:
logger.info('max_n_incon = %r' % (max_n_incon, ))
chosen = rng.choice(incon_idxs, max_n_incon, replace=False)
states[np.setdiff1d(incon_idxs, chosen)] = len(probs)
grouped_edges = ut.group_items(complement_edges, states)
for state, edges in grouped_edges.items():
truth = POSTV
if state == 0:
# Add in inconsistent edges
evidence_decision = NEGTV
# TODO: truth could be INCMP or POSTV
# new_edges.append((u, v, {'evidence_decision': NEGTV}))
elif state == 1:
evidence_decision = UNREV
# TODO: truth could be INCMP or POSTV
# new_edges.append((u, v, {'evidence_decision': UNREV}))
elif state == 2:
evidence_decision = INCMP
truth = INCMP
else:
continue
# Add in candidate edges
attrs = {
'evidence_decision': evidence_decision,
'truth': truth
}
for (u, v) in edges:
g.add_edge(u, v, **attrs)
new_ccs.append(g)
# (list(g.nodes()), new_edges))
pos_g = nx.union_all(new_ccs)
assert len(new_ccs) == len(list(nx.connected_components(pos_g)))
assert num_pccs == len(new_ccs)
# Add edges between the PCCS
neg_edges = []
if not kwalias('ignore_pair', False):
logger.info('making pairs')
pair_attrs_lookup = {
0: {
'evidence_decision': NEGTV,
'truth': NEGTV
},
1: {
'evidence_decision': INCMP,
'truth': INCMP
},
2: {
'evidence_decision': UNREV,
'truth': NEGTV
}, # could be incomp or neg
}
# These are the probabilities that one edge has this state
p_pair_neg = kwalias('p_pair_neg', 0.4)
p_pair_incmp = kwalias('p_pair_incmp', 0.2)
p_pair_unrev = kwalias('p_pair_unrev', 0)
# p_pair_neg = 1
cc_combos = ((list(g1.nodes()), list(g2.nodes()))
for (g1, g2) in it.combinations(new_ccs, 2))
valid_cc_combos = [(cc1, cc2) for cc1, cc2 in cc_combos
if len(cc1) and len(cc2)]
for cc1, cc2 in ut.ProgIter(valid_cc_combos, label='make neg-demo'):
possible_edges = ut.estarmap(nx_utils.e_, it.product(cc1, cc2))
# probability that any edge between these PCCs is negative
n_edges = len(possible_edges)
p_edge_neg = 1 - (1 - p_pair_neg)**(1 / n_edges)
p_edge_incmp = 1 - (1 - p_pair_incmp)**(1 / n_edges)
p_edge_unrev = 1 - (1 - p_pair_unrev)**(1 / n_edges)
# Create event space with sizes proportional to probabilities
pcumsum = np.cumsum([p_edge_neg, p_edge_incmp, p_edge_unrev])
# Roll dice for each of the edge to see which state it lands on
possible_pstate = rng.rand(len(possible_edges))
states = np.searchsorted(pcumsum, possible_pstate)
flags = states < len(pcumsum)
stateful_states = states.compress(flags)
stateful_edges = ut.compress(possible_edges, flags)
unique_states, groupxs_list = vt.group_indices(stateful_states)
for state, groupxs in zip(unique_states, groupxs_list):
# logger.info('state = %r' % (state,))
# Add in candidate edges
edges = ut.take(stateful_edges, groupxs)
attrs = pair_attrs_lookup[state]
for (u, v) in edges:
neg_edges.append((u, v, attrs))
logger.info('Made {} neg_edges between PCCS'.format(len(neg_edges)))
else:
logger.info('ignoring pairs')
import wbia
G = wbia.AnnotInference._graph_cls()
G.add_nodes_from(pos_g.nodes(data=True))
G.add_edges_from(pos_g.edges(data=True))
G.add_edges_from(neg_edges)
infr = wbia.AnnotInference.from_netx(G, infer=kwargs.get('infer', True))
infr.verbose = 3
infr.relabel_using_reviews(rectify=False)
# fontname = 'Ubuntu'
fontsize = 12
fontname = 'sans'
splines = 'spline'
# splines = 'ortho'
# splines = 'line'
infr.set_node_attrs('shape', 'circle')
infr.graph.graph['ignore_labels'] = True
infr.graph.graph['dark_background'] = False
infr.graph.graph['fontname'] = fontname
infr.graph.graph['fontsize'] = fontsize
infr.graph.graph['splines'] = splines
infr.set_node_attrs('width', 29)
infr.set_node_attrs('height', 29)
infr.set_node_attrs('fontsize', fontsize)
infr.set_node_attrs('fontname', fontname)
infr.set_node_attrs('fixed_size', True)
# Set synthetic ground-truth attributes for testing
# infr.apply_edge_truth()
infr.edge_truth = infr.get_edge_attrs('truth')
# Make synthetic verif
infr.dummy_verif = DummyVerif(infr)
infr.verifiers = {}
infr.verifiers['match_state'] = infr.dummy_verif
infr.demokw = kwargs
return infr
def types(pubman, want):
want = set(ut.ensure_iterable(want))
for pub in pubman.pub_list:
if pub.type in want:
yield pub
def get_training_featscores(qreq_, cm_list, disttype=None, namemode=True,
fsvx=slice(None, None, None), threshx=None,
thresh=.9, num=None, top_percent=None):
"""
Returns the flattened set of feature scores between each query and the
correct groundtruth annotations as well as the top scoring false
annotations.
Args:
qreq_ (ibeis.QueryRequest): query request object with hyper-parameters
cm_list (list):
disttype (None): (default = None)
namemode (bool): (default = True)
fsvx (slice): (default = slice(None, None, None))
threshx (None): (default = None)
thresh (float): only used if threshx is specified (default = 0.9)
SeeAlso:
TestResult.draw_feat_scoresep
Returns:
tuple: (tp_scores, tn_scores, scorecfg)
CommandLine:
python -m ibeis.algo.hots.scorenorm --exec-get_training_featscores
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.hots.scorenorm import * # NOQA
>>> import ibeis
>>> cm_list, qreq_ = ibeis.testdata_cmlist(defaultdb='PZ_MTEST', a=['default:qsize=10'])
>>> disttype = None
>>> namemode = True
>>> fsvx = None
>>> threshx = 1
>>> thresh = 0.5
>>> (tp_scores, tn_scores, scorecfg) = get_training_featscores(
>>> qreq_, cm_list, disttype, namemode, fsvx, threshx, thresh)
>>> result = scorecfg
>>> print(result)
(lnbnn*fg)[fg > 0.5]
lnbnn*fg[fg > 0.5]
"""
if fsvx is None:
fsvx = slice(None, None, None)
fsv_col_lbls = None
tp_fsvs_list = []
tn_fsvs_list = []
#cm_list = [ cm_list[key] for key in sorted(cm_list.keys()) ]
# Train on only positive examples
trainable = [
qreq_.ibs.get_annot_has_groundtruth(cm.qaid, daid_list=cm.daid_list) and
cm.get_top_nids()[0] == cm.qnid
for cm in cm_list
]
cm_list_ = ut.compress(cm_list, trainable)
print('training using %d chipmatches' % (len(cm_list)))
if disttype is None:
fsv_col_lbls = cm.fsv_col_lbls
train_getter = get_training_fsv
else:
fsv_col_lbls = ut.ensure_iterable(disttype)
# annots = {} # Hack for cached vector lookups
ibs = qreq_.ibs
data_annots = ut.KeyedDefaultDict(ibs.get_annot_lazy_dict, config2_=qreq_.data_config2_)
query_annots = ut.KeyedDefaultDict(ibs.get_annot_lazy_dict, config2_=qreq_.query_config2_)
train_getter = partial(get_training_desc_dist,
fsv_col_lbls=fsv_col_lbls, qreq_=qreq_,
data_annots=data_annots,
query_annots=query_annots)
for cm in ut.ProgIter(cm_list_, lbl='building train featscores',
adjust=True, freq=1):
try:
tp_fsv, tn_fsv = train_getter(
cm, namemode=namemode, top_percent=top_percent)
tp_fsvs_list.extend(tp_fsv)
tn_fsvs_list.extend(tn_fsv)
except UnbalancedExampleException:
continue
fsv_tp = np.vstack(tp_fsvs_list)
fsv_tn = np.vstack(tn_fsvs_list)
fsv_col_lbls_ = ut.take(fsv_col_lbls, fsvx)
fsv_tp_ = fsv_tp.T[fsvx].T
fsv_tn_ = fsv_tn.T[fsvx].T
if threshx is not None:
tp_scores = fsv_tp_[fsv_tp.T[threshx] > thresh].prod(axis=1)
tn_scores = fsv_tn_[fsv_tn.T[threshx] > thresh].prod(axis=1)
threshpart = ('[' + fsv_col_lbls[threshx] + ' > ' + str(thresh) + ']')
scorecfg = '(%s)%s' % ('*'.join(fsv_col_lbls_), threshpart)
else:
tp_scores = fsv_tp_.prod(axis=1)
tn_scores = fsv_tn_.prod(axis=1)
scorecfg = '*'.join(fsv_col_lbls_)
return tp_scores, tn_scores, scorecfg
def add_edge2(u, v, *args, **kwargs):
v = ut.ensure_iterable(v)
u = ut.ensure_iterable(u)
for _u, _v in ut.product(u, v):
graph.add_edge(_u, _v, *args, **kwargs)
def filterflags_general_tags(tags_list, has_any=None, has_all=None,
has_none=None, min_num=None, max_num=None,
any_startswith=None, any_endswith=None,
in_any=None, any_match=None, none_match=None,
logic='and', ignore_case=True):
r"""
maybe integrate into utool? Seems pretty general
Args:
tags_list (list):
has_any (None): (default = None)
has_all (None): (default = None)
min_num (None): (default = None)
max_num (None): (default = None)
Notes:
in_any should probably be ni_any
TODO: make this function more natural
CommandLine:
python -m utool.util_tags --exec-filterflags_general_tags
python -m utool.util_tags --exec-filterflags_general_tags:0 --helpx
python -m utool.util_tags --exec-filterflags_general_tags:0
python -m utool.util_tags --exec-filterflags_general_tags:0 --none_match n
python -m utool.util_tags --exec-filterflags_general_tags:0 --has_none=n,o
python -m utool.util_tags --exec-filterflags_general_tags:1
python -m utool.util_tags --exec-filterflags_general_tags:2
Example0:
>>> # ENABLE_DOCTEST
>>> from utool.util_tags import * # NOQA
>>> import utool as ut
>>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']]
>>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list)
>>> print('kwargs = %r' % (kwargs,))
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> print(flags)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example1:
>>> # ENABLE_DOCTEST
>>> from utool.util_tags import * # NOQA
>>> import utool as ut
>>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']]
>>> has_all = 'n'
>>> min_num = 1
>>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num)
>>> result = ut.compress(tags_list, flags)
>>> print('result = %r' % (result,))
Example2:
>>> # ENABLE_DOCTEST
>>> from utool.util_tags import * # NOQA
>>> import utool as ut
>>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']]
>>> kwargs = {
>>> 'any_endswith': 'n',
>>> 'any_match': None,
>>> 'any_startswith': 'n',
>>> 'has_all': None,
>>> 'has_any': None,
>>> 'has_none': None,
>>> 'max_num': 3,
>>> 'min_num': 1,
>>> 'none_match': ['P'],
>>> }
>>> flags = filterflags_general_tags(tags_list, **kwargs)
>>> filtered = ut.compress(tags_list, flags)
>>> result = ('result = %s' % (ut.repr2(filtered),))
result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']]
"""
import numpy as np
import utool as ut
def _fix_tags(tags):
if ignore_case:
return set([]) if tags is None else {six.text_type(t.lower()) for t in tags}
else:
return set([]) if tags is None else {six.text_type() for t in tags}
if logic is None:
logic = 'and'
logic_func = {
'and': np.logical_and,
'or': np.logical_or,
}[logic]
default_func = {
'and': np.ones,
'or': np.zeros,
}[logic]
tags_list_ = [_fix_tags(tags_) for tags_ in tags_list]
flags = default_func(len(tags_list_), dtype=np.bool)
if min_num is not None:
flags_ = [len(tags_) >= min_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if max_num is not None:
flags_ = [len(tags_) <= max_num for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_any is not None:
has_any = _fix_tags(set(ut.ensure_iterable(has_any)))
flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_none is not None:
has_none = _fix_tags(set(ut.ensure_iterable(has_none)))
flags_ = [len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
if has_all is not None:
has_all = _fix_tags(set(ut.ensure_iterable(has_all)))
flags_ = [len(has_all.intersection(tags_)) == len(has_all) for tags_ in tags_list_]
logic_func(flags, flags_, out=flags)
def _test_item(tags_, fields, op, compare):
t_flags = [any([compare(t, f) for f in fields]) for t in tags_]
num_passed = sum(t_flags)
flag = op(num_passed, 0)
return flag
def _flag_tags(tags_list, fields, op, compare):
flags = [_test_item(tags_, fields, op, compare) for tags_ in tags_list_]
return flags
def _exec_filter(flags, tags_list, fields, op, compare):
if fields is not None:
fields = ut.ensure_iterable(fields)
if ignore_case:
fields = [f.lower() for f in fields]
flags_ = _flag_tags(tags_list, fields, op, compare)
logic_func(flags, flags_, out=flags)
return flags
flags = _exec_filter(
flags, tags_list, any_startswith,
operator.gt, six.text_type.startswith)
flags = _exec_filter(
flags, tags_list, in_any,
operator.gt, operator.contains)
flags = _exec_filter(
flags, tags_list, any_endswith,
operator.gt, six.text_type.endswith)
flags = _exec_filter(
flags, tags_list, any_match,
operator.gt, lambda t, f: re.match(f, t))
flags = _exec_filter(
flags, tags_list, none_match,
operator.eq, lambda t, f: re.match(f, t))
return flags
def _make_rankings(
infr,
qaids=None,
daids=None,
prog_hook=None,
cfgdict=None,
name_method='node',
use_cache=None,
invalidate_supercache=None,
):
# from wbia.algo.graph import graph_iden
# TODO: expose other ranking algos like SMK
rank_algo = 'LNBNN'
infr.print('Exec {} ranking algorithm'.format(rank_algo), 1)
ibs = infr.ibs
if qaids is None:
qaids = infr.aids
qaids = ut.ensure_iterable(qaids)
if daids is None:
daids = infr.aids
if cfgdict is None:
cfgdict = {
# 'can_match_samename': False,
'can_match_samename': True,
'can_match_sameimg': True,
# 'augment_queryside_hack': True,
'K': 3,
'Knorm': 3,
'prescore_method': 'csum',
'score_method': 'csum',
}
cfgdict.update(infr.ranker_params)
infr.print('Using LNBNN config = %r' % (cfgdict,))
# hack for using current nids
if name_method == 'node':
aids = sorted(set(ut.aslist(qaids) + ut.aslist(daids)))
custom_nid_lookup = infr.get_node_attrs('name_label', aids)
elif name_method == 'edge':
custom_nid_lookup = {
aid: nid for nid, cc in infr.pos_graph._ccs.items() for aid in cc
}
elif name_method == 'wbia':
custom_nid_lookup = None
else:
raise KeyError('Unknown name_method={}'.format(name_method))
qreq_ = ibs.new_query_request(
qaids,
daids,
cfgdict=cfgdict,
custom_nid_lookup=custom_nid_lookup,
verbose=infr.verbose >= 2,
)
# cacher = qreq_.get_big_cacher()
# if not cacher.exists():
# pass
# # import sys
# # sys.exit(1)
cm_list = qreq_.execute(
prog_hook=prog_hook,
use_cache=use_cache,
invalidate_supercache=invalidate_supercache,
)
infr._set_vsmany_info(qreq_, cm_list)
edges = set(infr._cm_breaking(cm_list, review_cfg={'ranks_top': 5}))
return edges
def __delitem__(self, index):
import utool as ut
index = ut.ensure_iterable(index)
ut.delete_items_by_index(self.found_fpath_list, index)
ut.delete_items_by_index(self.found_lines_list, index)
ut.delete_items_by_index(self.found_lxs_list, index)
def __delitem__(self, index):
import utool as ut
index = ut.ensure_iterable(index)
ut.delete_items_by_index(self.found_fpath_list, index)
ut.delete_items_by_index(self.found_lines_list, index)
ut.delete_items_by_index(self.found_lxs_list, index)
def export(ibs, aid_pairs=None):
"""
3 - 4 different animals
2 views of each
matching keypoint coordinates on each annotation
"""
if aid_pairs is None:
if ibs.get_dbname() == 'PZ_MOTHERS':
aid_pair_list = MOTHERS_VIEWPOINT_EXPORT_PAIRS
if ibs.get_dbname() == 'GZ_ALL':
aid_pair_list = GZ_VIEWPOINT_EXPORT_PAIRS
ibs.update_query_cfg(ratio_thresh=1.6)
export_path = expanduser('~/Dropbox/Assignments/dataset')
#utool.view_directory(export_path)
# MOTHERS EG:
for aid_pair in aid_pair_list:
qaid2_qres = ibs.query_intra_encounter(aid_pair)
#ibeis.viz.show_qres(ibs, qaid2_qres.values()[1]); df2.iup()
mrids_list = []
mkpts_list = []
for qaid, qres in six.iteritems(qaid2_qres):
print('Getting kpts from %r' % qaid)
#qres.show_top(ibs)
posrid_list = utool.ensure_iterable(qres.get_classified_pos())
mrids_list.extend([(qaid, posrid) for posrid in posrid_list])
mkpts_list.extend(qres.get_matching_keypoints(ibs, posrid_list))
mkey2_kpts = {}
for mrids_tup, mkpts_tup in zip(mrids_list, mkpts_list):
assert len(mrids_tup) == 2, 'must be a match tuple'
mrids_ = np.array(mrids_tup)
sortx = mrids_.argsort()
mrids_ = mrids_[sortx]
mkpts_ = np.array(mkpts_tup)[sortx]
if sortx[0] == 0:
pass
mkey = tuple(mrids_.tolist())
try:
kpts_list = mkey2_kpts[mkey]
print('append to mkey=%r' % (mkey,))
except KeyError:
print('new mkey=%r' % (mkey,))
kpts_list = []
kpts_list.append(mkpts_)
mkey2_kpts[mkey] = kpts_list
mkeys_list = mkey2_kpts.keys()
mkeys_keypoints = mkey2_kpts.values()
for mkeys, mkpts_list in zip(mkeys_list, mkeys_keypoints):
print(mkeys)
print(len(kpts_list))
kpts1_m = np.vstack([mkpts[0] for mkpts in mkpts_list])
kpts2_m = np.vstack([mkpts[1] for mkpts in mkpts_list])
match_lines = [
repr(
(
tuple(kp1[ktool.LOC_DIMS].tolist()),
tuple(kp2[ktool.LOC_DIMS].tolist()),
)
) + ', '
for kp1, kp2 in zip(kpts1_m, kpts2_m)]
mcpaths_list = ibs.get_annot_cpaths(mkeys)
fnames_list = map(lambda x: split(x)[1], mcpaths_list)
for path in mcpaths_list:
utool.copy(path, export_path)
header_lines = ['# Exported keypoint matches (might be duplicates matches)',
'# matching_aids = %r' % (mkey,)]
header_lines += ['# img%d = %r' % (count, fname) for count, fname in enumerate(fnames_list)]
header_lines += ['# LINE FORMAT: match_pts = [(img1_xy, img2_xy) ... ]']
header_text = '\n'.join(header_lines)
match_text = '\n'.join(['match_pts = ['] + match_lines + [']'])
matchfile_text = '\n'.join([header_text, match_text])
matchfile_name = ('match_aids(%d,%d).txt' % mkey)
matchfile_path = join(export_path, matchfile_name)
utool.write_to(matchfile_path, matchfile_text)
print(header_text)
print(utool.truncate_str(match_text, maxlen=500))
def add_edge2(u, v, *args, **kwargs):
v = ut.ensure_iterable(v)
u = ut.ensure_iterable(u)
for _u, _v in ut.product(u, v):
graph.add_edge(_u, _v, *args, **kwargs)
