def get_summary(profile_block_list, maxlines=20):
"""
References:
https://github.com/rkern/line_profiler
"""
time_list = [get_block_totaltime(block) for block in profile_block_list]
time_list = [time if time is not None else -1 for time in time_list]
blockid_list = [get_block_id(block) for block in profile_block_list]
sortx = ut.list_argsort(time_list)
sorted_time_list = ut.take(time_list, sortx)
sorted_blockid_list = ut.take(blockid_list, sortx)
aligned_blockid_list = ut.util_str.align_lines(sorted_blockid_list, ':')
summary_lines = [('%6.2f seconds - ' % time) + line
for time, line in
zip(sorted_time_list, aligned_blockid_list)]
#summary_header = ut.codeblock(
# '''
# CLEANED PROFILE OUPUT
# The Pystone timings are not from kernprof, so they may include kernprof
# overhead, whereas kernprof timings do not (unless the line being
# profiled is also decorated with kernrof)
# The kernprof times are reported in Timer Units
# ''')
summary_lines_ = ut.listclip(summary_lines, maxlines, fromback=True)
summary_text = '\n'.join(summary_lines_)
return summary_text
def testdata_smk(*args, **kwargs):
"""
>>> from wbia.algo.smk.smk_pipeline import * # NOQA
>>> kwargs = {}
"""
import wbia
import sklearn
import sklearn.model_selection
# import sklearn.model_selection
ibs, aid_list = wbia.testdata_aids(defaultdb='PZ_MTEST')
nid_list = np.array(ibs.annots(aid_list).nids)
rng = ut.ensure_rng(0)
xvalkw = dict(n_splits=4, shuffle=False, random_state=rng)
skf = sklearn.model_selection.StratifiedKFold(**xvalkw)
train_idx, test_idx = six.next(skf.split(aid_list, nid_list))
daids = ut.take(aid_list, train_idx)
qaids = ut.take(aid_list, test_idx)
config = {
'num_words': 1000,
}
config.update(**kwargs)
qreq_ = SMKRequest(ibs, qaids, daids, config)
smk = qreq_.smk
# qreq_ = ibs.new_query_request(qaids, daids, cfgdict={'pipeline_root': 'smk', 'proot': 'smk'})
# qreq_ = ibs.new_query_request(qaids, daids, cfgdict={})
return ibs, smk, qreq_
def isect_info(self, other):
set1 = set(self.rel_fpath_list)
set2 = set(other.rel_fpath_list)
set_comparisons = ut.odict([
('s1', set1),
('s2', set2),
('union', set1.union(set2)),
('isect', set1.intersection(set2)),
('s1 - s2', set1.difference(set2)),
('s2 - s1', set1.difference(set1)),
])
stat_stats = ut.map_vals(len, set_comparisons)
print(ut.repr4(stat_stats))
return set_comparisons
if False:
idx_lookup1 = ut.make_index_lookup(self.rel_fpath_list)
idx_lookup2 = ut.make_index_lookup(other.rel_fpath_list)
uuids1 = ut.take(self.uuids,
ut.take(idx_lookup1, set_comparisons['union']))
uuids2 = ut.take(other.uuids,
ut.take(idx_lookup2, set_comparisons['union']))
uuids1 == uuids2
def make_factor_text(factor, name):
collapse_uniform = True
if collapse_uniform and ut.almost_allsame(factor.values):
# Reduce uniform text
ftext = name + ':\nuniform(%.3f)' % (factor.values[0], )
else:
values = factor.values
try:
rowstrs = [
'p(%s)=%.3f' % (
','.join(n),
v,
) for n, v in zip(zip(*factor.statenames), values)
]
except Exception:
rowstrs = [
'p(%s)=%.3f' % (
','.join(n),
v,
) for n, v in zip(factor._row_labels(False), values)
]
idxs = ut.list_argmaxima(values)
for idx in idxs:
rowstrs[idx] += '*'
thresh = 4
always_sort = True
if len(rowstrs) > thresh:
sortx = factor.values.argsort()[::-1]
rowstrs = ut.take(rowstrs, sortx[0:(thresh - 1)])
rowstrs += ['... %d more' % ((len(values) - len(rowstrs)), )]
elif always_sort:
sortx = factor.values.argsort()[::-1]
rowstrs = ut.take(rowstrs, sortx)
ftext = name + ': \n' + '\n'.join(rowstrs)
return ftext
def get_qreq_annot_nids(qreq_, aids):
# uses own internal state to grab name rowids instead of using wbia.
if not hasattr(qreq_, 'aid_to_idx'):
qreq_.ensure_nids()
idxs = ut.take(qreq_.aid_to_idx, aids)
nids = ut.take(qreq_.unique_nids, idxs)
return nids
def show_single_coverage_mask(qreq_, cm, weight_mask_m, weight_mask, daids, fnum=None):
import plottool as pt
from ibeis import viz
fnum = pt.ensure_fnum(fnum)
idx_list = ut.dict_take(cm.daid2_idx, daids)
nPlots = len(idx_list) + 1
nRows, nCols = pt.get_square_row_cols(nPlots)
pnum_ = pt.make_pnum_nextgen(nRows, nCols)
pt.figure(fnum=fnum, pnum=(1, 2, 1))
# Draw coverage masks with bbox
# <FlipHack>
#weight_mask_m = np.fliplr(np.flipud(weight_mask_m))
#weight_mask = np.fliplr(np.flipud(weight_mask))
# </FlipHack>
stacked_weights, offset_tup, sf_tup = vt.stack_images(weight_mask_m, weight_mask, return_sf=True)
(woff, hoff) = offset_tup[1]
wh1 = weight_mask_m.shape[0:2][::-1]
wh2 = weight_mask.shape[0:2][::-1]
pt.imshow(255 * (stacked_weights), fnum=fnum, pnum=pnum_(0), title='(query image) What did match vs what should match')
pt.draw_bbox(( 0, 0) + wh1, bbox_color=(0, 0, 1))
pt.draw_bbox((woff, hoff) + wh2, bbox_color=(0, 0, 1))
# Get contributing matches
qaid = cm.qaid
daid_list = daids
fm_list = ut.take(cm.fm_list, idx_list)
fs_list = ut.take(cm.fs_list, idx_list)
# Draw matches
for px, (daid, fm, fs) in enumerate(zip(daid_list, fm_list, fs_list), start=1):
viz.viz_matches.show_matches2(qreq_.ibs, qaid, daid, fm, fs,
draw_pts=False, draw_lines=True,
draw_ell=False, fnum=fnum, pnum=pnum_(px),
darken=.5)
coverage_score = score_matching_mask(weight_mask_m, weight_mask)
pt.set_figtitle('score=%.4f' % (coverage_score,))
def make_factor_text(factor, name):
collapse_uniform = True
if collapse_uniform and ut.almost_allsame(factor.values):
# Reduce uniform text
ftext = name + ':\nuniform(%.3f)' % (factor.values[0],)
else:
values = factor.values
try:
rowstrs = ['p(%s)=%.3f' % (','.join(n), v,)
for n, v in zip(zip(*factor.statenames), values)]
except Exception:
rowstrs = ['p(%s)=%.3f' % (','.join(n), v,)
for n, v in zip(factor._row_labels(False), values)]
idxs = ut.list_argmaxima(values)
for idx in idxs:
rowstrs[idx] += '*'
thresh = 4
always_sort = True
if len(rowstrs) > thresh:
sortx = factor.values.argsort()[::-1]
rowstrs = ut.take(rowstrs, sortx[0:(thresh - 1)])
rowstrs += ['... %d more' % ((len(values) - len(rowstrs)),)]
elif always_sort:
sortx = factor.values.argsort()[::-1]
rowstrs = ut.take(rowstrs, sortx)
ftext = name + ': \n' + '\n'.join(rowstrs)
return ftext
def phis_flags_list(X, idxs):
""" get subset of non-aggregated residual vectors """
phis_list = ut.take(X.rvecs_list, idxs)
flags_list = ut.take(X.flags_list, idxs)
if X.int_rvec:
phis_list = ut.lmap(smk_funcs.uncast_residual_integer, phis_list)
return phis_list, flags_list
def unnest_data(data):
unnested_cols = list(zip(ut.take(data, idxs2)))
nested_cols = ut.take(data, idxs1)
grouped_items = [nested_cols, unnested_cols]
groupxs = [idxs1, idxs2]
unflat = ut.ungroup(grouped_items, groupxs, nested_nCols - 1)
return tuple(ut.flatten(unflat))
def get_summary(profile_block_list, maxlines=20):
"""
References:
https://github.com/rkern/line_profiler
"""
time_list = [get_block_totaltime(block) for block in profile_block_list]
time_list = [time if time is not None else -1 for time in time_list]
blockid_list = [get_block_id(block) for block in profile_block_list]
sortx = ut.list_argsort(time_list)
sorted_time_list = ut.take(time_list, sortx)
sorted_blockid_list = ut.take(blockid_list, sortx)
aligned_blockid_list = ut.util_str.align_lines(sorted_blockid_list, ':')
summary_lines = [('%6.2f seconds - ' % time) + line
for time, line in
zip(sorted_time_list, aligned_blockid_list)]
#summary_header = ut.codeblock(
# '''
# CLEANED PROFILE OUPUT
# The Pystone timings are not from kernprof, so they may include kernprof
# overhead, whereas kernprof timings do not (unless the line being
# profiled is also decorated with kernrof)
# The kernprof times are reported in Timer Units
# ''')
summary_lines_ = ut.listclip(summary_lines, maxlines, fromback=True)
summary_text = '\n'.join(summary_lines_)
return summary_text
def get_topname_training_idxs(cm, num=5):
"""
gets the index of the annots in the top groundtrue name and the top
groundfalse names.
Args:
cm (ibeis.ChipMatch): object of feature correspondences and scores
num(int): number of false names (default = 5)
Returns:
tuple: (tp_idxs, tn_idxs)
cm.daid_list[tp_idxs] are all of the
annotations in the correct name.
cm.daid_list[tn_idxs] are all of the
annotations in the top `num_false` incorrect names.
CommandLine:
python -m ibeis --tf get_topname_training_idxs --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.hots.scorenorm import * # NOQA
>>> import ibeis
>>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best')
>>> num = 1
>>> (tp_idxs, tn_idxs) = get_topname_training_idxs(cm, num)
>>> result = ('(tp_idxs, tn_idxs) = %s' % (ut.repr2((tp_idxs, tn_idxs), nl=1),))
>>> print(result)
(tp_idxs, tn_idxs) = (
np.array([0, 1, 2], dtype=np.int64),
[3, 4, 5, 6],
)
"""
if num is None:
num = 5
sortx = cm.name_argsort()
sorted_nids = vt.take2(cm.unique_nids, sortx)
sorted_groupxs = ut.take(cm.name_groupxs, sortx)
# name ranks of the groundtrue name
tp_ranks = np.where(sorted_nids == cm.qnid)[0]
if len(tp_ranks) == 0:
#if ut.STRICT:
# raise Exception('tp_ranks=0')
#else:
raise UnbalancedExampleException('tp_ranks=0')
# name ranks of the top groundfalse names
tp_rank = tp_ranks[0]
tn_ranks = [rank for rank in range(num + 1)
if rank != tp_rank and rank < len(sorted_groupxs)]
if len(tn_ranks) == 0:
#if ut.STRICT:
# raise Exception('tn_ranks=0')
#else:
raise UnbalancedExampleException('tn_ranks=0')
# annot idxs of the examples
tp_idxs = sorted_groupxs[tp_rank]
tn_idxs = ut.flatten(ut.take(sorted_groupxs, tn_ranks))
return tp_idxs, tn_idxs
def biggest_files(drive):
print('Biggest Files in %r' % (drive,))
sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1]
sel = sortx[0:10]
biggest_nbytes = ut.take(drive.fpath_bytes_list, sel)
biggest_files = ut.take(drive.fpath_list, sel)
biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_files))
print(ut.list_str(biginfo_list, strvals=True))
def permute_columns(self, new_order, inplace=True):
import utool as ut
self.header = ut.take(self.header, new_order)
self.header_tags = ut.take(self.header_tags, new_order)
self.row_data = ut.listT(ut.take(ut.listT(self.row_data), new_order))
if self.short_header is not None:
self.short_header = ut.take(self.short_header, new_order)
return self
def permute_columns(self, new_order, inplace=True):
import utool as ut
self.header = ut.take(self.header, new_order)
self.header_tags = ut.take(self.header_tags, new_order)
self.row_data = ut.listT(ut.take(ut.listT(self.row_data), new_order))
if self.short_header is not None:
self.short_header = ut.take(self.short_header, new_order)
return self
def bow_vector(X, wx_to_weight, nwords):
import vtool as vt
wxs = sorted(list(X.wx_set))
tf = np.array(ut.take(X.termfreq, wxs))
idf = np.array(ut.take(wx_to_weight, wxs))
bow_ = tf * idf
bow_ = vt.normalize(bow_)
bow = SparseVector(dict(zip(wxs, bow_)))
X.bow = bow
def biggest_files(drive):
print('Biggest Files in %r' % (drive, ))
sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1]
sel = sortx[0:10]
biggest_nbytes = ut.take(drive.fpath_bytes_list, sel)
biggest_files = ut.take(drive.fpath_list, sel)
biginfo_list = list(
zip(map(ut.byte_str2, biggest_nbytes), biggest_files))
print(ut.repr2(biginfo_list, strvals=True))
def residual_args(vocab, vecs, nAssign, int_rvec):
fx_to_vecs = vecs
fx_to_wxs, fx_to_maws = smk_funcs.assign_to_words(vocab, fx_to_vecs, nAssign)
wx_to_fxs, wx_to_maws = smk_funcs.invert_assigns(fx_to_wxs, fx_to_maws)
wx_list = sorted(wx_to_fxs.keys())
word_list = ut.take(vocab.wx_to_word, wx_list)
fxs_list = ut.take(wx_to_fxs, wx_list)
maws_list = ut.take(wx_to_maws, wx_list)
argtup = (wx_list, word_list, fxs_list, maws_list, fx_to_vecs, int_rvec)
return argtup
def get_name_shortlist_aids(
daid_list,
dnid_list,
annot_score_list,
name_score_list,
nid2_nidx,
nNameShortList,
nAnnotPerName,
):
r"""
CommandLine:
python -m wbia.algo.hots.scoring --test-get_name_shortlist_aids
Example:
>>> # ENABLE_DOCTEST
>>> from wbia.algo.hots.scoring import * # NOQA
>>> daid_list = np.array([11, 12, 13, 14, 15, 16, 17])
>>> dnid_list = np.array([21, 21, 21, 22, 22, 23, 24])
>>> annot_score_list = np.array([ 6, 2, 3, 5, 6, 3, 2])
>>> name_score_list = np.array([ 8, 9, 5, 4])
>>> nid2_nidx = {21:0, 22:1, 23:2, 24:3}
>>> nNameShortList, nAnnotPerName = 3, 2
>>> args = (daid_list, dnid_list, annot_score_list, name_score_list,
... nid2_nidx, nNameShortList, nAnnotPerName)
>>> top_daids = get_name_shortlist_aids(*args)
>>> result = str(top_daids)
>>> print(result)
[15, 14, 11, 13, 16]
"""
unique_nids, groupxs = vt.group_indices(np.array(dnid_list))
grouped_annot_scores = vt.apply_grouping(annot_score_list, groupxs)
grouped_daids = vt.apply_grouping(np.array(daid_list), groupxs)
# Ensure name score list is aligned with the unique_nids
aligned_name_score_list = name_score_list.take(
ut.dict_take(nid2_nidx, unique_nids))
# Sort each group by the name score
group_sortx = aligned_name_score_list.argsort()[::-1]
_top_daid_groups = ut.take(grouped_daids, group_sortx)
_top_annot_score_groups = ut.take(grouped_annot_scores, group_sortx)
top_daid_groups = ut.listclip(_top_daid_groups, nNameShortList)
top_annot_score_groups = ut.listclip(_top_annot_score_groups,
nNameShortList)
# Sort within each group by the annotation score
top_daid_sortx_groups = [
annot_score_group.argsort()[::-1]
for annot_score_group in top_annot_score_groups
]
top_sorted_daid_groups = vt.ziptake(top_daid_groups, top_daid_sortx_groups)
top_clipped_daids = [
ut.listclip(sorted_daid_group, nAnnotPerName)
for sorted_daid_group in top_sorted_daid_groups
]
top_daids = ut.flatten(top_clipped_daids)
return top_daids
def show_single_coverage_mask(qreq_,
cm,
weight_mask_m,
weight_mask,
daids,
fnum=None):
import plottool as pt
from ibeis import viz
fnum = pt.ensure_fnum(fnum)
idx_list = ut.dict_take(cm.daid2_idx, daids)
nPlots = len(idx_list) + 1
nRows, nCols = pt.get_square_row_cols(nPlots)
pnum_ = pt.make_pnum_nextgen(nRows, nCols)
pt.figure(fnum=fnum, pnum=(1, 2, 1))
# Draw coverage masks with bbox
# <FlipHack>
#weight_mask_m = np.fliplr(np.flipud(weight_mask_m))
#weight_mask = np.fliplr(np.flipud(weight_mask))
# </FlipHack>
stacked_weights, offset_tup, sf_tup = vt.stack_images(weight_mask_m,
weight_mask,
return_sf=True)
(woff, hoff) = offset_tup[1]
wh1 = weight_mask_m.shape[0:2][::-1]
wh2 = weight_mask.shape[0:2][::-1]
pt.imshow(255 * (stacked_weights),
fnum=fnum,
pnum=pnum_(0),
title='(query image) What did match vs what should match')
pt.draw_bbox((0, 0) + wh1, bbox_color=(0, 0, 1))
pt.draw_bbox((woff, hoff) + wh2, bbox_color=(0, 0, 1))
# Get contributing matches
qaid = cm.qaid
daid_list = daids
fm_list = ut.take(cm.fm_list, idx_list)
fs_list = ut.take(cm.fs_list, idx_list)
# Draw matches
for px, (daid, fm, fs) in enumerate(zip(daid_list, fm_list, fs_list),
start=1):
viz.viz_matches.show_matches2(qreq_.ibs,
qaid,
daid,
fm,
fs,
draw_pts=False,
draw_lines=True,
draw_ell=False,
fnum=fnum,
pnum=pnum_(px),
darken=.5)
coverage_score = score_matching_mask(weight_mask_m, weight_mask)
pt.set_figtitle('score=%.4f' % (coverage_score, ))
def new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec):
wx_to_fxs, wx_to_maws = smk_funcs.invert_assigns(fx_to_wxs, fx_to_maws)
X = inverted_index.SingleAnnot()
X.aid = aid
# Build Aggregate Residual Vectors
X.wx_list = np.array(sorted(wx_to_fxs.keys()), dtype=np.int32)
X.wx_to_idx = ut.make_index_lookup(X.wx_list)
X.int_rvec = int_rvec
X.wx_set = set(X.wx_list)
# TODO: maybe use offset list structure instead of heavy nesting
X.fxs_list = ut.take(wx_to_fxs, X.wx_list)
X.maws_list = ut.take(wx_to_maws, X.wx_list)
return X
def take(self, idxs):
"""
Creates a subset of the list using the specified indices.
"""
rowids = ut.take(self._rowids, idxs)
# Create a new instance pointing only to the requested subset
newself = self.__class__(rowids, ibs=self._ibs, config=self._config,
caching=self._caching)
# Pass along any internally cached values
_new_internal = {key: ut.take(val, idxs)
for key, val in self._internal_attrs.items()}
newself._internal_attrs = _new_internal
return newself
def get_annotmatch_rowid_from_undirected_superkey(ibs, aids1, aids2):
# The directed nature of this makes a few things difficult and may cause
# odd behavior
am_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2)
idxs = ut.where([r is None for r in am_rowids])
# Check which ones are None
aids1_ = ut.take(aids1, idxs)
aids2_ = ut.take(aids2, idxs)
am_rowids_ = ibs.get_annotmatch_rowid_from_superkey(aids2_, aids1_)
# Use the other rowid if found
for idx, rowid in zip(idxs, am_rowids_):
am_rowids[idx] = rowid
return am_rowids
def biggest_dirs(drive):
print('Biggest Dirs in %r' % (drive,))
dpath_list = drive.dpath_list
fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list)
unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list, fidxs_list)
dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list))
sortx = ut.list_argsort(dpath_nbytes_list)[::-1]
sel = sortx[0:10]
biggest_nbytes = ut.take(dpath_nbytes_list, sel)
biggest_dpaths = ut.take(dpath_list, sel)
biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths))
print(ut.list_str(biginfo_list, strvals=True))
pass
def biggest_dirs(drive):
print('Biggest Dirs in %r' % (drive, ))
dpath_list = drive.dpath_list
fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list)
unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list,
fidxs_list)
dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list))
sortx = ut.list_argsort(dpath_nbytes_list)[::-1]
sel = sortx[0:10]
biggest_nbytes = ut.take(dpath_nbytes_list, sel)
biggest_dpaths = ut.take(dpath_list, sel)
biginfo_list = list(
zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths))
print(ut.repr2(biginfo_list, strvals=True))
pass
def predict_proba_df(clf, X_df, class_names=None):
"""
Calls sklearn classifier predict_proba but then puts results in a dataframe
using the same index as X_df and incorporating all possible class_names
given
"""
if class_names is not None:
columns = ut.take(class_names, clf.classes_)
else:
columns = None
if len(X_df) == 0:
return pd.DataFrame(columns=columns)
try:
probs = clf.predict_proba(X_df)
except ValueError:
# solves a problem when values are infinity for whatever reason
X = X_df.values.copy()
X[~np.isfinite(X)] = np.nan
probs = clf.predict_proba(X)
probs_df = pd.DataFrame(probs, columns=columns, index=X_df.index)
# add in zero probability for classes without training data
if class_names is not None:
missing = ut.setdiff(class_names, columns)
if missing:
for classname in missing:
probs_df = probs_df.assign(**{
classname: np.zeros(len(probs_df))})
return probs_df
def get_patches(invassign, wx):
ax_list = invassign.wx2_axs[wx]
fx_list = invassign.wx2_fxs[wx]
config = invassign.fstack.config
ibs = invassign.fstack.ibs
unique_axs, groupxs = vt.group_indices(ax_list)
fxs_groups = vt.apply_grouping(fx_list, groupxs)
unique_aids = ut.take(invassign.fstack.ax2_aid, unique_axs)
all_kpts_list = ibs.depc.d.get_feat_kpts(unique_aids, config=config)
sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0)
chip_list = ibs.depc_annot.d.get_chips_img(unique_aids)
# convert to approprate colorspace
#if colorspace is not None:
# chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
# ut.print_object_size(chip_list, 'chip_list')
patch_size = 64
grouped_patches_list = [
vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0]
for chip, kpts in ut.ProgIter(zip(chip_list, sub_kpts_list),
nTotal=len(unique_aids),
lbl='warping patches')
]
# Make it correspond with original fx_list and ax_list
word_patches = vt.invert_apply_grouping(grouped_patches_list, groupxs)
return word_patches
def flat_compute_rmi_edges(inputs):
"""
Defines order of computation that maps input_ids to target_ids.
CommandLine:
python -m dtool.input_helpers flat_compute_rmi_edges
Returns:
list: compute_edges
Each item is a tuple of input/output RootMostInputs
([parent_1, ..., parent_n], node_i)
All parents should be known before you reach the i-th item in
the list.
Results of the the i-th item may be used in subsequent item
computations.
Example:
>>> from dtool.input_helpers import * # NOQA
>>> from dtool.example_depcache2 import * # NOQA
>>> depc =testdata_custom_annot_depc([
... dict(tablename='chips', parents=['annot']),
... dict(tablename='Notch_Tips', parents=['annot']),
... dict(tablename='Cropped_Chips', parents=['chips', 'Notch_Tips']),
... ])
>>> table = depc['Cropped_Chips']
>>> inputs = exi_inputs = table.rootmost_inputs.total_expand()
>>> compute_rmi_edges = exi_inputs.flat_compute_rmi_edges()
>>> input_rmis = compute_rmi_edges[-1][0]
>>> result = ut.repr2(input_rmis)
>>> print(result)
[chips[t, t:1, 1:1], Notch_Tips[t, t:1, 1:1]]
"""
sink = list(ut.nx_sink_nodes(inputs.exi_graph))[0]
exi_graph = inputs.exi_graph
compute_rmi_edges = []
flat_compute_order = inputs.flat_compute_order()
exi_graph = inputs.exi_graph
for output_node in flat_compute_order:
input_nodes = list(exi_graph.predecessors(output_node))
# input_edges = [(node, output_node, node.branch_id.k) for node in input_nodes]
input_edges = [(node, output_node) for node in input_nodes]
# another sorting strategy. maybe this is correct.
sortx = [
exi_graph.get_edge_data(*e).get('parent_colx')
for e in input_edges
]
input_nodes = ut.take(input_nodes, sortx)
input_rmis = [
RootMostInput(node, sink, exi_graph) for node in input_nodes
]
# input_rmis = sort_rmi_list(input_rmis)
output_rmis = RootMostInput(output_node, sink, exi_graph)
edge = (input_rmis, output_rmis)
compute_rmi_edges.append(edge)
return compute_rmi_edges
def execute_singles(qreq_):
if qreq_.use_single_cache:
qaid_to_hit = _load_singles(qreq_)
else:
qaid_to_hit = {}
hit_all = len(qaid_to_hit) == len(qreq_.qaids)
hit_any = len(qaid_to_hit) > 0
if hit_all:
qaid_to_cm = qaid_to_hit
else:
if hit_any:
logger.info('... partial cm cache hit %d/%d' %
(len(qaid_to_hit), len(qreq_)))
hit_aids = list(qaid_to_hit.keys())
miss_aids = ut.setdiff(qreq_.qaids, hit_aids)
qreq_miss = qreq_.shallowcopy(miss_aids)
else:
qreq_miss = qreq_
# Compute misses
qreq_miss.ensure_data()
qaid_to_cm = execute_and_save(qreq_miss)
# Merge misses with hits
if hit_any:
qaid_to_cm.update(qaid_to_hit)
cm_list = ut.take(qaid_to_cm, qreq_.qaids)
return cm_list
def sort_rmi_list(rmi_list):
"""
CommandLine:
python -m dtool.input_helpers sort_rmi_list
Example:
>>> from dtool.input_helpers import * # NOQA
>>> from dtool.example_depcache2 import * # NOQA
>>> depc =testdata_custom_annot_depc([
... dict(tablename='Notch_Tips', parents=['annot']),
... dict(tablename='chips', parents=['annot']),
... dict(tablename='Cropped_Chips', parents=['chips', 'Notch_Tips']),
... ])
>>> table = depc['Cropped_Chips']
>>> inputs = exi_inputs = table.rootmost_inputs
>>> compute_rmi_edges = exi_inputs.flat_compute_rmi_edges()
>>> input_rmis = compute_rmi_edges[-1][0]
>>> rmi_list = input_rmis[::-1]
>>> rmi_list = sort_rmi_list(rmi_list)
>>> assert rmi_list[0].node[0] == 'chips'
"""
# Order the input rmis via declaration
reverse_compute_branches = [rmi.compute_order()[::-1] for rmi in rmi_list]
# print('rmi_list = %r' % (rmi_list,))
# rmi = rmi_list[0] # hack
# reverse_compute_branches = [path[::-1] for path in nx.all_simple_paths(rmi.exi_graph, rmi.node, rmi.sink)]
sort_keys = [
tuple([r.branch_id.parent_colx for r in rs])
for rs in reverse_compute_branches
]
sortx = ut.argsort(sort_keys)
rmi_list = ut.take(rmi_list, sortx)
return rmi_list
def _cm_breaking(infr, cm_list=None, review_cfg={}):
"""
>>> from wbia.algo.graph.core import * # NOQA
>>> review_cfg = {}
"""
if cm_list is None:
cm_list = infr.cm_list
ranks_top = review_cfg.get('ranks_top', None)
ranks_bot = review_cfg.get('ranks_bot', None)
# Construct K-broken graph
edges = []
if ranks_bot is None:
ranks_bot = 0
for count, cm in enumerate(cm_list):
score_list = cm.annot_score_list
rank_list = ut.argsort(score_list)[::-1]
sortx = ut.argsort(rank_list)
top_sortx = sortx[:ranks_top]
bot_sortx = sortx[len(sortx) - ranks_bot :]
short_sortx = ut.unique(top_sortx + bot_sortx)
daid_list = ut.take(cm.daid_list, short_sortx)
for daid in daid_list:
u, v = (cm.qaid, daid)
if v < u:
u, v = v, u
edges.append((u, v))
return edges
def report_partitioning_statistics(new_reduced_joint):
# compute partitioning statistics
import vtool as vt
vals, idxs = vt.group_indices(new_reduced_joint.values.ravel())
#groupsize = list(map(len, idxs))
#groupassigns = ut.unflat_vecmap(new_reduced_joint.assignment, idxs)
all_states = new_reduced_joint._row_labels(asindex=True)
clusterstats = [tuple(sorted(list(ut.dict_hist(a).values())))
for a in all_states]
grouped_vals = ut.group_items(new_reduced_joint.values.ravel(),
clusterstats)
#probs_assigned_to_clustertype = [(
# sorted(np.unique(np.array(b).round(decimals=5)).tolist())[::-1], a)
# for a, b in grouped_vals.items()]
probs_assigned_to_clustertype = [(
ut.dict_hist(np.array(b).round(decimals=5)), a)
for a, b in grouped_vals.items()]
sortx = ut.argsort([max(c[0].keys())
for c in probs_assigned_to_clustertype])
probs_assigned_to_clustertype = ut.take(probs_assigned_to_clustertype, sortx)
# This list of 2-tuples with the first item being the unique
# probabilies that are assigned to a cluster type along with the number
# of times they were assigned. A cluster type is the second item. Every
# number represents how many annotations were assigned to a specific
# label. The length of that list is the number of total labels. For
# all low scores you will see [[{somenum: 1}, {0: 800}], [1, 1, 1, ... 1]]
# indicating that that the assignment of everyone to a different label happend once
# where the probability was somenum and a 800 times where the probability was 0.
#print(sorted([(b, a) for a, b in ut.map_dict_vals(sum, x)]).items())
#z = sorted([(b, a) for a, b in ut.map_dict_vals(sum, grouped_vals).items()])
print(ut.repr2(probs_assigned_to_clustertype, nl=2, precision=2, sorted_=True))
def report_partitioning_statistics(new_reduced_joint):
# compute partitioning statistics
import vtool as vt
vals, idxs = vt.group_indices(new_reduced_joint.values.ravel())
#groupsize = list(map(len, idxs))
#groupassigns = ut.unflat_vecmap(new_reduced_joint.assignment, idxs)
all_states = new_reduced_joint._row_labels(asindex=True)
clusterstats = [tuple(sorted(list(ut.dict_hist(a).values())))
for a in all_states]
grouped_vals = ut.group_items(new_reduced_joint.values.ravel(),
clusterstats)
#probs_assigned_to_clustertype = [(
# sorted(np.unique(np.array(b).round(decimals=5)).tolist())[::-1], a)
# for a, b in grouped_vals.items()]
probs_assigned_to_clustertype = [(
ut.dict_hist(np.array(b).round(decimals=5)), a)
for a, b in grouped_vals.items()]
sortx = ut.argsort([max(c[0].keys())
for c in probs_assigned_to_clustertype])
probs_assigned_to_clustertype = ut.take(probs_assigned_to_clustertype, sortx)
# This list of 2-tuples with the first item being the unique
# probabilies that are assigned to a cluster type along with the number
# of times they were assigned. A cluster type is the second item. Every
# number represents how many annotations were assigned to a specific
# label. The length of that list is the number of total labels. For
# all low scores you will see [[{somenum: 1}, {0: 800}], [1, 1, 1, ... 1]]
# indicating that that the assignment of everyone to a different label happend once
# where the probability was somenum and a 800 times where the probability was 0.
#print(sorted([(b, a) for a, b in ut.map_dict_vals(sum, x)]).items())
#z = sorted([(b, a) for a, b in ut.map_dict_vals(sum, grouped_vals).items()])
print(ut.repr2(probs_assigned_to_clustertype, nl=2, precision=2, sorted_=True))
def find_requested_hrefs(all_href_list, py_version, pkg_list):
"""
Filters out everything but the requested urls
Returns the urls to download the requested installers
"""
print('Filtering to only requested HREFS')
href_list1, missing1 = filter_href_list(all_href_list, pkg_list,
OS_VERSION, py_version)
#print('missing1 = %r' % (missing1,))
href_list2, missing2 = filter_href_list(all_href_list, missing1,
OS_VERSION, py_version)
#print('missing2 = %r' % (missing2,))
#print(href_list2)
href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64',
py_version.replace('p', 'P'))
#print('missing3 = %r' % (missing3,))
href_list4, missing4 = filter_href_list(
all_href_list, missing3, 'any',
py_version.replace('cp', 'py')[0:3])
if len(missing4) > 0:
print('Could not find a match for missing4=%r' % (missing4, ))
#import Levenshtein
for pkg in missing4:
#dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list]
dist_list = [
0 if (href.find(pkg) > -1) else 100 for href in all_href_list
]
closest_matche_xs = ut.list_argsort(dist_list)[::1]
print('Perhaps pkg=%r could match one of these?' % (pkg, ))
closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3])
print(ut.indentjoin(closest_hrefs, '\n '))
href_list = href_list1 + href_list2 + href_list3 + href_list4
return href_list
def get_patches(invassign, wx):
ax_list = invassign.wx2_axs[wx]
fx_list = invassign.wx2_fxs[wx]
config = invassign.fstack.config
ibs = invassign.fstack.ibs
unique_axs, groupxs = vt.group_indices(ax_list)
fxs_groups = vt.apply_grouping(fx_list, groupxs)
unique_aids = ut.take(invassign.fstack.ax2_aid, unique_axs)
all_kpts_list = ibs.depc.d.get_feat_kpts(unique_aids, config=config)
sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0)
chip_list = ibs.depc_annot.d.get_chips_img(unique_aids)
# convert to approprate colorspace
#if colorspace is not None:
# chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
# ut.print_object_size(chip_list, 'chip_list')
patch_size = 64
grouped_patches_list = [
vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0]
for chip, kpts in ut.ProgIter(zip(chip_list, sub_kpts_list),
nTotal=len(unique_aids),
lbl='warping patches')
]
# Make it correspond with original fx_list and ax_list
word_patches = vt.invert_apply_grouping(grouped_patches_list, groupxs)
return word_patches
def load_ordered_annots(data_uri_order, query_uri_order):
# Open the wbia version of oxford
import wbia
ibs = wbia.opendb('Oxford')
def reorder_annots(_annots, uri_order):
intern_uris = get_annots_imgid(_annots)
lookup = ut.make_index_lookup(intern_uris)
_reordered = _annots.take(ut.take(lookup, uri_order))
return _reordered
# Load database annotations and reorder them to agree with internals
_dannots = ibs.annots(ibs.filter_annots_general(has_none='query'))
data_annots = reorder_annots(_dannots, data_uri_order)
# Load query annototations and reorder to standard order
_qannots = ibs.annots(ibs.filter_annots_general(has_any='query'))
query_annots = reorder_annots(_qannots, query_uri_order)
# Map each query annot to its corresponding data index
dgid_to_dx = ut.make_index_lookup(data_annots.gids)
qx_to_dx = ut.take(dgid_to_dx, query_annots.gids)
return ibs, query_annots, data_annots, qx_to_dx
def find_needsmove_to_other(self, other):
hash1 = self.get_prop('md5_stride')
hash2 = other.get_prop('md5_stride')
idxs1 = list(range(len(hash1)))
hash_to_idxs = ut.group_items(idxs1, hash1)
# Find what we have that other doesnt have and move it there
other_missing = set(hash1).difference(hash2)
missing_idxs1 = ut.flatten(ut.take(hash_to_idxs, other_missing))
data = ut.ColumnLists({
'idx':
missing_idxs1,
'fname':
self.get_prop('fname', missing_idxs1),
'dname':
self.get_prop('dname', missing_idxs1),
'full_path':
self.get_prop('full_path', missing_idxs1),
'nbytes':
self.get_prop('nbytes', missing_idxs1),
})
data = data.compress([f != 'Thumbs.db' for f in data['fname']])
data['ext'] = self.get_prop('ext', data['idx'])
ut.dict_hist(data['ext'])
data.print(ignore=['full_path', 'dname'])
def _heuristic_weighting(nodes, avail_uv):
avail_uv = np.array(avail_uv)
weights = np.ones(len(avail_uv))
if 'view_weight' in enabled_heuristics:
from vtool import _rhomb_dist
view_edge = [(node_to_view[u], node_to_view[v]) for (u, v) in avail_uv]
view_weight = np.array(
[_rhomb_dist.VIEW_CODE_DIST[(v1, v2)] for (v1, v2) in view_edge]
)
# Assume comparable by default and prefer undefined
# more than probably not, but less than definately so.
view_weight[np.isnan(view_weight)] = 1.5
# Prefer viewpoint 10x more than time
weights += 10 * view_weight
if 'time_weight' in enabled_heuristics:
# Prefer linking annotations closer in time
times = ut.take(node_to_time, nodes)
maxtime = vt.safe_max(times, fill=1, nans=False)
mintime = vt.safe_min(times, fill=0, nans=False)
time_denom = maxtime - mintime
# Try linking by time for lynx data
time_delta = np.array(
[abs(node_to_time[u] - node_to_time[v]) for u, v in avail_uv]
)
time_weight = time_delta / time_denom
weights += time_weight
weights = np.array(weights)
weights[np.isnan(weights)] = 1.0
avail = [(u, v, {'weight': w}) for (u, v), w in zip(avail_uv, weights)]
return avail
def duplicates(self):
uuid_to_dupxs = ut.find_duplicate_items(self.uuids)
dup_fpaths = [
ut.take(self.rel_fpath_list, idxs)
for idxs in uuid_to_dupxs.values()
]
return dup_fpaths
def compute_gammas(inva, alpha, thresh):
"""
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.smk.inverted_index import * # NOQA
>>> qreq_, inva = testdata_inva()
>>> inva.wx_to_weight = inva.compute_word_weights('uniform')
>>> alpha = 3.0
>>> thresh = 0.0
>>> gamma_list = inva.compute_gammas(alpha, thresh)
"""
# TODO: sep
wx_to_weight = inva.wx_to_weight
_prog = ut.ProgPartial(length=len(inva.wx_lists),
bs=True,
lbl='gamma',
adjust=True)
_iter = zip(inva.wx_lists, inva.agg_rvecs, inva.agg_flags)
gamma_list = []
for wx_list, phiX_list, flagsX_list in _prog(_iter):
if inva.int_rvec:
phiX_list = smk_funcs.uncast_residual_integer(phiX_list)
weight_list = np.array(ut.take(wx_to_weight, wx_list))
gammaX = smk_funcs.gamma_agg(phiX_list, flagsX_list, weight_list,
alpha, thresh)
gamma_list.append(gammaX)
return gamma_list
def find_requested_hrefs(all_href_list, py_version, pkg_list):
"""
Filters out everything but the requested urls
Returns the urls to download the requested installers
"""
print('Filtering to only requested HREFS')
href_list1, missing1 = filter_href_list(all_href_list, pkg_list, OS_VERSION, py_version)
#print('missing1 = %r' % (missing1,))
href_list2, missing2 = filter_href_list(all_href_list, missing1, OS_VERSION, py_version)
#print('missing2 = %r' % (missing2,))
#print(href_list2)
href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64', py_version.replace('p', 'P'))
#print('missing3 = %r' % (missing3,))
href_list4, missing4 = filter_href_list(all_href_list, missing3, 'any', py_version.replace('cp', 'py')[0:3])
if len(missing4) > 0:
print('Could not find a match for missing4=%r' % (missing4,))
#import Levenshtein
for pkg in missing4:
#dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list]
dist_list = [0 if (href.find(pkg) > -1) else 100 for href in all_href_list]
closest_matche_xs = ut.list_argsort(dist_list)[::1]
print('Perhaps pkg=%r could match one of these?' % (pkg,))
closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3])
print(ut.indentjoin(closest_hrefs, '\n '))
href_list = href_list1 + href_list2 + href_list3 + href_list4
return href_list
def compute_annot_occurrence_ids(ibs, aid_list):
from ibeis.algo.preproc import preproc_occurrence
gid_list = ibs.get_annot_gids(aid_list)
gid2_aids = ut.group_items(aid_list, gid_list)
flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False)
occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()}
return occurid2_aids
def _update_state(model):
import networkx as nx
nodes = sorted(list(model.graph.nodes()))
edges = list(model.graph.edges())
edge2_weights = nx.get_edge_attributes(model.graph, 'weight')
node2_labeling = nx.get_node_attributes(model.graph, 'name_label')
edge_weights = ut.take(edge2_weights, edges)
labeling = ut.take(node2_labeling, nodes)
n_nodes = len(nodes)
# Model state
model.n_nodes = n_nodes
model.edges = edges
model.edge_weights = edge_weights
# Model parameters
model.labeling = np.zeros(model.n_nodes, dtype=np.int32)
model._update_labels(labeling=labeling)
model._update_weights()
def get_total_nbytes(drive, fidx_list=None):
if fidx_list is None:
if drive.fpath_bytes_list_ is None:
return None
if fidx_list is None:
return np.nansum(drive.fpath_bytes_list)
else:
return np.nansum(ut.take(drive.fpath_bytes_list, fidx_list))
def chain_crf():
from pystruct.datasets import load_letters
letters = load_letters()
X, y, folds = ut.take(letters, ['data', 'labels', 'folds'])
X, y = np.array(X), np.array(y)
X_train, X_test = X[folds == 1], X[folds != 1]
y_train, y_test = y[folds == 1], y[folds != 1]
len(X_train)
def get_graph_bounding_box(graph):
import utool as ut
import networkx as nx
import vtool as vt
#nx.get_node_attrs = nx.get_node_attributes
nodes = list(graph.nodes())
pos_list = ut.take(nx.get_node_attributes(graph, 'pos'), nodes)
shape_list = ut.take(nx.get_node_attributes(graph, 'size'), nodes)
node_extents = np.array([
vt.extent_from_bbox(vt.bbox_from_center_wh(xy, wh))
for xy, wh in zip(pos_list, shape_list)
])
tl_x, br_x, tl_y, br_y = node_extents.T
extent = tl_x.min(), br_x.max(), tl_y.min(), br_y.max()
bbox = vt.bbox_from_extent(extent)
return bbox
def get_name_shortlist_aids(daid_list, dnid_list, annot_score_list,
name_score_list, nid2_nidx,
nNameShortList, nAnnotPerName):
r"""
CommandLine:
python -m ibeis.algo.hots.scoring --test-get_name_shortlist_aids
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.hots.scoring import * # NOQA
>>> # build test data
>>> daid_list = np.array([11, 12, 13, 14, 15, 16, 17])
>>> dnid_list = np.array([21, 21, 21, 22, 22, 23, 24])
>>> annot_score_list = np.array([ 6, 2, 3, 5, 6, 3, 2])
>>> name_score_list = np.array([ 8, 9, 5, 4])
>>> nid2_nidx = {21:0, 22:1, 23:2, 24:3}
>>> nNameShortList, nAnnotPerName = 3, 2
>>> # execute function
>>> args = (daid_list, dnid_list, annot_score_list, name_score_list,
... nid2_nidx, nNameShortList, nAnnotPerName)
>>> top_daids = get_name_shortlist_aids(*args)
>>> # verify results
>>> result = str(top_daids)
>>> print(result)
[15, 14, 11, 13, 16]
"""
unique_nids, groupxs = vt.group_indices(np.array(dnid_list))
grouped_annot_scores = vt.apply_grouping(annot_score_list, groupxs)
grouped_daids = vt.apply_grouping(np.array(daid_list), groupxs)
# Ensure name score list is aligned with the unique_nids
aligned_name_score_list = name_score_list.take(ut.dict_take(nid2_nidx, unique_nids))
# Sort each group by the name score
group_sortx = aligned_name_score_list.argsort()[::-1]
_top_daid_groups = ut.take(grouped_daids, group_sortx)
_top_annot_score_groups = ut.take(grouped_annot_scores, group_sortx)
top_daid_groups = ut.listclip(_top_daid_groups, nNameShortList)
top_annot_score_groups = ut.listclip(_top_annot_score_groups, nNameShortList)
# Sort within each group by the annotation score
top_daid_sortx_groups = [annot_score_group.argsort()[::-1]
for annot_score_group in top_annot_score_groups]
top_sorted_daid_groups = vt.ziptake(top_daid_groups, top_daid_sortx_groups)
top_clipped_daids = [ut.listclip(sorted_daid_group, nAnnotPerName)
for sorted_daid_group in top_sorted_daid_groups]
top_daids = ut.flatten(top_clipped_daids)
return top_daids
def show_name(ibs, nid, in_image=True, fnum=0, sel_aids=[], subtitle='',
annote=False, aid_list=None, index_list=None, **kwargs):
r"""
Args:
ibs (IBEISController): ibeis controller object
nid (?):
in_image (bool):
fnum (int): figure number
sel_aids (list):
subtitle (str):
annote (bool):
CommandLine:
python -m ibeis.viz.viz_name --test-show_name --dpath ~/latex/crall-candidacy-2015 --save 'figures/{name}.jpg' --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage --diskshow
python -m ibeis.viz.viz_name --test-show_name --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage --show
python -m ibeis.viz.viz_name --test-show_name --show
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.viz.viz_name import * # NOQA
>>> ibs, nid, in_image, index_list = testdata_showname()
>>> fnum = 0
>>> sel_aids = []
>>> subtitle = ''
>>> annote = False
>>> # execute function
>>> show_name(ibs, nid, in_image, fnum, sel_aids, subtitle, annote, index_list=index_list)
>>> ut.show_if_requested()
"""
print('[viz_name] show_name nid=%r, index_list=%r, aid_list=%r' % (nid, index_list, aid_list))
if aid_list is None:
aid_list = ibs.get_name_aids(nid)
else:
assert ut.list_all_eq_to(ibs.get_annot_nids(aid_list), nid)
if index_list is not None:
aid_list = ut.take(aid_list, index_list)
name = ibs.get_name_texts((nid,))
print('[viz_name] * name=%r aid_list=%r' % (name, aid_list))
show_multiple_chips(ibs, aid_list, in_image=in_image, fnum=fnum,
sel_aids=sel_aids, annote=annote, **kwargs)
if isinstance(nid, np.ndarray):
nid = nid[0]
if isinstance(name, np.ndarray):
name = name[0]
use_figtitle = not ut.get_argflag('--no-figtitle')
if use_figtitle:
figtitle = 'Name View nid=%r name=%r' % (nid, name)
df2.set_figtitle(figtitle)
def level_order(graph):
import utool as ut
node_to_level = ut.nx_dag_node_rank(graph)
#source = ut.nx_source_nodes(graph)[0]
#longest_paths = dict([(target, dag_longest_path(graph, source, target))
# for target in graph.nodes()])
#node_to_level = ut.map_dict_vals(len, longest_paths)
grouped = ut.group_items(node_to_level.keys(), node_to_level.values())
levels = ut.take(grouped, range(1, len(grouped) + 1))
return levels
def make_prob_names(infr):
cm_list = infr.cm_list
# Consolodate information from a series of chip matches
unique_nids = sorted(ut.list_union(*[cm.unique_nids for cm in cm_list]))
#nid2_nidx = ut.make_index_lookup(unique_nids)
# Populate matrix of raw name scores
prob_names = np.zeros((len(cm_list), len(unique_nids)))
for count, cm in enumerate(cm_list):
try:
name_scores = ut.dict_take(cm.nid2_name_score, unique_nids, 0)
except AttributeError:
unique_nidxs = ut.take(cm.nid2_nidx, unique_nids)
name_scores = ut.take(cm.name_score_list, unique_nidxs)
prob_names[count][:] = name_scores
# Normalize to row stochastic matrix
prob_names /= prob_names.sum(axis=1)[:, None]
#print(ut.hz_str('prob_names = ', ut.array2string2(prob_names,
#precision=2, max_line_width=140, suppress_small=True)))
return unique_nids, prob_names
def jagged_stats_info(arr_, lbl, col_lbls):
arr = ut.recursive_replace(arr_, np.inf, np.nan)
# Treat infinite as nan
stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls)
sel_col_lbls = ut.take(col_lbls, sel_indices)
statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True)
stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str
return stat_str, sel_stat_str
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False):
"""
issues a command on a repo
CommandLine:
python -m utool.util_git --exec-repocmd
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_git import * # NOQA
>>> import utool as ut
>>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True))
>>> command = 'git status'
>>> sudo = False
>>> result = repocmd(repo, command, sudo)
>>> print(result)
"""
import utool as ut
if ut.WIN32:
assert not sudo, 'cant sudo on windows'
if command == 'short_status':
return repo.short_status()
command_list = ut.ensure_iterable(command)
cmdstr = '\n '.join([cmd_ for cmd_ in command_list])
if not dry:
print('+--- *** repocmd(%s) *** ' % (cmdstr,))
print('repo=%s' % ut.color_text(repo.dpath, 'yellow'))
verbose = True
with repo.chdir_context():
ret = None
for count, cmd in enumerate(command_list):
if dry:
print(cmd)
continue
if not sudo or ut.WIN32:
# ret = os.system(cmd)
cmdinfo = ut.cmd2(cmd, verbout=True)
out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret'])
else:
# cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1)
out, err, ret = ut.cmd(cmd, sudo=True)
if verbose > 1:
print('ret(%d) = %r' % (count, ret,))
if ret != 0:
if error == 'raise':
raise Exception('Failed command %r' % (cmd,))
elif error == 'return':
return out
else:
raise ValueError('unknown flag error=%r' % (error,))
if return_out:
return out
if not dry:
print('L____')
def __init__(invassign, fstack, vocab, wx2_idxs, wx2_maws, wx2_fxs, wx2_axs):
invassign.fstack = fstack
invassign.vocab = vocab
invassign.wx2_idxs = wx2_idxs
invassign.wx2_maws = wx2_maws
invassign.wx2_fxs = wx2_fxs
invassign.wx2_axs = wx2_axs
invassign.wx2_num = ut.map_dict_vals(len, invassign.wx2_axs)
invassign.wx_list = sorted(invassign.wx2_num.keys())
invassign.num_list = ut.take(invassign.wx2_num, invassign.wx_list)
invassign.perword_stats = ut.get_stats(invassign.num_list)
def intersect_hack():
failed = testres.rank_mat > 0
colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
#failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
#failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)
qaids_expt = failed_col1_only_aids
gt_avl_aids1 = ut.flatten(gt_aids1)
gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
#jsontext = ut.to_json({
# 'qaids': list(qaids_expt),
# 'dinclude_aids1': list(gt_aids_expt1),
# 'dinclude_aids2': list(gt_aids_expt2),
#})
#annotation_configs.varysize_pzm
#from ibeis.expt import annotation_configs
acfg = testres.acfg_list[0]
import copy
acfg1 = copy.deepcopy(acfg)
acfg2 = copy.deepcopy(acfg)
acfg1['qcfg']['min_pername'] = None
acfg2['qcfg']['min_pername'] = None
acfg1['dcfg']['min_pername'] = None
acfg2['dcfg']['min_gt_per_name'] = None
acfg1['qcfg']['default_aids'] = qaids_expt
acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
acfg2['qcfg']['default_aids'] = qaids_expt
acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2
from ibeis.init import filter_annots
from ibeis.expt import experiment_helpers
annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)
acfg_name_list = dict( # NOQA
acfg_list=[acfg1, acfg2],
expanded_aids_list=[annots1, annots2],
)
test_cfg_name_list = ['candidacy_k']
cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)
t1, t2 = testres_list # NOQA
def group_images_by_label(label_arr, gid_arr):
"""
Input: Length N list of labels and ids
Output: Length M list of unique labels, and lenth M list of lists of ids
"""
# Reverse the image to cluster index mapping
import vtool as vt
labels_, groupxs_ = vt.group_indices(label_arr)
sortx = np.array(list(map(len, groupxs_))).argsort()[::-1]
labels = labels_.take(sortx, axis=0)
groupxs = ut.take(groupxs_, sortx)
label_gids = vt.apply_grouping(gid_arr, groupxs)
return labels, label_gids
def get_training_fsv(cm, namemode=True, num=None, top_percent=None):
"""
CommandLine:
python -m ibeis.algo.hots.scorenorm --exec-get_training_fsv --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.hots.scorenorm import * # NOQA
>>> import ibeis
>>> num = None
>>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best')
>>> (tp_fsv, tn_fsv) = get_training_fsv(cm, namemode=False)
>>> result = ('(tp_fsv, tn_fsv) = %s' % (ut.repr2((tp_fsv, tn_fsv), nl=1),))
>>> print(result)
"""
if namemode:
tp_idxs, tn_idxs = get_topname_training_idxs(cm, num=num)
else:
tp_idxs, tn_idxs = get_topannot_training_idxs(cm, num=num)
# Keep only the top scoring half of the feature matches
# top_percent = None
if top_percent is not None:
cm_orig = cm
#cm_orig.assert_self(qreq_)
tophalf_indicies = [
ut.take_percentile(fs.argsort()[::-1], top_percent)
for fs in cm.get_fsv_prod_list()
]
cm = cm_orig.take_feature_matches(tophalf_indicies, keepscores=True)
assert np.all(cm_orig.daid_list.take(tp_idxs) == cm.daid_list.take(tp_idxs))
assert np.all(cm_orig.daid_list.take(tn_idxs) == cm.daid_list.take(tn_idxs))
#cm.assert_self(qreq_)
tp_fsv = np.vstack(ut.take(cm.fsv_list, tp_idxs))
tn_fsv = np.vstack(ut.take(cm.fsv_list, tn_idxs))
return tp_fsv, tn_fsv
def simplify_graph(graph):
"""
strips out everything but connectivity
Args:
graph (nx.Graph):
Returns:
nx.Graph: new_graph
CommandLine:
python3 -m utool.util_graph simplify_graph --show
python2 -m utool.util_graph simplify_graph --show
python2 -c "import networkx as nx; print(nx.__version__)"
python3 -c "import networkx as nx; print(nx.__version__)"
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_graph import * # NOQA
>>> import utool as ut
>>> import networkx as nx
>>> graph = nx.DiGraph([('a', 'b'), ('a', 'c'), ('a', 'e'),
>>> ('a', 'd'), ('b', 'd'), ('c', 'e'),
>>> ('d', 'e'), ('c', 'e'), ('c', 'd')])
>>> new_graph = simplify_graph(graph)
>>> result = ut.repr2(list(new_graph.edges()))
>>> #adj_list = sorted(list(nx.generate_adjlist(new_graph)))
>>> #result = ut.repr2(adj_list)
>>> print(result)
[(0, 1), (0, 2), (0, 3), (0, 4), (1, 3), (2, 3), (2, 4), (3, 4)]
['0 1 2 3 4', '1 3 4', '2 4', '3', '4 3']
"""
import utool as ut
nodes = sorted(list(graph.nodes()))
node_lookup = ut.make_index_lookup(nodes)
if graph.is_multigraph():
edges = list(graph.edges(keys=True))
else:
edges = list(graph.edges())
new_nodes = ut.take(node_lookup, nodes)
if graph.is_multigraph():
new_edges = [(node_lookup[e[0]], node_lookup[e[1]], e[2], {}) for e in edges]
else:
new_edges = [(node_lookup[e[0]], node_lookup[e[1]]) for e in edges]
cls = graph.__class__
new_graph = cls()
new_graph.add_nodes_from(new_nodes)
new_graph.add_edges_from(new_edges)
return new_graph
def get_prop(self, attrname, idxs=None):
"""
Caching getter
"""
if attrname not in self.attrs:
self.attrs[attrname] = [None for _ in range(len(self))]
prop_list = self.attrs[attrname]
if idxs is None:
idxs = list(range(len(prop_list)))
props = prop_list
else:
props = ut.take(prop_list, idxs)
miss_flags = ut.flag_None_items(props)
if any(miss_flags):
miss_idxs = ut.compress(idxs, miss_flags)
miss_fpaths = self._abs(ut.take(self.rel_fpath_list, miss_idxs))
miss_iter = getattr(self, '_' + attrname)(miss_fpaths)
miss_iter = ut.ProgIter(miss_iter, length=len(miss_idxs),
label='Compute %s' % (attrname,))
for idx, val in zip(miss_idxs, miss_iter):
prop_list[idx] = val
props = ut.take(prop_list, idxs)
return props
def execute(request, parent_rowids=None, use_cache=None, postprocess=True):
""" HACKY REIMPLEMENTATION """
ut.colorprint('[req] Executing request %s' % (request,), 'yellow')
table = request.depc[request.tablename]
if use_cache is None:
use_cache = not ut.get_argflag('--nocache')
if parent_rowids is None:
parent_rowids = request.parent_rowids
else:
print('given %d specific parent_rowids' % (len(parent_rowids),))
# vsone hack (i,j) same as (j,i)
if request._symmetric:
import vtool as vt
directed_edges = np.array(parent_rowids)
undirected_edges = vt.to_undirected_edges(directed_edges)
edge_ids = vt.compute_unique_data_ids(undirected_edges)
unique_rows, unique_rowx, inverse_idx = np.unique(edge_ids, return_index=True, return_inverse=True)
parent_rowids_ = ut.take(parent_rowids, unique_rowx)
else:
parent_rowids_ = parent_rowids
# Compute and cache any uncomputed results
rowids = table.get_rowid(parent_rowids_, config=request,
recompute=not use_cache)
# Load all results
result_list = table.get_row_data(rowids)
if request._symmetric:
result_list = ut.take(result_list, inverse_idx)
if postprocess and hasattr(request, 'postprocess_execute'):
print('Converting results')
result_list = request.postprocess_execute(parent_rowids, result_list)
pass
return result_list
def isect_info(self, other):
set1 = set(self.rel_fpath_list)
set2 = set(other.rel_fpath_list)
set_comparisons = ut.odict([
('s1', set1),
('s2', set2),
('union', set1.union(set2)),
('isect', set1.intersection(set2)),
('s1 - s2', set1.difference(set2)),
('s2 - s1', set1.difference(set1)),
])
stat_stats = ut.map_vals(len, set_comparisons)
print(ut.repr4(stat_stats))
return set_comparisons
if False:
idx_lookup1 = ut.make_index_lookup(self.rel_fpath_list)
idx_lookup2 = ut.make_index_lookup(other.rel_fpath_list)
uuids1 = ut.take(self.uuids, ut.take(idx_lookup1, set_comparisons['union']))
uuids2 = ut.take(other.uuids, ut.take(idx_lookup2, set_comparisons['union']))
uuids1 == uuids2
