def get_summary(profile_block_list, maxlines=20): """ References: https://github.com/rkern/line_profiler """ time_list = [get_block_totaltime(block) for block in profile_block_list] time_list = [time if time is not None else -1 for time in time_list] blockid_list = [get_block_id(block) for block in profile_block_list] sortx = ut.list_argsort(time_list) sorted_time_list = ut.take(time_list, sortx) sorted_blockid_list = ut.take(blockid_list, sortx) aligned_blockid_list = ut.util_str.align_lines(sorted_blockid_list, ':') summary_lines = [('%6.2f seconds - ' % time) + line for time, line in zip(sorted_time_list, aligned_blockid_list)] #summary_header = ut.codeblock( # ''' # CLEANED PROFILE OUPUT # The Pystone timings are not from kernprof, so they may include kernprof # overhead, whereas kernprof timings do not (unless the line being # profiled is also decorated with kernrof) # The kernprof times are reported in Timer Units # ''') summary_lines_ = ut.listclip(summary_lines, maxlines, fromback=True) summary_text = '\n'.join(summary_lines_) return summary_text
def testdata_smk(*args, **kwargs): """ >>> from wbia.algo.smk.smk_pipeline import * # NOQA >>> kwargs = {} """ import wbia import sklearn import sklearn.model_selection # import sklearn.model_selection ibs, aid_list = wbia.testdata_aids(defaultdb='PZ_MTEST') nid_list = np.array(ibs.annots(aid_list).nids) rng = ut.ensure_rng(0) xvalkw = dict(n_splits=4, shuffle=False, random_state=rng) skf = sklearn.model_selection.StratifiedKFold(**xvalkw) train_idx, test_idx = six.next(skf.split(aid_list, nid_list)) daids = ut.take(aid_list, train_idx) qaids = ut.take(aid_list, test_idx) config = { 'num_words': 1000, } config.update(**kwargs) qreq_ = SMKRequest(ibs, qaids, daids, config) smk = qreq_.smk # qreq_ = ibs.new_query_request(qaids, daids, cfgdict={'pipeline_root': 'smk', 'proot': 'smk'}) # qreq_ = ibs.new_query_request(qaids, daids, cfgdict={}) return ibs, smk, qreq_
def isect_info(self, other): set1 = set(self.rel_fpath_list) set2 = set(other.rel_fpath_list) set_comparisons = ut.odict([ ('s1', set1), ('s2', set2), ('union', set1.union(set2)), ('isect', set1.intersection(set2)), ('s1 - s2', set1.difference(set2)), ('s2 - s1', set1.difference(set1)), ]) stat_stats = ut.map_vals(len, set_comparisons) print(ut.repr4(stat_stats)) return set_comparisons if False: idx_lookup1 = ut.make_index_lookup(self.rel_fpath_list) idx_lookup2 = ut.make_index_lookup(other.rel_fpath_list) uuids1 = ut.take(self.uuids, ut.take(idx_lookup1, set_comparisons['union'])) uuids2 = ut.take(other.uuids, ut.take(idx_lookup2, set_comparisons['union'])) uuids1 == uuids2
def make_factor_text(factor, name): collapse_uniform = True if collapse_uniform and ut.almost_allsame(factor.values): # Reduce uniform text ftext = name + ':\nuniform(%.3f)' % (factor.values[0], ) else: values = factor.values try: rowstrs = [ 'p(%s)=%.3f' % ( ','.join(n), v, ) for n, v in zip(zip(*factor.statenames), values) ] except Exception: rowstrs = [ 'p(%s)=%.3f' % ( ','.join(n), v, ) for n, v in zip(factor._row_labels(False), values) ] idxs = ut.list_argmaxima(values) for idx in idxs: rowstrs[idx] += '*' thresh = 4 always_sort = True if len(rowstrs) > thresh: sortx = factor.values.argsort()[::-1] rowstrs = ut.take(rowstrs, sortx[0:(thresh - 1)]) rowstrs += ['... %d more' % ((len(values) - len(rowstrs)), )] elif always_sort: sortx = factor.values.argsort()[::-1] rowstrs = ut.take(rowstrs, sortx) ftext = name + ': \n' + '\n'.join(rowstrs) return ftext
def get_qreq_annot_nids(qreq_, aids): # uses own internal state to grab name rowids instead of using wbia. if not hasattr(qreq_, 'aid_to_idx'): qreq_.ensure_nids() idxs = ut.take(qreq_.aid_to_idx, aids) nids = ut.take(qreq_.unique_nids, idxs) return nids
def show_single_coverage_mask(qreq_, cm, weight_mask_m, weight_mask, daids, fnum=None): import plottool as pt from ibeis import viz fnum = pt.ensure_fnum(fnum) idx_list = ut.dict_take(cm.daid2_idx, daids) nPlots = len(idx_list) + 1 nRows, nCols = pt.get_square_row_cols(nPlots) pnum_ = pt.make_pnum_nextgen(nRows, nCols) pt.figure(fnum=fnum, pnum=(1, 2, 1)) # Draw coverage masks with bbox # <FlipHack> #weight_mask_m = np.fliplr(np.flipud(weight_mask_m)) #weight_mask = np.fliplr(np.flipud(weight_mask)) # </FlipHack> stacked_weights, offset_tup, sf_tup = vt.stack_images(weight_mask_m, weight_mask, return_sf=True) (woff, hoff) = offset_tup[1] wh1 = weight_mask_m.shape[0:2][::-1] wh2 = weight_mask.shape[0:2][::-1] pt.imshow(255 * (stacked_weights), fnum=fnum, pnum=pnum_(0), title='(query image) What did match vs what should match') pt.draw_bbox(( 0, 0) + wh1, bbox_color=(0, 0, 1)) pt.draw_bbox((woff, hoff) + wh2, bbox_color=(0, 0, 1)) # Get contributing matches qaid = cm.qaid daid_list = daids fm_list = ut.take(cm.fm_list, idx_list) fs_list = ut.take(cm.fs_list, idx_list) # Draw matches for px, (daid, fm, fs) in enumerate(zip(daid_list, fm_list, fs_list), start=1): viz.viz_matches.show_matches2(qreq_.ibs, qaid, daid, fm, fs, draw_pts=False, draw_lines=True, draw_ell=False, fnum=fnum, pnum=pnum_(px), darken=.5) coverage_score = score_matching_mask(weight_mask_m, weight_mask) pt.set_figtitle('score=%.4f' % (coverage_score,))
def make_factor_text(factor, name): collapse_uniform = True if collapse_uniform and ut.almost_allsame(factor.values): # Reduce uniform text ftext = name + ':\nuniform(%.3f)' % (factor.values[0],) else: values = factor.values try: rowstrs = ['p(%s)=%.3f' % (','.join(n), v,) for n, v in zip(zip(*factor.statenames), values)] except Exception: rowstrs = ['p(%s)=%.3f' % (','.join(n), v,) for n, v in zip(factor._row_labels(False), values)] idxs = ut.list_argmaxima(values) for idx in idxs: rowstrs[idx] += '*' thresh = 4 always_sort = True if len(rowstrs) > thresh: sortx = factor.values.argsort()[::-1] rowstrs = ut.take(rowstrs, sortx[0:(thresh - 1)]) rowstrs += ['... %d more' % ((len(values) - len(rowstrs)),)] elif always_sort: sortx = factor.values.argsort()[::-1] rowstrs = ut.take(rowstrs, sortx) ftext = name + ': \n' + '\n'.join(rowstrs) return ftext
def phis_flags_list(X, idxs): """ get subset of non-aggregated residual vectors """ phis_list = ut.take(X.rvecs_list, idxs) flags_list = ut.take(X.flags_list, idxs) if X.int_rvec: phis_list = ut.lmap(smk_funcs.uncast_residual_integer, phis_list) return phis_list, flags_list
def unnest_data(data): unnested_cols = list(zip(ut.take(data, idxs2))) nested_cols = ut.take(data, idxs1) grouped_items = [nested_cols, unnested_cols] groupxs = [idxs1, idxs2] unflat = ut.ungroup(grouped_items, groupxs, nested_nCols - 1) return tuple(ut.flatten(unflat))
def get_topname_training_idxs(cm, num=5): """ gets the index of the annots in the top groundtrue name and the top groundfalse names. Args: cm (ibeis.ChipMatch): object of feature correspondences and scores num(int): number of false names (default = 5) Returns: tuple: (tp_idxs, tn_idxs) cm.daid_list[tp_idxs] are all of the annotations in the correct name. cm.daid_list[tn_idxs] are all of the annotations in the top `num_false` incorrect names. CommandLine: python -m ibeis --tf get_topname_training_idxs --show Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.scorenorm import * # NOQA >>> import ibeis >>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best') >>> num = 1 >>> (tp_idxs, tn_idxs) = get_topname_training_idxs(cm, num) >>> result = ('(tp_idxs, tn_idxs) = %s' % (ut.repr2((tp_idxs, tn_idxs), nl=1),)) >>> print(result) (tp_idxs, tn_idxs) = ( np.array([0, 1, 2], dtype=np.int64), [3, 4, 5, 6], ) """ if num is None: num = 5 sortx = cm.name_argsort() sorted_nids = vt.take2(cm.unique_nids, sortx) sorted_groupxs = ut.take(cm.name_groupxs, sortx) # name ranks of the groundtrue name tp_ranks = np.where(sorted_nids == cm.qnid)[0] if len(tp_ranks) == 0: #if ut.STRICT: # raise Exception('tp_ranks=0') #else: raise UnbalancedExampleException('tp_ranks=0') # name ranks of the top groundfalse names tp_rank = tp_ranks[0] tn_ranks = [rank for rank in range(num + 1) if rank != tp_rank and rank < len(sorted_groupxs)] if len(tn_ranks) == 0: #if ut.STRICT: # raise Exception('tn_ranks=0') #else: raise UnbalancedExampleException('tn_ranks=0') # annot idxs of the examples tp_idxs = sorted_groupxs[tp_rank] tn_idxs = ut.flatten(ut.take(sorted_groupxs, tn_ranks)) return tp_idxs, tn_idxs
def biggest_files(drive): print('Biggest Files in %r' % (drive,)) sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(drive.fpath_bytes_list, sel) biggest_files = ut.take(drive.fpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_files)) print(ut.list_str(biginfo_list, strvals=True))
def permute_columns(self, new_order, inplace=True): import utool as ut self.header = ut.take(self.header, new_order) self.header_tags = ut.take(self.header_tags, new_order) self.row_data = ut.listT(ut.take(ut.listT(self.row_data), new_order)) if self.short_header is not None: self.short_header = ut.take(self.short_header, new_order) return self
def bow_vector(X, wx_to_weight, nwords): import vtool as vt wxs = sorted(list(X.wx_set)) tf = np.array(ut.take(X.termfreq, wxs)) idf = np.array(ut.take(wx_to_weight, wxs)) bow_ = tf * idf bow_ = vt.normalize(bow_) bow = SparseVector(dict(zip(wxs, bow_))) X.bow = bow
def biggest_files(drive): print('Biggest Files in %r' % (drive, )) sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(drive.fpath_bytes_list, sel) biggest_files = ut.take(drive.fpath_list, sel) biginfo_list = list( zip(map(ut.byte_str2, biggest_nbytes), biggest_files)) print(ut.repr2(biginfo_list, strvals=True))
def residual_args(vocab, vecs, nAssign, int_rvec): fx_to_vecs = vecs fx_to_wxs, fx_to_maws = smk_funcs.assign_to_words(vocab, fx_to_vecs, nAssign) wx_to_fxs, wx_to_maws = smk_funcs.invert_assigns(fx_to_wxs, fx_to_maws) wx_list = sorted(wx_to_fxs.keys()) word_list = ut.take(vocab.wx_to_word, wx_list) fxs_list = ut.take(wx_to_fxs, wx_list) maws_list = ut.take(wx_to_maws, wx_list) argtup = (wx_list, word_list, fxs_list, maws_list, fx_to_vecs, int_rvec) return argtup
def get_name_shortlist_aids( daid_list, dnid_list, annot_score_list, name_score_list, nid2_nidx, nNameShortList, nAnnotPerName, ): r""" CommandLine: python -m wbia.algo.hots.scoring --test-get_name_shortlist_aids Example: >>> # ENABLE_DOCTEST >>> from wbia.algo.hots.scoring import * # NOQA >>> daid_list = np.array([11, 12, 13, 14, 15, 16, 17]) >>> dnid_list = np.array([21, 21, 21, 22, 22, 23, 24]) >>> annot_score_list = np.array([ 6, 2, 3, 5, 6, 3, 2]) >>> name_score_list = np.array([ 8, 9, 5, 4]) >>> nid2_nidx = {21:0, 22:1, 23:2, 24:3} >>> nNameShortList, nAnnotPerName = 3, 2 >>> args = (daid_list, dnid_list, annot_score_list, name_score_list, ... nid2_nidx, nNameShortList, nAnnotPerName) >>> top_daids = get_name_shortlist_aids(*args) >>> result = str(top_daids) >>> print(result) [15, 14, 11, 13, 16] """ unique_nids, groupxs = vt.group_indices(np.array(dnid_list)) grouped_annot_scores = vt.apply_grouping(annot_score_list, groupxs) grouped_daids = vt.apply_grouping(np.array(daid_list), groupxs) # Ensure name score list is aligned with the unique_nids aligned_name_score_list = name_score_list.take( ut.dict_take(nid2_nidx, unique_nids)) # Sort each group by the name score group_sortx = aligned_name_score_list.argsort()[::-1] _top_daid_groups = ut.take(grouped_daids, group_sortx) _top_annot_score_groups = ut.take(grouped_annot_scores, group_sortx) top_daid_groups = ut.listclip(_top_daid_groups, nNameShortList) top_annot_score_groups = ut.listclip(_top_annot_score_groups, nNameShortList) # Sort within each group by the annotation score top_daid_sortx_groups = [ annot_score_group.argsort()[::-1] for annot_score_group in top_annot_score_groups ] top_sorted_daid_groups = vt.ziptake(top_daid_groups, top_daid_sortx_groups) top_clipped_daids = [ ut.listclip(sorted_daid_group, nAnnotPerName) for sorted_daid_group in top_sorted_daid_groups ] top_daids = ut.flatten(top_clipped_daids) return top_daids
def show_single_coverage_mask(qreq_, cm, weight_mask_m, weight_mask, daids, fnum=None): import plottool as pt from ibeis import viz fnum = pt.ensure_fnum(fnum) idx_list = ut.dict_take(cm.daid2_idx, daids) nPlots = len(idx_list) + 1 nRows, nCols = pt.get_square_row_cols(nPlots) pnum_ = pt.make_pnum_nextgen(nRows, nCols) pt.figure(fnum=fnum, pnum=(1, 2, 1)) # Draw coverage masks with bbox # <FlipHack> #weight_mask_m = np.fliplr(np.flipud(weight_mask_m)) #weight_mask = np.fliplr(np.flipud(weight_mask)) # </FlipHack> stacked_weights, offset_tup, sf_tup = vt.stack_images(weight_mask_m, weight_mask, return_sf=True) (woff, hoff) = offset_tup[1] wh1 = weight_mask_m.shape[0:2][::-1] wh2 = weight_mask.shape[0:2][::-1] pt.imshow(255 * (stacked_weights), fnum=fnum, pnum=pnum_(0), title='(query image) What did match vs what should match') pt.draw_bbox((0, 0) + wh1, bbox_color=(0, 0, 1)) pt.draw_bbox((woff, hoff) + wh2, bbox_color=(0, 0, 1)) # Get contributing matches qaid = cm.qaid daid_list = daids fm_list = ut.take(cm.fm_list, idx_list) fs_list = ut.take(cm.fs_list, idx_list) # Draw matches for px, (daid, fm, fs) in enumerate(zip(daid_list, fm_list, fs_list), start=1): viz.viz_matches.show_matches2(qreq_.ibs, qaid, daid, fm, fs, draw_pts=False, draw_lines=True, draw_ell=False, fnum=fnum, pnum=pnum_(px), darken=.5) coverage_score = score_matching_mask(weight_mask_m, weight_mask) pt.set_figtitle('score=%.4f' % (coverage_score, ))
def new_external_annot(aid, fx_to_wxs, fx_to_maws, int_rvec): wx_to_fxs, wx_to_maws = smk_funcs.invert_assigns(fx_to_wxs, fx_to_maws) X = inverted_index.SingleAnnot() X.aid = aid # Build Aggregate Residual Vectors X.wx_list = np.array(sorted(wx_to_fxs.keys()), dtype=np.int32) X.wx_to_idx = ut.make_index_lookup(X.wx_list) X.int_rvec = int_rvec X.wx_set = set(X.wx_list) # TODO: maybe use offset list structure instead of heavy nesting X.fxs_list = ut.take(wx_to_fxs, X.wx_list) X.maws_list = ut.take(wx_to_maws, X.wx_list) return X
def take(self, idxs): """ Creates a subset of the list using the specified indices. """ rowids = ut.take(self._rowids, idxs) # Create a new instance pointing only to the requested subset newself = self.__class__(rowids, ibs=self._ibs, config=self._config, caching=self._caching) # Pass along any internally cached values _new_internal = {key: ut.take(val, idxs) for key, val in self._internal_attrs.items()} newself._internal_attrs = _new_internal return newself
def get_annotmatch_rowid_from_undirected_superkey(ibs, aids1, aids2): # The directed nature of this makes a few things difficult and may cause # odd behavior am_rowids = ibs.get_annotmatch_rowid_from_superkey(aids1, aids2) idxs = ut.where([r is None for r in am_rowids]) # Check which ones are None aids1_ = ut.take(aids1, idxs) aids2_ = ut.take(aids2, idxs) am_rowids_ = ibs.get_annotmatch_rowid_from_superkey(aids2_, aids1_) # Use the other rowid if found for idx, rowid in zip(idxs, am_rowids_): am_rowids[idx] = rowid return am_rowids
def biggest_dirs(drive): print('Biggest Dirs in %r' % (drive,)) dpath_list = drive.dpath_list fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list) unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list, fidxs_list) dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list)) sortx = ut.list_argsort(dpath_nbytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(dpath_nbytes_list, sel) biggest_dpaths = ut.take(dpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths)) print(ut.list_str(biginfo_list, strvals=True)) pass
def biggest_dirs(drive): print('Biggest Dirs in %r' % (drive, )) dpath_list = drive.dpath_list fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list) unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list, fidxs_list) dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list)) sortx = ut.list_argsort(dpath_nbytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(dpath_nbytes_list, sel) biggest_dpaths = ut.take(dpath_list, sel) biginfo_list = list( zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths)) print(ut.repr2(biginfo_list, strvals=True)) pass
def predict_proba_df(clf, X_df, class_names=None): """ Calls sklearn classifier predict_proba but then puts results in a dataframe using the same index as X_df and incorporating all possible class_names given """ if class_names is not None: columns = ut.take(class_names, clf.classes_) else: columns = None if len(X_df) == 0: return pd.DataFrame(columns=columns) try: probs = clf.predict_proba(X_df) except ValueError: # solves a problem when values are infinity for whatever reason X = X_df.values.copy() X[~np.isfinite(X)] = np.nan probs = clf.predict_proba(X) probs_df = pd.DataFrame(probs, columns=columns, index=X_df.index) # add in zero probability for classes without training data if class_names is not None: missing = ut.setdiff(class_names, columns) if missing: for classname in missing: probs_df = probs_df.assign(**{ classname: np.zeros(len(probs_df))}) return probs_df
def get_patches(invassign, wx): ax_list = invassign.wx2_axs[wx] fx_list = invassign.wx2_fxs[wx] config = invassign.fstack.config ibs = invassign.fstack.ibs unique_axs, groupxs = vt.group_indices(ax_list) fxs_groups = vt.apply_grouping(fx_list, groupxs) unique_aids = ut.take(invassign.fstack.ax2_aid, unique_axs) all_kpts_list = ibs.depc.d.get_feat_kpts(unique_aids, config=config) sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0) chip_list = ibs.depc_annot.d.get_chips_img(unique_aids) # convert to approprate colorspace #if colorspace is not None: # chip_list = vt.convert_image_list_colorspace(chip_list, colorspace) # ut.print_object_size(chip_list, 'chip_list') patch_size = 64 grouped_patches_list = [ vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0] for chip, kpts in ut.ProgIter(zip(chip_list, sub_kpts_list), nTotal=len(unique_aids), lbl='warping patches') ] # Make it correspond with original fx_list and ax_list word_patches = vt.invert_apply_grouping(grouped_patches_list, groupxs) return word_patches
def flat_compute_rmi_edges(inputs): """ Defines order of computation that maps input_ids to target_ids. CommandLine: python -m dtool.input_helpers flat_compute_rmi_edges Returns: list: compute_edges Each item is a tuple of input/output RootMostInputs ([parent_1, ..., parent_n], node_i) All parents should be known before you reach the i-th item in the list. Results of the the i-th item may be used in subsequent item computations. Example: >>> from dtool.input_helpers import * # NOQA >>> from dtool.example_depcache2 import * # NOQA >>> depc =testdata_custom_annot_depc([ ... dict(tablename='chips', parents=['annot']), ... dict(tablename='Notch_Tips', parents=['annot']), ... dict(tablename='Cropped_Chips', parents=['chips', 'Notch_Tips']), ... ]) >>> table = depc['Cropped_Chips'] >>> inputs = exi_inputs = table.rootmost_inputs.total_expand() >>> compute_rmi_edges = exi_inputs.flat_compute_rmi_edges() >>> input_rmis = compute_rmi_edges[-1][0] >>> result = ut.repr2(input_rmis) >>> print(result) [chips[t, t:1, 1:1], Notch_Tips[t, t:1, 1:1]] """ sink = list(ut.nx_sink_nodes(inputs.exi_graph))[0] exi_graph = inputs.exi_graph compute_rmi_edges = [] flat_compute_order = inputs.flat_compute_order() exi_graph = inputs.exi_graph for output_node in flat_compute_order: input_nodes = list(exi_graph.predecessors(output_node)) # input_edges = [(node, output_node, node.branch_id.k) for node in input_nodes] input_edges = [(node, output_node) for node in input_nodes] # another sorting strategy. maybe this is correct. sortx = [ exi_graph.get_edge_data(*e).get('parent_colx') for e in input_edges ] input_nodes = ut.take(input_nodes, sortx) input_rmis = [ RootMostInput(node, sink, exi_graph) for node in input_nodes ] # input_rmis = sort_rmi_list(input_rmis) output_rmis = RootMostInput(output_node, sink, exi_graph) edge = (input_rmis, output_rmis) compute_rmi_edges.append(edge) return compute_rmi_edges
def execute_singles(qreq_): if qreq_.use_single_cache: qaid_to_hit = _load_singles(qreq_) else: qaid_to_hit = {} hit_all = len(qaid_to_hit) == len(qreq_.qaids) hit_any = len(qaid_to_hit) > 0 if hit_all: qaid_to_cm = qaid_to_hit else: if hit_any: logger.info('... partial cm cache hit %d/%d' % (len(qaid_to_hit), len(qreq_))) hit_aids = list(qaid_to_hit.keys()) miss_aids = ut.setdiff(qreq_.qaids, hit_aids) qreq_miss = qreq_.shallowcopy(miss_aids) else: qreq_miss = qreq_ # Compute misses qreq_miss.ensure_data() qaid_to_cm = execute_and_save(qreq_miss) # Merge misses with hits if hit_any: qaid_to_cm.update(qaid_to_hit) cm_list = ut.take(qaid_to_cm, qreq_.qaids) return cm_list
def sort_rmi_list(rmi_list): """ CommandLine: python -m dtool.input_helpers sort_rmi_list Example: >>> from dtool.input_helpers import * # NOQA >>> from dtool.example_depcache2 import * # NOQA >>> depc =testdata_custom_annot_depc([ ... dict(tablename='Notch_Tips', parents=['annot']), ... dict(tablename='chips', parents=['annot']), ... dict(tablename='Cropped_Chips', parents=['chips', 'Notch_Tips']), ... ]) >>> table = depc['Cropped_Chips'] >>> inputs = exi_inputs = table.rootmost_inputs >>> compute_rmi_edges = exi_inputs.flat_compute_rmi_edges() >>> input_rmis = compute_rmi_edges[-1][0] >>> rmi_list = input_rmis[::-1] >>> rmi_list = sort_rmi_list(rmi_list) >>> assert rmi_list[0].node[0] == 'chips' """ # Order the input rmis via declaration reverse_compute_branches = [rmi.compute_order()[::-1] for rmi in rmi_list] # print('rmi_list = %r' % (rmi_list,)) # rmi = rmi_list[0] # hack # reverse_compute_branches = [path[::-1] for path in nx.all_simple_paths(rmi.exi_graph, rmi.node, rmi.sink)] sort_keys = [ tuple([r.branch_id.parent_colx for r in rs]) for rs in reverse_compute_branches ] sortx = ut.argsort(sort_keys) rmi_list = ut.take(rmi_list, sortx) return rmi_list
def _cm_breaking(infr, cm_list=None, review_cfg={}): """ >>> from wbia.algo.graph.core import * # NOQA >>> review_cfg = {} """ if cm_list is None: cm_list = infr.cm_list ranks_top = review_cfg.get('ranks_top', None) ranks_bot = review_cfg.get('ranks_bot', None) # Construct K-broken graph edges = [] if ranks_bot is None: ranks_bot = 0 for count, cm in enumerate(cm_list): score_list = cm.annot_score_list rank_list = ut.argsort(score_list)[::-1] sortx = ut.argsort(rank_list) top_sortx = sortx[:ranks_top] bot_sortx = sortx[len(sortx) - ranks_bot :] short_sortx = ut.unique(top_sortx + bot_sortx) daid_list = ut.take(cm.daid_list, short_sortx) for daid in daid_list: u, v = (cm.qaid, daid) if v < u: u, v = v, u edges.append((u, v)) return edges
def report_partitioning_statistics(new_reduced_joint): # compute partitioning statistics import vtool as vt vals, idxs = vt.group_indices(new_reduced_joint.values.ravel()) #groupsize = list(map(len, idxs)) #groupassigns = ut.unflat_vecmap(new_reduced_joint.assignment, idxs) all_states = new_reduced_joint._row_labels(asindex=True) clusterstats = [tuple(sorted(list(ut.dict_hist(a).values()))) for a in all_states] grouped_vals = ut.group_items(new_reduced_joint.values.ravel(), clusterstats) #probs_assigned_to_clustertype = [( # sorted(np.unique(np.array(b).round(decimals=5)).tolist())[::-1], a) # for a, b in grouped_vals.items()] probs_assigned_to_clustertype = [( ut.dict_hist(np.array(b).round(decimals=5)), a) for a, b in grouped_vals.items()] sortx = ut.argsort([max(c[0].keys()) for c in probs_assigned_to_clustertype]) probs_assigned_to_clustertype = ut.take(probs_assigned_to_clustertype, sortx) # This list of 2-tuples with the first item being the unique # probabilies that are assigned to a cluster type along with the number # of times they were assigned. A cluster type is the second item. Every # number represents how many annotations were assigned to a specific # label. The length of that list is the number of total labels. For # all low scores you will see [[{somenum: 1}, {0: 800}], [1, 1, 1, ... 1]] # indicating that that the assignment of everyone to a different label happend once # where the probability was somenum and a 800 times where the probability was 0. #print(sorted([(b, a) for a, b in ut.map_dict_vals(sum, x)]).items()) #z = sorted([(b, a) for a, b in ut.map_dict_vals(sum, grouped_vals).items()]) print(ut.repr2(probs_assigned_to_clustertype, nl=2, precision=2, sorted_=True))
def find_requested_hrefs(all_href_list, py_version, pkg_list): """ Filters out everything but the requested urls Returns the urls to download the requested installers """ print('Filtering to only requested HREFS') href_list1, missing1 = filter_href_list(all_href_list, pkg_list, OS_VERSION, py_version) #print('missing1 = %r' % (missing1,)) href_list2, missing2 = filter_href_list(all_href_list, missing1, OS_VERSION, py_version) #print('missing2 = %r' % (missing2,)) #print(href_list2) href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64', py_version.replace('p', 'P')) #print('missing3 = %r' % (missing3,)) href_list4, missing4 = filter_href_list( all_href_list, missing3, 'any', py_version.replace('cp', 'py')[0:3]) if len(missing4) > 0: print('Could not find a match for missing4=%r' % (missing4, )) #import Levenshtein for pkg in missing4: #dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list] dist_list = [ 0 if (href.find(pkg) > -1) else 100 for href in all_href_list ] closest_matche_xs = ut.list_argsort(dist_list)[::1] print('Perhaps pkg=%r could match one of these?' % (pkg, )) closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3]) print(ut.indentjoin(closest_hrefs, '\n ')) href_list = href_list1 + href_list2 + href_list3 + href_list4 return href_list
def load_ordered_annots(data_uri_order, query_uri_order): # Open the wbia version of oxford import wbia ibs = wbia.opendb('Oxford') def reorder_annots(_annots, uri_order): intern_uris = get_annots_imgid(_annots) lookup = ut.make_index_lookup(intern_uris) _reordered = _annots.take(ut.take(lookup, uri_order)) return _reordered # Load database annotations and reorder them to agree with internals _dannots = ibs.annots(ibs.filter_annots_general(has_none='query')) data_annots = reorder_annots(_dannots, data_uri_order) # Load query annototations and reorder to standard order _qannots = ibs.annots(ibs.filter_annots_general(has_any='query')) query_annots = reorder_annots(_qannots, query_uri_order) # Map each query annot to its corresponding data index dgid_to_dx = ut.make_index_lookup(data_annots.gids) qx_to_dx = ut.take(dgid_to_dx, query_annots.gids) return ibs, query_annots, data_annots, qx_to_dx
def find_needsmove_to_other(self, other): hash1 = self.get_prop('md5_stride') hash2 = other.get_prop('md5_stride') idxs1 = list(range(len(hash1))) hash_to_idxs = ut.group_items(idxs1, hash1) # Find what we have that other doesnt have and move it there other_missing = set(hash1).difference(hash2) missing_idxs1 = ut.flatten(ut.take(hash_to_idxs, other_missing)) data = ut.ColumnLists({ 'idx': missing_idxs1, 'fname': self.get_prop('fname', missing_idxs1), 'dname': self.get_prop('dname', missing_idxs1), 'full_path': self.get_prop('full_path', missing_idxs1), 'nbytes': self.get_prop('nbytes', missing_idxs1), }) data = data.compress([f != 'Thumbs.db' for f in data['fname']]) data['ext'] = self.get_prop('ext', data['idx']) ut.dict_hist(data['ext']) data.print(ignore=['full_path', 'dname'])
def _heuristic_weighting(nodes, avail_uv): avail_uv = np.array(avail_uv) weights = np.ones(len(avail_uv)) if 'view_weight' in enabled_heuristics: from vtool import _rhomb_dist view_edge = [(node_to_view[u], node_to_view[v]) for (u, v) in avail_uv] view_weight = np.array( [_rhomb_dist.VIEW_CODE_DIST[(v1, v2)] for (v1, v2) in view_edge] ) # Assume comparable by default and prefer undefined # more than probably not, but less than definately so. view_weight[np.isnan(view_weight)] = 1.5 # Prefer viewpoint 10x more than time weights += 10 * view_weight if 'time_weight' in enabled_heuristics: # Prefer linking annotations closer in time times = ut.take(node_to_time, nodes) maxtime = vt.safe_max(times, fill=1, nans=False) mintime = vt.safe_min(times, fill=0, nans=False) time_denom = maxtime - mintime # Try linking by time for lynx data time_delta = np.array( [abs(node_to_time[u] - node_to_time[v]) for u, v in avail_uv] ) time_weight = time_delta / time_denom weights += time_weight weights = np.array(weights) weights[np.isnan(weights)] = 1.0 avail = [(u, v, {'weight': w}) for (u, v), w in zip(avail_uv, weights)] return avail
def duplicates(self): uuid_to_dupxs = ut.find_duplicate_items(self.uuids) dup_fpaths = [ ut.take(self.rel_fpath_list, idxs) for idxs in uuid_to_dupxs.values() ] return dup_fpaths
def compute_gammas(inva, alpha, thresh): """ Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.smk.inverted_index import * # NOQA >>> qreq_, inva = testdata_inva() >>> inva.wx_to_weight = inva.compute_word_weights('uniform') >>> alpha = 3.0 >>> thresh = 0.0 >>> gamma_list = inva.compute_gammas(alpha, thresh) """ # TODO: sep wx_to_weight = inva.wx_to_weight _prog = ut.ProgPartial(length=len(inva.wx_lists), bs=True, lbl='gamma', adjust=True) _iter = zip(inva.wx_lists, inva.agg_rvecs, inva.agg_flags) gamma_list = [] for wx_list, phiX_list, flagsX_list in _prog(_iter): if inva.int_rvec: phiX_list = smk_funcs.uncast_residual_integer(phiX_list) weight_list = np.array(ut.take(wx_to_weight, wx_list)) gammaX = smk_funcs.gamma_agg(phiX_list, flagsX_list, weight_list, alpha, thresh) gamma_list.append(gammaX) return gamma_list
def find_requested_hrefs(all_href_list, py_version, pkg_list): """ Filters out everything but the requested urls Returns the urls to download the requested installers """ print('Filtering to only requested HREFS') href_list1, missing1 = filter_href_list(all_href_list, pkg_list, OS_VERSION, py_version) #print('missing1 = %r' % (missing1,)) href_list2, missing2 = filter_href_list(all_href_list, missing1, OS_VERSION, py_version) #print('missing2 = %r' % (missing2,)) #print(href_list2) href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64', py_version.replace('p', 'P')) #print('missing3 = %r' % (missing3,)) href_list4, missing4 = filter_href_list(all_href_list, missing3, 'any', py_version.replace('cp', 'py')[0:3]) if len(missing4) > 0: print('Could not find a match for missing4=%r' % (missing4,)) #import Levenshtein for pkg in missing4: #dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list] dist_list = [0 if (href.find(pkg) > -1) else 100 for href in all_href_list] closest_matche_xs = ut.list_argsort(dist_list)[::1] print('Perhaps pkg=%r could match one of these?' % (pkg,)) closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3]) print(ut.indentjoin(closest_hrefs, '\n ')) href_list = href_list1 + href_list2 + href_list3 + href_list4 return href_list
def compute_annot_occurrence_ids(ibs, aid_list): from ibeis.algo.preproc import preproc_occurrence gid_list = ibs.get_annot_gids(aid_list) gid2_aids = ut.group_items(aid_list, gid_list) flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False) occurid2_gids = ut.group_items(flat_gids, flat_imgsetids) occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()} return occurid2_aids
def _update_state(model): import networkx as nx nodes = sorted(list(model.graph.nodes())) edges = list(model.graph.edges()) edge2_weights = nx.get_edge_attributes(model.graph, 'weight') node2_labeling = nx.get_node_attributes(model.graph, 'name_label') edge_weights = ut.take(edge2_weights, edges) labeling = ut.take(node2_labeling, nodes) n_nodes = len(nodes) # Model state model.n_nodes = n_nodes model.edges = edges model.edge_weights = edge_weights # Model parameters model.labeling = np.zeros(model.n_nodes, dtype=np.int32) model._update_labels(labeling=labeling) model._update_weights()
def get_total_nbytes(drive, fidx_list=None): if fidx_list is None: if drive.fpath_bytes_list_ is None: return None if fidx_list is None: return np.nansum(drive.fpath_bytes_list) else: return np.nansum(ut.take(drive.fpath_bytes_list, fidx_list))
def chain_crf(): from pystruct.datasets import load_letters letters = load_letters() X, y, folds = ut.take(letters, ['data', 'labels', 'folds']) X, y = np.array(X), np.array(y) X_train, X_test = X[folds == 1], X[folds != 1] y_train, y_test = y[folds == 1], y[folds != 1] len(X_train)
def get_graph_bounding_box(graph): import utool as ut import networkx as nx import vtool as vt #nx.get_node_attrs = nx.get_node_attributes nodes = list(graph.nodes()) pos_list = ut.take(nx.get_node_attributes(graph, 'pos'), nodes) shape_list = ut.take(nx.get_node_attributes(graph, 'size'), nodes) node_extents = np.array([ vt.extent_from_bbox(vt.bbox_from_center_wh(xy, wh)) for xy, wh in zip(pos_list, shape_list) ]) tl_x, br_x, tl_y, br_y = node_extents.T extent = tl_x.min(), br_x.max(), tl_y.min(), br_y.max() bbox = vt.bbox_from_extent(extent) return bbox
def get_name_shortlist_aids(daid_list, dnid_list, annot_score_list, name_score_list, nid2_nidx, nNameShortList, nAnnotPerName): r""" CommandLine: python -m ibeis.algo.hots.scoring --test-get_name_shortlist_aids Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.scoring import * # NOQA >>> # build test data >>> daid_list = np.array([11, 12, 13, 14, 15, 16, 17]) >>> dnid_list = np.array([21, 21, 21, 22, 22, 23, 24]) >>> annot_score_list = np.array([ 6, 2, 3, 5, 6, 3, 2]) >>> name_score_list = np.array([ 8, 9, 5, 4]) >>> nid2_nidx = {21:0, 22:1, 23:2, 24:3} >>> nNameShortList, nAnnotPerName = 3, 2 >>> # execute function >>> args = (daid_list, dnid_list, annot_score_list, name_score_list, ... nid2_nidx, nNameShortList, nAnnotPerName) >>> top_daids = get_name_shortlist_aids(*args) >>> # verify results >>> result = str(top_daids) >>> print(result) [15, 14, 11, 13, 16] """ unique_nids, groupxs = vt.group_indices(np.array(dnid_list)) grouped_annot_scores = vt.apply_grouping(annot_score_list, groupxs) grouped_daids = vt.apply_grouping(np.array(daid_list), groupxs) # Ensure name score list is aligned with the unique_nids aligned_name_score_list = name_score_list.take(ut.dict_take(nid2_nidx, unique_nids)) # Sort each group by the name score group_sortx = aligned_name_score_list.argsort()[::-1] _top_daid_groups = ut.take(grouped_daids, group_sortx) _top_annot_score_groups = ut.take(grouped_annot_scores, group_sortx) top_daid_groups = ut.listclip(_top_daid_groups, nNameShortList) top_annot_score_groups = ut.listclip(_top_annot_score_groups, nNameShortList) # Sort within each group by the annotation score top_daid_sortx_groups = [annot_score_group.argsort()[::-1] for annot_score_group in top_annot_score_groups] top_sorted_daid_groups = vt.ziptake(top_daid_groups, top_daid_sortx_groups) top_clipped_daids = [ut.listclip(sorted_daid_group, nAnnotPerName) for sorted_daid_group in top_sorted_daid_groups] top_daids = ut.flatten(top_clipped_daids) return top_daids
def show_name(ibs, nid, in_image=True, fnum=0, sel_aids=[], subtitle='', annote=False, aid_list=None, index_list=None, **kwargs): r""" Args: ibs (IBEISController): ibeis controller object nid (?): in_image (bool): fnum (int): figure number sel_aids (list): subtitle (str): annote (bool): CommandLine: python -m ibeis.viz.viz_name --test-show_name --dpath ~/latex/crall-candidacy-2015 --save 'figures/{name}.jpg' --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage --diskshow python -m ibeis.viz.viz_name --test-show_name --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage --show python -m ibeis.viz.viz_name --test-show_name --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.viz.viz_name import * # NOQA >>> ibs, nid, in_image, index_list = testdata_showname() >>> fnum = 0 >>> sel_aids = [] >>> subtitle = '' >>> annote = False >>> # execute function >>> show_name(ibs, nid, in_image, fnum, sel_aids, subtitle, annote, index_list=index_list) >>> ut.show_if_requested() """ print('[viz_name] show_name nid=%r, index_list=%r, aid_list=%r' % (nid, index_list, aid_list)) if aid_list is None: aid_list = ibs.get_name_aids(nid) else: assert ut.list_all_eq_to(ibs.get_annot_nids(aid_list), nid) if index_list is not None: aid_list = ut.take(aid_list, index_list) name = ibs.get_name_texts((nid,)) print('[viz_name] * name=%r aid_list=%r' % (name, aid_list)) show_multiple_chips(ibs, aid_list, in_image=in_image, fnum=fnum, sel_aids=sel_aids, annote=annote, **kwargs) if isinstance(nid, np.ndarray): nid = nid[0] if isinstance(name, np.ndarray): name = name[0] use_figtitle = not ut.get_argflag('--no-figtitle') if use_figtitle: figtitle = 'Name View nid=%r name=%r' % (nid, name) df2.set_figtitle(figtitle)
def level_order(graph): import utool as ut node_to_level = ut.nx_dag_node_rank(graph) #source = ut.nx_source_nodes(graph)[0] #longest_paths = dict([(target, dag_longest_path(graph, source, target)) # for target in graph.nodes()]) #node_to_level = ut.map_dict_vals(len, longest_paths) grouped = ut.group_items(node_to_level.keys(), node_to_level.values()) levels = ut.take(grouped, range(1, len(grouped) + 1)) return levels
def make_prob_names(infr): cm_list = infr.cm_list # Consolodate information from a series of chip matches unique_nids = sorted(ut.list_union(*[cm.unique_nids for cm in cm_list])) #nid2_nidx = ut.make_index_lookup(unique_nids) # Populate matrix of raw name scores prob_names = np.zeros((len(cm_list), len(unique_nids))) for count, cm in enumerate(cm_list): try: name_scores = ut.dict_take(cm.nid2_name_score, unique_nids, 0) except AttributeError: unique_nidxs = ut.take(cm.nid2_nidx, unique_nids) name_scores = ut.take(cm.name_score_list, unique_nidxs) prob_names[count][:] = name_scores # Normalize to row stochastic matrix prob_names /= prob_names.sum(axis=1)[:, None] #print(ut.hz_str('prob_names = ', ut.array2string2(prob_names, #precision=2, max_line_width=140, suppress_small=True))) return unique_nids, prob_names
def jagged_stats_info(arr_, lbl, col_lbls): arr = ut.recursive_replace(arr_, np.inf, np.nan) # Treat infinite as nan stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True) sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls) sel_col_lbls = ut.take(col_lbls, sel_indices) statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True) stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw) sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw) sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str return stat_str, sel_stat_str
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, 'cant sudo on windows' if command == 'short_status': return repo.short_status() command_list = ut.ensure_iterable(command) cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) if not dry: print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ut.color_text(repo.dpath, 'yellow')) verbose = True with repo.chdir_context(): ret = None for count, cmd in enumerate(command_list): if dry: print(cmd) continue if not sudo or ut.WIN32: # ret = os.system(cmd) cmdinfo = ut.cmd2(cmd, verbout=True) out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret']) else: # cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1) out, err, ret = ut.cmd(cmd, sudo=True) if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: if error == 'raise': raise Exception('Failed command %r' % (cmd,)) elif error == 'return': return out else: raise ValueError('unknown flag error=%r' % (error,)) if return_out: return out if not dry: print('L____')
def __init__(invassign, fstack, vocab, wx2_idxs, wx2_maws, wx2_fxs, wx2_axs): invassign.fstack = fstack invassign.vocab = vocab invassign.wx2_idxs = wx2_idxs invassign.wx2_maws = wx2_maws invassign.wx2_fxs = wx2_fxs invassign.wx2_axs = wx2_axs invassign.wx2_num = ut.map_dict_vals(len, invassign.wx2_axs) invassign.wx_list = sorted(invassign.wx2_num.keys()) invassign.num_list = ut.take(invassign.wx2_num, invassign.wx_list) invassign.perword_stats = ut.get_stats(invassign.num_list)
def intersect_hack(): failed = testres.rank_mat > 0 colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T] #failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0]) #failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only) failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1]) failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only) gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids) gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids) qaids_expt = failed_col1_only_aids gt_avl_aids1 = ut.flatten(gt_aids1) gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1)) ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1) ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2) #jsontext = ut.to_json({ # 'qaids': list(qaids_expt), # 'dinclude_aids1': list(gt_aids_expt1), # 'dinclude_aids2': list(gt_aids_expt2), #}) #annotation_configs.varysize_pzm #from ibeis.expt import annotation_configs acfg = testres.acfg_list[0] import copy acfg1 = copy.deepcopy(acfg) acfg2 = copy.deepcopy(acfg) acfg1['qcfg']['min_pername'] = None acfg2['qcfg']['min_pername'] = None acfg1['dcfg']['min_pername'] = None acfg2['dcfg']['min_gt_per_name'] = None acfg1['qcfg']['default_aids'] = qaids_expt acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1 acfg2['qcfg']['default_aids'] = qaids_expt acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2 from ibeis.init import filter_annots from ibeis.expt import experiment_helpers annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True) annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True) acfg_name_list = dict( # NOQA acfg_list=[acfg1, acfg2], expanded_aids_list=[annots1, annots2], ) test_cfg_name_list = ['candidacy_k'] cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs) t1, t2 = testres_list # NOQA
def group_images_by_label(label_arr, gid_arr): """ Input: Length N list of labels and ids Output: Length M list of unique labels, and lenth M list of lists of ids """ # Reverse the image to cluster index mapping import vtool as vt labels_, groupxs_ = vt.group_indices(label_arr) sortx = np.array(list(map(len, groupxs_))).argsort()[::-1] labels = labels_.take(sortx, axis=0) groupxs = ut.take(groupxs_, sortx) label_gids = vt.apply_grouping(gid_arr, groupxs) return labels, label_gids
def get_training_fsv(cm, namemode=True, num=None, top_percent=None): """ CommandLine: python -m ibeis.algo.hots.scorenorm --exec-get_training_fsv --show Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.scorenorm import * # NOQA >>> import ibeis >>> num = None >>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best') >>> (tp_fsv, tn_fsv) = get_training_fsv(cm, namemode=False) >>> result = ('(tp_fsv, tn_fsv) = %s' % (ut.repr2((tp_fsv, tn_fsv), nl=1),)) >>> print(result) """ if namemode: tp_idxs, tn_idxs = get_topname_training_idxs(cm, num=num) else: tp_idxs, tn_idxs = get_topannot_training_idxs(cm, num=num) # Keep only the top scoring half of the feature matches # top_percent = None if top_percent is not None: cm_orig = cm #cm_orig.assert_self(qreq_) tophalf_indicies = [ ut.take_percentile(fs.argsort()[::-1], top_percent) for fs in cm.get_fsv_prod_list() ] cm = cm_orig.take_feature_matches(tophalf_indicies, keepscores=True) assert np.all(cm_orig.daid_list.take(tp_idxs) == cm.daid_list.take(tp_idxs)) assert np.all(cm_orig.daid_list.take(tn_idxs) == cm.daid_list.take(tn_idxs)) #cm.assert_self(qreq_) tp_fsv = np.vstack(ut.take(cm.fsv_list, tp_idxs)) tn_fsv = np.vstack(ut.take(cm.fsv_list, tn_idxs)) return tp_fsv, tn_fsv
def simplify_graph(graph): """ strips out everything but connectivity Args: graph (nx.Graph): Returns: nx.Graph: new_graph CommandLine: python3 -m utool.util_graph simplify_graph --show python2 -m utool.util_graph simplify_graph --show python2 -c "import networkx as nx; print(nx.__version__)" python3 -c "import networkx as nx; print(nx.__version__)" Example: >>> # ENABLE_DOCTEST >>> from utool.util_graph import * # NOQA >>> import utool as ut >>> import networkx as nx >>> graph = nx.DiGraph([('a', 'b'), ('a', 'c'), ('a', 'e'), >>> ('a', 'd'), ('b', 'd'), ('c', 'e'), >>> ('d', 'e'), ('c', 'e'), ('c', 'd')]) >>> new_graph = simplify_graph(graph) >>> result = ut.repr2(list(new_graph.edges())) >>> #adj_list = sorted(list(nx.generate_adjlist(new_graph))) >>> #result = ut.repr2(adj_list) >>> print(result) [(0, 1), (0, 2), (0, 3), (0, 4), (1, 3), (2, 3), (2, 4), (3, 4)] ['0 1 2 3 4', '1 3 4', '2 4', '3', '4 3'] """ import utool as ut nodes = sorted(list(graph.nodes())) node_lookup = ut.make_index_lookup(nodes) if graph.is_multigraph(): edges = list(graph.edges(keys=True)) else: edges = list(graph.edges()) new_nodes = ut.take(node_lookup, nodes) if graph.is_multigraph(): new_edges = [(node_lookup[e[0]], node_lookup[e[1]], e[2], {}) for e in edges] else: new_edges = [(node_lookup[e[0]], node_lookup[e[1]]) for e in edges] cls = graph.__class__ new_graph = cls() new_graph.add_nodes_from(new_nodes) new_graph.add_edges_from(new_edges) return new_graph
def get_prop(self, attrname, idxs=None): """ Caching getter """ if attrname not in self.attrs: self.attrs[attrname] = [None for _ in range(len(self))] prop_list = self.attrs[attrname] if idxs is None: idxs = list(range(len(prop_list))) props = prop_list else: props = ut.take(prop_list, idxs) miss_flags = ut.flag_None_items(props) if any(miss_flags): miss_idxs = ut.compress(idxs, miss_flags) miss_fpaths = self._abs(ut.take(self.rel_fpath_list, miss_idxs)) miss_iter = getattr(self, '_' + attrname)(miss_fpaths) miss_iter = ut.ProgIter(miss_iter, length=len(miss_idxs), label='Compute %s' % (attrname,)) for idx, val in zip(miss_idxs, miss_iter): prop_list[idx] = val props = ut.take(prop_list, idxs) return props
def execute(request, parent_rowids=None, use_cache=None, postprocess=True): """ HACKY REIMPLEMENTATION """ ut.colorprint('[req] Executing request %s' % (request,), 'yellow') table = request.depc[request.tablename] if use_cache is None: use_cache = not ut.get_argflag('--nocache') if parent_rowids is None: parent_rowids = request.parent_rowids else: print('given %d specific parent_rowids' % (len(parent_rowids),)) # vsone hack (i,j) same as (j,i) if request._symmetric: import vtool as vt directed_edges = np.array(parent_rowids) undirected_edges = vt.to_undirected_edges(directed_edges) edge_ids = vt.compute_unique_data_ids(undirected_edges) unique_rows, unique_rowx, inverse_idx = np.unique(edge_ids, return_index=True, return_inverse=True) parent_rowids_ = ut.take(parent_rowids, unique_rowx) else: parent_rowids_ = parent_rowids # Compute and cache any uncomputed results rowids = table.get_rowid(parent_rowids_, config=request, recompute=not use_cache) # Load all results result_list = table.get_row_data(rowids) if request._symmetric: result_list = ut.take(result_list, inverse_idx) if postprocess and hasattr(request, 'postprocess_execute'): print('Converting results') result_list = request.postprocess_execute(parent_rowids, result_list) pass return result_list