def update_registry(drive): print('Updating registered files in %r' % (drive,)) # Update existing files fpath_exists_list = list(map(exists, ut.ProgIter(drive.fpath_list, 'checkexist fpath', freq=1000))) dpath_exists_list = list(map(exists, ut.ProgIter(drive.dpath_list, 'checkexist dpath', freq=1000))) if all(fpath_exists_list): print('No change in file structure') else: print('%d/%d files no longer exist' % ( len(drive.fpath_list) - sum(fpath_exists_list), len(drive.fpath_list))) removed_fpaths = ut.compress(drive.fpath_list, ut.not_list(fpath_exists_list)) print('removed_fpaths = %s' % (ut.list_str(removed_fpaths),)) if all(dpath_exists_list): print('No change in dpath structure') else: print('%d/%d dirs no longer exist' % ( len(drive.dpath_list) - sum(dpath_exists_list), len(drive.dpath_list))) removed_dpaths = ut.compress( drive.dpath_list, ut.not_list(dpath_exists_list)) print('removed_dpaths = %s' % (ut.list_str(removed_dpaths),)) drive.fpath_list = ut.compress(drive.fpath_list, fpath_exists_list) drive.dpath_list = ut.compress(drive.dpath_list, dpath_exists_list) drive.cache.save('fpath_list', drive.fpath_list) drive.cache.save('dpath_list', drive.dpath_list)
def color_by_nids(graph, unique_nids=None, ibs=None, nid2_color_=None): """ Colors edges and nodes by nid """ # TODO use ut.color_nodes import plottool as pt ensure_graph_nid_labels(graph, unique_nids, ibs=ibs) node_to_nid = nx.get_node_attributes(graph, 'nid') unique_nids = ut.unique(node_to_nid.values()) ncolors = len(unique_nids) if (ncolors) == 1: unique_colors = [pt.UNKNOWN_PURP] else: if nid2_color_ is not None: unique_colors = pt.distinct_colors(ncolors + len(nid2_color_) * 2) else: unique_colors = pt.distinct_colors(ncolors) # Find edges and aids strictly between two nids nid_to_color = dict(zip(unique_nids, unique_colors)) if nid2_color_ is not None: # HACK NEED TO ENSURE COLORS ARE NOT REUSED nid_to_color.update(nid2_color_) edge_aids = list(graph.edges()) edge_nids = ut.unflat_take(node_to_nid, edge_aids) flags = [nids[0] == nids[1] for nids in edge_nids] flagged_edge_aids = ut.compress(edge_aids, flags) flagged_edge_nids = ut.compress(edge_nids, flags) flagged_edge_colors = [nid_to_color[nids[0]] for nids in flagged_edge_nids] edge_to_color = dict(zip(flagged_edge_aids, flagged_edge_colors)) node_to_color = ut.map_dict_vals(ut.partial(ut.take, nid_to_color), node_to_nid) nx.set_edge_attributes(graph, 'color', edge_to_color) nx.set_node_attributes(graph, 'color', node_to_color)
def update_registry(drive): print('Updating registered files in %r' % (drive, )) # Update existing files fpath_exists_list = list( map(exists, ut.ProgIter(drive.fpath_list, 'checkexist fpath', freq=1000))) dpath_exists_list = list( map(exists, ut.ProgIter(drive.dpath_list, 'checkexist dpath', freq=1000))) if all(fpath_exists_list): print('No change in file structure') else: print('%d/%d files no longer exist' % (len(drive.fpath_list) - sum(fpath_exists_list), len(drive.fpath_list))) removed_fpaths = ut.compress(drive.fpath_list, ut.not_list(fpath_exists_list)) print('removed_fpaths = %s' % (ut.repr2(removed_fpaths), )) if all(dpath_exists_list): print('No change in dpath structure') else: print('%d/%d dirs no longer exist' % (len(drive.dpath_list) - sum(dpath_exists_list), len(drive.dpath_list))) removed_dpaths = ut.compress(drive.dpath_list, ut.not_list(dpath_exists_list)) print('removed_dpaths = %s' % (ut.repr2(removed_dpaths), )) drive.fpath_list = ut.compress(drive.fpath_list, fpath_exists_list) drive.dpath_list = ut.compress(drive.dpath_list, dpath_exists_list) drive.cache.save('fpath_list', drive.fpath_list) drive.cache.save('dpath_list', drive.dpath_list)
def get_valid_imgsetids(ibs, min_num_gids=0, processed=None, shipped=None, is_occurrence=None): r""" FIX NAME imgagesetids Returns: list_ (list): list of all imageset ids RESTful: Method: GET URL: /api/imageset/ """ imgsetid_list = ibs._get_all_imgsetids() if min_num_gids > 0: num_gids_list = ibs.get_imageset_num_gids(imgsetid_list) flag_list = [num_gids >= min_num_gids for num_gids in num_gids_list] imgsetid_list = ut.compress(imgsetid_list, flag_list) if processed is not None: flag_list = ibs.get_imageset_processed_flags(imgsetid_list) isvalid_list = [flag == 1 if processed else flag == 0 for flag in flag_list] imgsetid_list = ut.compress(imgsetid_list, isvalid_list) if shipped is not None: flag_list = ibs.get_imageset_shipped_flags(imgsetid_list) isvalid_list = [flag == 1 if shipped else flag == 0 for flag in flag_list] imgsetid_list = ut.compress(imgsetid_list, isvalid_list) if is_occurrence is not None: flag_list = ibs.get_imageset_isoccurrence(imgsetid_list) isvalid_list = [flag == is_occurrence for flag in flag_list] imgsetid_list = ut.compress(imgsetid_list, isvalid_list) return imgsetid_list
def _load_singles(qreq_): # Find existing cached chip matches # Try loading as many as possible fpath_list = qreq_.get_chipmatch_fpaths(qreq_.qaids) exists_flags = [exists(fpath) for fpath in fpath_list] qaids_hit = ut.compress(qreq_.qaids, exists_flags) fpaths_hit = ut.compress(fpath_list, exists_flags) # First, try a fast reload assuming no errors fpath_iter = ut.ProgIter( fpaths_hit, length=len(fpaths_hit), enabled=len(fpaths_hit) > 1, label='loading cache hits', adjust=True, freq=1, ) try: qaid_to_hit = { qaid: chip_match.ChipMatch.load_from_fpath(fpath, verbose=False) for qaid, fpath in zip(qaids_hit, fpath_iter) } except chip_match.NeedRecomputeError as ex: # Fallback to a slow reload ut.printex(ex, 'Some cached results need to recompute', iswarning=True) qaid_to_hit = _load_singles_fallback(fpaths_hit) return qaid_to_hit
def nx_from_matrix(weight_matrix, nodes=None, remove_self=True): import networkx as nx import utool as ut import numpy as np if nodes is None: nodes = list(range(len(weight_matrix))) weight_list = weight_matrix.ravel() flat_idxs_ = np.arange(weight_matrix.size) multi_idxs_ = np.unravel_index(flat_idxs_, weight_matrix.shape) # Remove 0 weight edges flags = np.logical_not(np.isclose(weight_list, 0)) weight_list = ut.compress(weight_list, flags) multi_idxs = ut.compress(list(zip(*multi_idxs_)), flags) edge_list = ut.lmap(tuple, ut.unflat_take(nodes, multi_idxs)) if remove_self: flags = [e1 != e2 for e1, e2 in edge_list] edge_list = ut.compress(edge_list, flags) weight_list = ut.compress(weight_list, flags) graph = nx.Graph() graph.add_nodes_from(nodes) graph.add_edges_from(edge_list) label_list = ['%.2f' % w for w in weight_list] nx.set_edge_attributes(graph, 'weight', dict(zip(edge_list, weight_list))) nx.set_edge_attributes(graph, 'label', dict(zip(edge_list, label_list))) return graph
def set_annot_lblannot_from_value(ibs, aid_list, value_list, _lbltype, ensure=True): """ Associates the annot and lblannot of a specific type and value Adds the lblannot if it doesnt exist. Wrapper around convenience function for set_annot_from_lblannot_rowid """ assert value_list is not None assert _lbltype is not None if ensure: pass # a value consisting of an empty string or all spaces is set to the default DEFAULT_VALUE = const.KEY_DEFAULTS[_lbltype] EMPTY_KEY = const.EMPTY_KEY # setting a name to DEFAULT_VALUE or EMPTY is equivalent to unnaming it value_list_ = [DEFAULT_VALUE if value.strip() == EMPTY_KEY else value for value in value_list] notdefault_list = [value != DEFAULT_VALUE for value in value_list_] aid_list_to_delete = ut.get_dirty_items(aid_list, notdefault_list) # Set all the valid valids aids_to_set = ut.compress(aid_list, notdefault_list) values_to_set = ut.compress(value_list_, notdefault_list) ibs.delete_annot_relations_oftype(aid_list_to_delete, _lbltype) # remove the relationships that have now been unnamed # Convert names into lblannot_rowid # FIXME: This function should not be able to set label realationships # to labels that have not been added!! # This is an inefficient way of getting lblannot_rowids! lbltype_rowid_list = [ibs.lbltype_ids[_lbltype]] * len(values_to_set) # auto ensure lblannot_rowid_list = ibs.add_lblannots(lbltype_rowid_list, values_to_set) # Call set_annot_from_lblannot_rowid to finish the conditional adding ibs.set_annot_lblannot_from_rowid(aids_to_set, lblannot_rowid_list, _lbltype)
def show_function_usage(fname, funcname_list, dpath_list): # Check to see for function usage funcname_list = [r'\b%s\b' % (funcname.strip(),) for funcname in funcname_list if len(funcname) > 0] flagged_funcnames = [] for funcname in funcname_list: found_filestr_list, found_lines_list, found_lxs_list = ut.grep([funcname], dpath_list=dpath_list) total = 0 for lines in found_lines_list: total += len(lines) funcname_ = funcname.replace('\\b', '') print(funcname_ + ' ' + str(total)) if total == 1: flagged_funcnames.append(funcname_) # See where external usage is isexternal_list = [fname == fname_ for fname_ in found_filestr_list] external_filestr_list = ut.compress(found_filestr_list, isexternal_list) external_lines_list = ut.compress(found_lines_list, isexternal_list) #external_lxs_list = ut.compress(found_lxs_list, isexternal_list) if len(external_filestr_list) == 0: print(' no external usage') else: for filename, lines in zip(external_filestr_list, external_lines_list): print(' * filename=%r' % (filename,)) print(ut.list_str(lines)) #print(ut.list_str(list(zip(external_filestr_list, external_lines_list)))) print('----------') print('flagged:') print('\n'.join(flagged_funcnames))
def set_annot_lblannot_from_value(ibs, aid_list, value_list, _lbltype, ensure=True): """ Associates the annot and lblannot of a specific type and value Adds the lblannot if it doesnt exist. Wrapper around convenience function for set_annot_from_lblannot_rowid """ assert value_list is not None assert _lbltype is not None if ensure: pass # a value consisting of an empty string or all spaces is set to the default DEFAULT_VALUE = const.KEY_DEFAULTS[_lbltype] EMPTY_KEY = const.EMPTY_KEY # setting a name to DEFAULT_VALUE or EMPTY is equivalent to unnaming it value_list_ = [ DEFAULT_VALUE if value.strip() == EMPTY_KEY else value for value in value_list ] notdefault_list = [value != DEFAULT_VALUE for value in value_list_] aid_list_to_delete = ut.get_dirty_items(aid_list, notdefault_list) # Set all the valid valids aids_to_set = ut.compress(aid_list, notdefault_list) values_to_set = ut.compress(value_list_, notdefault_list) ibs.delete_annot_relations_oftype(aid_list_to_delete, _lbltype) # remove the relationships that have now been unnamed # Convert names into lblannot_rowid # FIXME: This function should not be able to set label realationships # to labels that have not been added!! # This is an inefficient way of getting lblannot_rowids! lbltype_rowid_list = [ibs.lbltype_ids[_lbltype]] * len(values_to_set) # auto ensure lblannot_rowid_list = ibs.add_lblannots(lbltype_rowid_list, values_to_set) # Call set_annot_from_lblannot_rowid to finish the conditional adding ibs.set_annot_lblannot_from_rowid(aids_to_set, lblannot_rowid_list, _lbltype)
def update_imageset_info(ibs, imageset_rowid_list): r""" sets start and end time for imagesets FIXME: should not need to bulk update, should be handled as it goes RESTful: Method: PUT URL: /api/imageset/info/ Example: >>> # DOCTEST_DISABLE >>> imageset_rowid_list = ibs.get_valid_imgsetids() """ gids_list_ = ibs.get_imageset_gids(imageset_rowid_list) hasgids_list = [len(gids) > 0 for gids in gids_list_] gids_list = ut.compress(gids_list_, hasgids_list) imgsetid_list = ut.compress(imageset_rowid_list, hasgids_list) unixtimes_list = ibs.unflat_map(ibs.get_image_unixtime, gids_list) # TODO: replace -1's with nans and do nanmin imageset_start_time_posix_list = [ min(unixtimes) for unixtimes in unixtimes_list ] imageset_end_time_posix_list = [ max(unixtimes) for unixtimes in unixtimes_list ] ibs.set_imageset_start_time_posix(imgsetid_list, imageset_start_time_posix_list) ibs.set_imageset_end_time_posix(imgsetid_list, imageset_end_time_posix_list)
def filter_part_set( ibs, part_rowid_list, include_only_aid_list=None, is_staged=False, viewpoint='no-filter', minqual=None, ): # -- valid part_rowid filtering -- # filter by is_staged if is_staged is True: # corresponding unoptimized hack for is_staged flag_list = ibs.get_part_staged_flags(part_rowid_list) part_rowid_list = ut.compress(part_rowid_list, flag_list) elif is_staged is False: flag_list = ibs.get_part_staged_flags(part_rowid_list) part_rowid_list = ut.filterfalse_items(part_rowid_list, flag_list) if include_only_aid_list is not None: gid_list = ibs.get_part_gids(part_rowid_list) is_valid_gid = [gid in include_only_aid_list for gid in gid_list] part_rowid_list = ut.compress(part_rowid_list, is_valid_gid) if viewpoint != 'no-filter': viewpoint_list = ibs.get_part_viewpoints(part_rowid_list) is_valid_viewpoint = [viewpoint == flag for flag in viewpoint_list] part_rowid_list = ut.compress(part_rowid_list, is_valid_viewpoint) if minqual is not None: part_rowid_list = ibs.filter_part_rowids_to_quality(part_rowid_list, minqual, unknown_ok=True) part_rowid_list = sorted(part_rowid_list) return part_rowid_list
def hardcase_review_gen(infr): """ Subiterator for hardcase review Re-review non-confident edges that vsone did not classify correctly """ infr.print('==============================', color='white') infr.print('--- HARDCASE PRIORITY LOOP ---', color='white') verifiers = infr.learn_evaluation_verifiers() verif = verifiers['match_state'] edges_ = list(infr.edges()) real_ = list(infr.edge_decision_from(edges_)) flags_ = [r in {POSTV, NEGTV, INCMP} for r in real_] real = ut.compress(real_, flags_) edges = ut.compress(edges_, flags_) hardness = 1 - verif.easiness(edges, real) if True: df = pd.DataFrame({'edges': edges, 'real': real}) df['hardness'] = hardness pred = verif.predict(edges) df['pred'] = pred.values df.sort_values('hardness', ascending=False) infr.print('hardness analysis') infr.print(str(df)) infr.print('infr status: ' + ut.repr4(infr.status())) # Don't re-review anything that was confidently reviewed # CONFIDENCE = const.CONFIDENCE # CODE_TO_INT = CONFIDENCE.CODE_TO_INT.copy() # CODE_TO_INT[CONFIDENCE.CODE.UNKNOWN] = 0 # conf = ut.take(CODE_TO_INT, infr.gen_edge_values( # 'confidence', edges, on_missing='default', # default=CONFIDENCE.CODE.UNKNOWN)) # This should only be run with certain params assert not infr.params['autoreview.enabled'] assert not infr.params['redun.enabled'] assert not infr.params['ranking.enabled'] assert infr.params['inference.enabled'] # const.CONFIDENCE.CODE.PRETTY_SURE if infr.params['queue.conf.thresh'] is None: # != 'pretty_sure': infr.print('WARNING: should queue.conf.thresh = "pretty_sure"?') # work around add_candidate_edges infr.prioritize(metric='hardness', edges=edges, scores=hardness) infr.set_edge_attrs('hardness', ut.dzip(edges, hardness)) for _ in infr._inner_priority_gen(use_refresh=False): yield _
def fuzzy_filter_columns(self, fuzzy_headers): import utool as ut col_flags = ut.filterflags_general_tags( self.header_tags, logic='or', in_any=fuzzy_headers) self.header = ut.compress(self.header, col_flags) self.header_tags = ut.compress(self.header_tags, col_flags) self.row_data = ut.listT(ut.compress(ut.listT(self.row_data), col_flags)) if self.short_header is not None: self.short_header = ut.compress(self.short_header, col_flags)
def fuzzy_filter_columns(self, fuzzy_headers): import utool as ut col_flags = ut.filterflags_general_tags(self.header_tags, logic='or', in_any=fuzzy_headers) self.header = ut.compress(self.header, col_flags) self.header_tags = ut.compress(self.header_tags, col_flags) self.row_data = ut.listT( ut.compress(ut.listT(self.row_data), col_flags)) if self.short_header is not None: self.short_header = ut.compress(self.short_header, col_flags)
def compress_rows(self, flags, with_header=True, inplace=True): if not inplace: import copy self = copy.deepcopy(self) import utool as ut if with_header: assert flags[0] is True self.row_data = ut.compress(self.row_data, flags) else: self.row_data = self.row_data[0:1] + ut.compress(self.row_data[1:], flags) return self
def compress_rows(self, flags, with_header=True, inplace=True): if not inplace: import copy self = copy.deepcopy(self) import utool as ut if with_header: assert flags[0] is True self.row_data = ut.compress(self.row_data, flags) else: self.row_data = self.row_data[0:1] + ut.compress( self.row_data[1:], flags) return self
def possible_mana_combinations(land_list, deck=None): """ CommandLine: python -m mtgmonte.mtgutils --test-possible_mana_combinations Example: >>> # ENABLE_DOCTEST >>> from mtgmonte.mtgutils import * # NOQA >>> from mtgmonte import mtgobjs >>> deck = mtgobjs.Deck(mtgobjs.load_cards(['Tropical Island', 'Sunken Hollow', 'Island'])) >>> land_list = mtgobjs.load_cards(['Ancient Tomb', 'Island', 'Flooded Strand', 'Flooded Strand', 'Shivan Reef']) >>> card = land_list[-1] >>> mana_combos = possible_mana_combinations(land_list, deck) >>> result = (ut.repr2(mana_combos, nl=1, strvals=True, nobraces=True)) >>> print(result) ({CC}, {U}, {G}, {U}, {C}), ({CC}, {U}, {G}, {B}, {C}), ({CC}, {U}, {U}, {U}, {C}), ({CC}, {U}, {U}, {B}, {C}), ({CC}, {U}, {G}, {U}, {R}), ({CC}, {U}, {G}, {B}, {R}), ({CC}, {U}, {U}, {U}, {R}), ({CC}, {U}, {U}, {B}, {R}), """ from mtgmonte import mtgobjs avail_mana = [land.mana_potential2(deck=deck, recurse=False) for land in land_list] avail_mana = filter(len, avail_mana) mana_combos1 = list(ut.iprod(*avail_mana)) # Encode the idea that two fetches cant fetch the same land non_class1 = [[c for c in co if not isinstance(c, six.string_types)] for co in mana_combos1] flags = [len(co) == 0 or len(set(co)) == len(co) for co in non_class1] mana_combos2 = ut.compress(mana_combos1, flags) mana_combos3 = [ [[c] if isinstance(c, mtgobjs.ManaSet) else c.mana_potential2(deck=deck) for c in co] for co in mana_combos2 ] unflat_combos3 = [list(ut.iprod(*co)) for co in mana_combos3] mana_combos4 = ut.flatten(unflat_combos3) # mana_combos4 = [reduce(operator.add, m) for m in mana_combos4] # z = reduce(operator.add, m) # import utool # utool.embed() # avail_mana = [land.mana_potential(deck=deck) for land in land_list] # avail_mana = filter(len, avail_mana) # mana_combos4 = list(ut.iprod(*avail_mana)) combo_ids = [tuple(sorted(x)) for x in mana_combos4] flags = ut.flag_unique_items(combo_ids) mana_combos = ut.compress(mana_combos4, flags) # mana_combos = list(map(tuple, [''.join(c) for c in mana_combos])) return mana_combos
def get_name_exemplar_aids(ibs, nid_list): r""" Returns: list_ (list): a list of list of cids in each name CommandLine: python -m ibeis.control.manual_name_funcs --test-get_name_exemplar_aids RESTful: Method: GET URL: /api/name/exemplar_aids/ Example: >>> # ENABLE_DOCTEST >>> from ibeis.control.manual_name_funcs import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> aid_list = ibs.get_valid_aids() >>> nid_list = ibs.get_annot_name_rowids(aid_list) >>> exemplar_aids_list = ibs.get_name_exemplar_aids(nid_list) >>> result = exemplar_aids_list >>> print(result) [[], [2, 3], [2, 3], [], [5, 6], [5, 6], [7], [8], [], [10], [], [12], [13]] """ # Get all annot ids for each name aids_list = ibs.get_name_aids(nid_list, enable_unknown_fix=True) # Flag any annots that are not exemplar and remove them flags_list = ibsfuncs.unflat_map(ibs.get_annot_exemplar_flags, aids_list) exemplar_aids_list = [ut.compress(aids, flags) for aids, flags in zip(aids_list, flags_list)] return exemplar_aids_list
def get_valid_nids(ibs, imgsetid=None, filter_empty=False, min_pername=None): r""" Returns: list_ (list): all valid names with at least one animal (does not include unknown names) RESTful: Method: GET URL: /api/name/ """ if imgsetid is None: _nid_list = ibs._get_all_known_name_rowids() else: _nid_list = ibs.get_imageset_nids(imgsetid) # HACK FOR UNKNOWN. Makes things crash #_nid_list += [0] nid_list = _nid_list if filter_empty: min_pername = 1 if min_pername is None else max(min_pername, 1) if min_pername is not None: nAnnot_list = ibs.get_name_num_annotations(nid_list) flag_list = np.array(nAnnot_list) >= min_pername nid_list = ut.compress(nid_list, flag_list) return nid_list
def get_annot_alrids_oftype(ibs, aid_list, lbltype_rowid, configid=None): """ Get all the relationship ids belonging to the input annotations where the relationship ids are filtered to be only of a specific lbltype/category/type """ alrids_list = ibs.get_annot_alrids(aid_list, configid=configid) # Get lblannot_rowid of each relationship lblannot_rowids_list = ibsfuncs.unflat_map(ibs.get_alr_lblannot_rowids, alrids_list) # Get the type of each lblannot lbltype_rowids_list = ibsfuncs.unflat_map(ibs.get_lblannot_lbltypes_rowids, lblannot_rowids_list) # only want the nids of individuals, not species, for example valids_list = [[typeid == lbltype_rowid for typeid in rowids] for rowids in lbltype_rowids_list] alrids_list = [ut.compress(alrids, valids) for alrids, valids in zip(alrids_list, valids_list)] if configid is None: def resolution_func_first(alrid_list): return [ alrid_list[0] ] def resolution_func_lowest_config(alrid_list): config_rowid_list = ibs.get_alr_config(alrid_list) temp = sorted(list(zip(config_rowid_list, alrid_list))) return [ temp[0][0] ] alrids_list = [ resolution_func_first(alrid_list) if len(alrid_list) > 1 else alrid_list for alrid_list in alrids_list ] assert all([len(alrid_list) < 2 for alrid_list in alrids_list]),\ ("More than one type per lbltype. ALRIDS: " + str(alrids_list) + ", ROW: " + str(lbltype_rowid) + ", KEYS:" + str(ibs.lbltype_ids)) return alrids_list
def shark_misc(): import ibeis ibs = ibeis.opendb('WS_ALL') aid_list = ibs.get_valid_aids() flag_list = ibs.get_annot_been_adjusted(aid_list) adjusted_aids = ut.compress(aid_list, flag_list) return adjusted_aids
def get_valid_nids(ibs, eid=None, filter_empty=False, min_pername=None): r""" Returns: list_ (list): all valid names with at least one animal (does not include unknown names) RESTful: Method: GET URL: /api/name/ """ if eid is None: _nid_list = ibs._get_all_known_name_rowids() else: _nid_list = ibs.get_encounter_nids(eid) # HACK FOR UNKNOWN. Makes things crash #_nid_list += [0] nid_list = _nid_list if filter_empty: min_pername = 1 if min_pername is None else max(min_pername, 1) if min_pername is not None: nAnnot_list = ibs.get_name_num_annotations(nid_list) flag_list = np.array(nAnnot_list) >= min_pername nid_list = ut.compress(nid_list, flag_list) return nid_list
def debug_depcache(ibs): r""" CommandLine: python -m ibeis_flukematch.plugin --exec-debug_depcache python -m ibeis_flukematch.plugin --exec-debug_depcache --show --no-cnn python -m ibeis_flukematch.plugin --exec-debug_depcache --clear-all-depcache --db humbpacks python -m ibeis_flukematch.plugin --exec-debug_depcache --show --no-cnn --db humpbacks python -m ibeis_flukematch.plugin --exec-preproc_notch_tips --db humpbacks --no-cnn --show Example: >>> # SCRIPT >>> from ibeis_flukematch.plugin import * # NOQA >>> ibs = ibeis.opendb(defaultdb='PZ_MTEST') >>> debug_depcache(ibs) >>> ut.show_if_requested() """ print(ibs.depc) nas_notch_deps = ibs.depc.get_dependencies('Has_Notch') print('nas_notch_deps = %r' % (nas_notch_deps,)) te_deps = ibs.depc.get_dependencies('Trailing_Edge') print('te_deps = %r' % (te_deps,)) notch_tip_deps = ibs.depc.get_dependencies('Notch_Tips') print('notch_tip_deps = %r' % (notch_tip_deps,)) ibs.depc.print_schemas() try: ibs.depc.show_graph() except Exception as ex: ut.printex(ex, iswarning=True) all_aids = ibs.get_valid_aids() isvalid = ibs.depc.get('Has_Notch', all_aids, 'flag') aid_list = ut.compress(all_aids, isvalid) aid_list = aid_list[0:10] ibs.depc.print_config_tables()
def delete_empty_eids(ibs): """ Removes encounters without images Args: ibs (IBEISController): ibeis controller object CommandLine: python -m ibeis.control.manual_egrelate_funcs --test-delete_empty_eids Example: >>> # ENABLE_DOCTEST >>> from ibeis.control.manual_egrelate_funcs import * # NOQA >>> import ibeis >>> # build test data >>> ibs = ibeis.opendb('testdb1') >>> # execute function >>> result = ibs.delete_empty_eids() >>> # verify results >>> print(result) """ eid_list = ibs.get_valid_eids(min_num_gids=0) nGids_list = ibs.get_encounter_num_gids(eid_list) is_invalid = [nGids == 0 for nGids in nGids_list] invalid_eids = ut.compress(eid_list, is_invalid) ibs.delete_encounters(invalid_eids)
def compress(self, flags, inplace=False): subarr = self.kparr.compress(flags, axis=0) info = { key: ut.compress(val, flags) for key, val in self.info.items() } return Keypoints(subarr, info)
def find_close_incorrect_match(ibs, qaids): use_cache = False save_qcache = False cfgdict_vsmany = dict( index_method='single', pipeline_root='vsmany', ) qres_vsmany_list, qreq_vsmany_ = ibs.query_chips(qaids, ibs.get_valid_aids(), cfgdict=cfgdict_vsmany, return_request=True, use_cache=use_cache, save_qcache=save_qcache, verbose=True) qres_vsmany = qres_vsmany_list[0] qres_vsmany.ishow_top(ibs) top_aids = qres_vsmany.get_top_aids() top_nids = ibs.get_annot_nids(top_aids) qaid = qaids[0] qnid = ibs.get_annot_nids(qaid) is_groundfalse = [nid != qnid for nid in top_nids] top_gf_aids = ut.compress(top_aids, is_groundfalse) #top_gt_aids = ut.filterfalse_items(top_aids, is_groundfalse) top_gf_vuuids = ibs.get_annot_visual_uuids(top_gf_aids) qvuuid = ibs.get_annot_visual_uuids(qaid) gf_mapping = {qvuuid: top_gf_vuuids[0:1]} print('gf_mapping = ' + ut.dict_str(gf_mapping)) pass
def _estimate_threshold(model, method=None, curve=None): """ import plottool as pt idx3 = vt.find_elbow_point(curve[idx1:idx2 + 1]) + idx1 pt.plot(curve) pt.plot(idx1, curve[idx1], 'bo') pt.plot(idx2, curve[idx2], 'ro') pt.plot(idx3, curve[idx3], 'go') """ if curve is None: isvalid = ~np.isnan(model.edge_weights) curve = sorted(ut.compress(model.edge_weights, isvalid)) thresh = estimate_threshold(curve, method) #if len(curve) == 0: # return 0 #if method is None: # method = 'mean' #if method == 'mean': # thresh = np.mean(curve) #elif method == 'elbow': # idx1 = vt.find_elbow_point(curve) # idx2 = vt.find_elbow_point(curve[idx1:]) + idx1 # thresh = curve[idx2] #else: # raise ValueError('method = %r' % (method,)) return thresh
def find_maxval_spell_sequence(player): # sim land in play # really need available mana from mtgmonte import mtgutils land_in_play = player.get_cards_in_play(["Land"]) nonland_in_hand = player.get_cards_in_hand(["Land"], invert=True) land_list = land_in_play spell_list = nonland_in_hand max_avail_cmc = mtgutils.get_max_avail_cmc(land_list, deck=player.deck) cmc_feasible_sequences = mtgutils.get_cmc_feasible_sequences(spell_list, max_avail_cmc) if len(cmc_feasible_sequences) == 0: sequence = [] value = 0 else: mana_combos = mtgutils.possible_mana_combinations(land_list, player.deck) flags = [mtgutils.can_cast(spell_sequence, mana_combos) for spell_sequence in cmc_feasible_sequences] feasible_sequences = ut.compress(cmc_feasible_sequences, flags) if len(feasible_sequences) == 0: sequence = [] value = 0 else: # Find best value in feasible solutions value_list = [sum([card.get_goldfish_value() for card in combo]) for combo in feasible_sequences] index = ut.list_argmax(value_list) sequence = feasible_sequences[index] value = len(sequence) return sequence, value
def get_unconverted_hsdbs(workdir=None): r""" Args: workdir (None): (default = None) CommandLine: python -m wbia.dbio.ingest_hsdb --test-get_unconverted_hsdbs Example: >>> # SCRIPT >>> from wbia.dbio.ingest_hsdb import * # NOQA >>> workdir = None >>> result = get_unconverted_hsdbs(workdir) >>> print(result) """ import os import numpy as np if workdir is None: workdir = sysres.get_workdir() dbname_list = os.listdir(workdir) dbpath_list = np.array([join(workdir, name) for name in dbname_list]) needs_convert = list(map(check_unconverted_hsdb, dbpath_list)) needs_convert_hsdbs = ut.compress(dbpath_list, needs_convert) return needs_convert_hsdbs
def delete_empty_imgsetids(ibs): """ Removes imagesets without images Args: ibs (IBEISController): wbia controller object CommandLine: python -m wbia.control.manual_gsgrelate_funcs --test-delete_empty_imgsetids Example: >>> # ENABLE_DOCTEST >>> from wbia.control.manual_gsgrelate_funcs import * # NOQA >>> import wbia >>> # build test data >>> ibs = wbia.opendb('testdb1') >>> # execute function >>> result = ibs.delete_empty_imgsetids() >>> # verify results >>> print(result) """ imgsetid_list = ibs.get_valid_imgsetids(min_num_gids=0) nGids_list = ibs.get_imageset_num_gids(imgsetid_list) is_invalid = [nGids == 0 for nGids in nGids_list] invalid_imgsetids = ut.compress(imgsetid_list, is_invalid) ibs.delete_imagesets(invalid_imgsetids)
def get_annot_alrids_oftype(ibs, aid_list, lbltype_rowid): """ Get all the relationship ids belonging to the input annotations where the relationship ids are filtered to be only of a specific lbltype/category/type """ alrids_list = ibs.get_annot_alrids(aid_list) # Get lblannot_rowid of each relationship lblannot_rowids_list = ibsfuncs.unflat_map(ibs.get_alr_lblannot_rowids, alrids_list) # Get the type of each lblannot lbltype_rowids_list = ibsfuncs.unflat_map(ibs.get_lblannot_lbltypes_rowids, lblannot_rowids_list) # only want the nids of individuals, not species, for example valids_list = [[typeid == lbltype_rowid for typeid in rowids] for rowids in lbltype_rowids_list] alrids_list = [ ut.compress(alrids, valids) for alrids, valids in zip(alrids_list, valids_list) ] def resolution_func_first(alrid_list): return [alrid_list[0]] alrids_list = [ resolution_func_first(alrid_list) if len(alrid_list) > 1 else alrid_list for alrid_list in alrids_list ] assert all([ len(alrid_list) < 2 for alrid_list in alrids_list ]), ('More than one type per lbltype. ALRIDS: ' + str(alrids_list) + ', ROW: ' + str(lbltype_rowid) + ', KEYS:' + str(ibs.lbltype_ids)) return alrids_list
def debug_depcache(ibs): r""" CommandLine: python -m ibeis_flukematch.plugin --exec-debug_depcache python -m ibeis_flukematch.plugin --exec-debug_depcache --show --no-cnn python -m ibeis_flukematch.plugin --exec-debug_depcache --clear-all-depcache --db humbpacks python -m ibeis_flukematch.plugin --exec-debug_depcache --show --no-cnn --db humpbacks python -m ibeis_flukematch.plugin --exec-preproc_notch_tips --db humpbacks --no-cnn --show Example: >>> # SCRIPT >>> from ibeis_flukematch.plugin import * # NOQA >>> ibs = ibeis.opendb(defaultdb='PZ_MTEST') >>> debug_depcache(ibs) >>> ut.show_if_requested() """ print(ibs.depc) nas_notch_deps = ibs.depc.get_dependencies('Has_Notch') print('nas_notch_deps = %r' % (nas_notch_deps, )) te_deps = ibs.depc.get_dependencies('Trailing_Edge') print('te_deps = %r' % (te_deps, )) notch_tip_deps = ibs.depc.get_dependencies('Notch_Tips') print('notch_tip_deps = %r' % (notch_tip_deps, )) ibs.depc.print_schemas() try: ibs.depc.show_graph() except Exception as ex: ut.printex(ex, iswarning=True) all_aids = ibs.get_valid_aids() isvalid = ibs.depc.get('Has_Notch', all_aids, 'flag') aid_list = ut.compress(all_aids, isvalid) aid_list = aid_list[0:10] ibs.depc.print_config_tables()
def find_module_callers(): """ TODO: attempt to build a call graph between module functions to make it easy to see what can be removed and what cannot. """ import utool as ut from os.path import normpath mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_analyzer.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_all.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_organizer.py') module = ut.import_module_from_fpath(mod_fpath) user_profile = ut.ensure_user_profile() doctestables = list( ut.iter_module_doctestable(module, include_builtin=False)) grepkw = {} grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['verbose'] = True usage_map = {} for funcname, func in doctestables: print('Searching for funcname = %r' % (funcname, )) found_fpath_list, found_lines_list, found_lxs_list = ut.grep( [funcname], **grepkw) used_in = (found_fpath_list, found_lines_list, found_lxs_list) usage_map[funcname] = used_in external_usage_map = {} for funcname, used_in in usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in isexternal_flag = [ normpath(fpath) != normpath(mod_fpath) for fpath in found_fpath_list ] ext_used_in = (ut.compress(found_fpath_list, isexternal_flag), ut.compress(found_lines_list, isexternal_flag), ut.compress(found_lxs_list, isexternal_flag)) external_usage_map[funcname] = ext_used_in for funcname, used_in in external_usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in print('Calling modules: \n' + ut.repr2(ut.unique_ordered( ut.flatten([used_in[0] for used_in in external_usage_map.values()])), nl=True))
def ensure_task_probs(infr, edges): """ Ensures that probabilities are assigned to the edges. This gaurentees that infr.task_probs contains data for edges. (Currently only the primary task is actually ensured) CommandLine: python -m ibeis.algo.graph.mixin_matching ensure_task_probs Doctest: >>> from ibeis.algo.graph.mixin_matching import * >>> import ibeis >>> infr = ibeis.AnnotInference('PZ_MTEST', aids='all', >>> autoinit='staging') >>> edges = list(infr.edges())[0:3] >>> infr.load_published() >>> assert len(infr.task_probs['match_state']) == 0 >>> infr.ensure_task_probs(edges) >>> assert len(infr.task_probs['match_state']) == 3 >>> infr.ensure_task_probs(edges) >>> assert len(infr.task_probs['match_state']) == 3 Doctest: >>> from ibeis.algo.graph.mixin_matching import * >>> from ibeis.algo.graph import demo >>> infr = demo.demodata_infr(num_pccs=6, p_incon=.5, size_std=2) >>> edges = list(infr.edges()) >>> infr.ensure_task_probs(edges) >>> assert all([np.isclose(sum(p.values()), 1) >>> for p in infr.task_probs['match_state'].values()]) """ if not infr.verifiers: raise Exception('Verifiers are needed to predict probabilities') # Construct pairwise features on edges in infr primary_task = 'match_state' match_task = infr.task_probs[primary_task] need_flags = [e not in match_task for e in edges] if any(need_flags): need_edges = ut.compress(edges, need_flags) infr.print( 'There are {} edges without probabilities'.format( len(need_edges)), 1) # Only recompute for the needed edges task_probs = infr._make_task_probs(need_edges) # Store task probs in internal data structure # FIXME: this is slow for task, probs in task_probs.items(): probs_dict = probs.to_dict(orient='index') if task not in infr.task_probs: infr.task_probs[task] = probs_dict else: infr.task_probs[task].update(probs_dict) # Set edge task attribute as well infr.set_edge_attrs(task, probs_dict)
def testdata_humpbacks(): import ibeis ibs = ibeis.opendb(defaultdb='humpbacks') all_aids = ibs.get_valid_aids() isvalid = ibs.depc.get('Has_Notch', all_aids, 'flag') aid_list = ut.compress(all_aids, isvalid) aid_list = aid_list[0:10] return ibs, aid_list
def inplace_filter_results(self, filter_pat): import utool as ut self.filter_pats.append(filter_pat) # Get zipflags flags_list = self.pattern_filterflags(filter_pat) # Check to see if there are any survivors flags = ut.lmap(any, flags_list) # found_lines_list = ut.zipcompress(self.found_lines_list, flags_list) found_lxs_list = ut.zipcompress(self.found_lxs_list, flags_list) # found_fpath_list = ut.compress(self.found_fpath_list, flags) found_lines_list = ut.compress(found_lines_list, flags) found_lxs_list = ut.compress(found_lxs_list, flags) # In place modification self.found_fpath_list = found_fpath_list self.found_lines_list = found_lines_list self.found_lxs_list = found_lxs_list
def bzip(*args): """ broadcasting zip. Only broadcasts on the first dimension args = [np.array([1, 2, 3, 4]), [[1, 2, 3]]] args = [np.array([1, 2, 3, 4]), [[1, 2, 3]]] """ needs_cast = [isinstance(arg, list) for arg in args] arg_containers = [np.empty(len(arg), dtype=object) if flag else arg for arg, flag in zip(args, needs_cast)] empty_containers = ut.compress(arg_containers, needs_cast) tocast_args = ut.compress(args, needs_cast) for container, arg in zip(empty_containers, tocast_args): container[:] = arg #[a.shape for a in arg_containers] bc = np.broadcast(*arg_containers) return bc
def get_diffranks(rank_mat, qaids): """ Find rows which scored differently over the various configs FIXME: duplicated """ isdiff_flags = [not np.all(row == row[0]) for row in rank_mat] diff_aids = ut.compress(qaids, isdiff_flags) diff_rank = rank_mat.compress(isdiff_flags, axis=0) diff_qxs = np.where(isdiff_flags)[0] return diff_aids, diff_rank, diff_qxs
def convert_tests_from_ibeis_to_nose(module_list): # PARSE OUT TESTABLE DOCTESTTUPS #import utool as ut testtup_list = [] seen_ = set() topimport_list = [] for module in module_list: mod_doctest_tup = ut.get_module_doctest_tup(module=module, verbose=False, allexamples=True) enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup flags = [tup.src not in seen_ for tup in enabled_testtup_list] enabled_testtup_list = ut.compress(enabled_testtup_list, flags) testtup_list.extend(enabled_testtup_list) if len(enabled_testtup_list) > 0: topimport_list.append('from %s import * # NOQA' % (module.__name__,)) print('Found %d test tups' % (len(testtup_list))) autogen_test_src_funcs = [] #import redbaron for testtup in testtup_list: name = testtup.name num = testtup.num src = testtup.src want = testtup.want import re src = re.sub('# ENABLE_DOCTEST\n', '', src) src = re.sub('from [^*]* import \* *# NOQA\n', '', src) src = re.sub(r'from [^*]* import \*\n', '', src) src = ut.str_between(src, None, 'ut.quit_if_noshow').rstrip('\n') src = ut.str_between(src, None, 'ut.show_if_requested').rstrip('\n') # import utool # utool.embed() """ """ #flag = testtup.flag if want.endswith('\n'): want = want[:-1] if want: #src_node = redbaron.RedBaron(src) #if len(src_node.find_all('name', 'result')) > 0: # src_node.append('assert result == %r' % (want,)) if '\nresult = ' in src: src += '\nassert str(result) == %r' % (want,) func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src) autogen_test_src_funcs.append(func_src) autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n' from ibeis import tests from os.path import join moddir = ut.get_module_dir(tests) ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
def predict_proba_df(verif, edges): """ CommandLine: python -m wbia.algo.graph.demo DummyVerif.predict_edges Example: >>> # ENABLE_DOCTEST >>> from wbia.algo.graph.demo import * # NOQA >>> from wbia.algo.graph import demo >>> import networkx as nx >>> kwargs = dict(num_pccs=40, size=2) >>> infr = demo.demodata_infr(**kwargs) >>> verif = infr.dummy_verif >>> edges = list(infr.graph.edges()) >>> probs = verif.predict_proba_df(edges) >>> #print('scores = %r' % (scores,)) >>> #hashid = ut.hash_data(scores) >>> #print('hashid = %r' % (hashid,)) >>> #assert hashid == 'cdlkytilfeqgmtsihvhqwffmhczqmpil' """ infr = verif.infr edges = list(it.starmap(verif.infr.e_, edges)) prob_cache = infr.task_probs['match_state'] is_miss = np.array([e not in prob_cache for e in edges]) # is_hit = ~is_miss if np.any(is_miss): miss_edges = ut.compress(edges, is_miss) miss_truths = [verif._get_truth(edge) for edge in miss_edges] grouped_edges = ut.group_items(miss_edges, miss_truths, sorted_=False) # Need to make this determenistic too states = [POSTV, NEGTV, INCMP] for key in sorted(grouped_edges.keys()): group = grouped_edges[key] probs0 = randn( shape=[len(group)], rng=verif.rng, a_max=1, a_min=0, **verif.dummy_params[key], ) # Just randomly assign other probs probs1 = verif.rng.rand(len(group)) * (1 - probs0) probs2 = 1 - (probs0 + probs1) for edge, probs in zip(group, zip(probs0, probs1, probs2)): prob_cache[edge] = ut.dzip(states, probs) from wbia.algo.graph import nx_utils as nxu import pandas as pd probs = pd.DataFrame( ut.take(prob_cache, edges), index=nxu.ensure_multi_index(edges, ('aid1', 'aid2')), ) return probs
def set_annot_lblannot_from_rowid(ibs, aid_list, lblannot_rowid_list, _lbltype): """ Sets items/lblannot_rowids of a list of annotations.""" # Get the alrids_list for the aids, using the lbltype as a filter alrids_list = ibs.get_annot_alrids_oftype(aid_list, ibs.lbltype_ids[_lbltype]) # Find the aids which already have relationships (of _lbltype) setflag_list = [len(alrids) > 0 for alrids in alrids_list] # Add the relationship if it doesn't exist aid_list_to_add = ut.get_dirty_items(aid_list, setflag_list) lblannot_rowid_list_to_add = ut.get_dirty_items(lblannot_rowid_list, setflag_list) # set the existing relationship if one already exists alrids_list_to_set = ut.compress(alrids_list, setflag_list) lblannot_rowid_list_to_set = ut.compress(lblannot_rowid_list, setflag_list) # Assert each annot has only one relationship of this type ibsfuncs.assert_singleton_relationship(ibs, alrids_list_to_set) alrid_list_to_set = ut.flatten(alrids_list_to_set) # Add the new relationships ibs.add_annot_relationship(aid_list_to_add, lblannot_rowid_list_to_add) # Set the old relationships ibs.set_alr_lblannot_rowids(alrid_list_to_set, lblannot_rowid_list_to_set)
def get_other_nids(self): ibs = self.ibs all_nid_list = ibs.get_annot_name_rowids(self.all_aid_list) unique_nid_list = ut.unique_ordered(all_nid_list) is_unknown = ibs.is_nid_unknown(unique_nid_list) is_name1 = [nid == self.nid1 for nid in unique_nid_list] is_name2 = [nid == self.nid2 for nid in unique_nid_list] is_other = ut.and_lists(*tuple(map(ut.not_list, (is_name1, is_name2, is_unknown)))) other_nid_list = ut.compress(unique_nid_list, is_other) return other_nid_list
def filter_and_relabel(labels, label_gids, min_imgs_per_occurence, occur_unixtimes=None): """ Removes clusters with too few members. Relabels clusters-labels such that label 0 has the most members """ label_nGids = np.array(list(map(len, label_gids))) label_isvalid = label_nGids >= min_imgs_per_occurence occur_gids = ut.compress(label_gids, label_isvalid) if occur_unixtimes is not None: occur_unixtimes = ut.compress(occur_unixtimes, label_isvalid) # Rebase ids so occurrence0 has the most images # occur_ids = list(range(label_isvalid.sum())) # else: # sort by time instead unixtime_arr = np.array(occur_unixtimes) # Reorder occurrences so the oldest has the lowest number occur_gids = ut.take(label_gids, unixtime_arr.argsort()) occur_ids = list(range(len(occur_gids))) return occur_ids, occur_gids
def check_chip_external_storage(ibs, cid_list): chip_fpath_list = get_chip_fpath(ibs, cid_list, check_external_storage=False) notexists_flags = [not exists(cfpath) for cfpath in chip_fpath_list] if any(notexists_flags): invalid_cids = ut.compress(cid_list, notexists_flags) print('ERROR: %d CHIPS DO NOT EXIST' % (len(invalid_cids))) print('ATTEMPING TO FIX %d / %d non-existing chip paths' % (len(invalid_cids), len(cid_list))) ibs.delete_chips(invalid_cids) raise controller_inject.ExternalStorageException('NON-EXISTING EXTRENAL STORAGE ERROR. REQUIRES RECOMPUTE. TRY AGAIN') return chip_fpath_list
def get_cards_in_hand(player, valid_types=None, invert=False): card_list = player.hand if valid_types is None: valid_cards = card_list else: flags = [ut.is_superset(c.types, valid_types) for c in card_list] if invert: flags = ut.not_list(flags) valid_cards = ut.compress(card_list, flags) return valid_cards
def get_other_nids(self): ibs = self.ibs all_nid_list = ibs.get_annot_name_rowids(self.all_aid_list) unique_nid_list = ut.unique_ordered(all_nid_list) is_unknown = ibs.is_nid_unknown(unique_nid_list) is_name1 = [nid == self.nid1 for nid in unique_nid_list] is_name2 = [nid == self.nid2 for nid in unique_nid_list] is_other = ut.and_lists( *tuple(map(ut.not_list, (is_name1, is_name2, is_unknown)))) other_nid_list = ut.compress(unique_nid_list, is_other) return other_nid_list
def get_where(group, attr, cmp_, target): if isinstance(cmp_, six.string_types): cmp_ = { '==': op.eq, '<': op.lt, '<=': op.le, '>': op.gt, '>=': op.ge, }[cmp_] #.get(cmp_, cmp_) attrs = group.get_attrs(attr) flags = [cmp_(val, target) for val in attrs] return ut.compress(group.cards, flags)
def handle_cache_misses(ibs, getter_func, rowid_list, ismiss_list, vals_list, cache_, kwargs): miss_indices = ut.list_where(ismiss_list) miss_rowids = ut.compress(rowid_list, ismiss_list) # call wrapped function miss_vals = getter_func(ibs, miss_rowids, **kwargs) # overwrite missed output for index, val in zip(miss_indices, miss_vals): vals_list[index] = val # Output write # cache save for rowid, val in zip(miss_rowids, miss_vals): cache_[rowid] = val # Cache write
def test_openworkdirs(): """ problems: PZ_DanExt_All PZ_DanExt_Test GZ_March2012 Wildebeest_ONLY_MATCHES python dev.py --convert --dbdir /raid/work/PZ_Marianne --force-delete python dev.py --convert --dbdir /raid/work/SL_Siva --force-delete python dev.py --convert --dbdir /raid/work/PZ_SweatwaterSmall --force-delete """ canskip = [ '/raid/work/NAUT_test2', '/raid/work/WD_Siva', '/raid/work/PZ_FlankHack', '/raid/work/PZ_Mothers', '/raid/work/GZ_Foals', '/raid/work/PZ_MTEST', '/raid/work/GIR_Tanya', '/raid/work/GZ_Siva', '/raid/work/Wildebeest', '/raid/work/sonograms', '/raid/work/MISC_Jan12', '/raid/work/GZ_Master0', '/raid/work/LF_OPTIMIZADAS_NI_V_E', '/raid/work/LF_Bajo_bonito', '/raid/work/Frogs', '/raid/work/GZ_ALL', '/raid/work/JAG_Kelly', '/raid/work/NAUT_test (copy)', '/raid/work/WS_hard', '/raid/work/WY_Toads', '/raid/work/NAUT_Dan', '/raid/work/LF_WEST_POINT_OPTIMIZADAS', '/raid/work/Seals', '/raid/work/Rhinos_Stewart', '/raid/work/Elephants_Stewart', '/raid/work/NAUT_test', ] import ibeis from ibeis.init import sysres import os import utool as ut # NOQA from os.path import join from ibeis.dbio import ingest_hsdb import ibeis.other.dbinfo ibeis.other.dbinfo.rrr() workdir = sysres.get_workdir() dbname_list = os.listdir(workdir) dbpath_list = [join(workdir, name) for name in dbname_list] is_hsdb_list = list(map(ingest_hsdb.is_hsdb, dbpath_list)) hsdb_list = ut.compress(dbpath_list, is_hsdb_list) #is_ibs_cvt_list = np.array(list(map(is_succesful_convert, dbpath_list))) regen_cmds = [] for hsdb_dpath in hsdb_list: if hsdb_dpath in canskip: continue try: ibs = ibeis.opendb(hsdb_dpath) # NOQA print('Succesfully opened hsdb: ' + hsdb_dpath) print(ibs.get_dbinfo_str()) except Exception as ex: ut.printex(ex, 'Failed to convert hsdb: ' + hsdb_dpath) regen_cmd = 'python dev.py --convert --dbdir ' + hsdb_dpath regen_cmds.append(regen_cmd) print('\n'.join(regen_cmds))
def get_cmc_feasible_sequences(spell_list, max_avail_cmc): # Get spells castable on their own flags = [spell.cmc <= max_avail_cmc for spell in spell_list] feasible_spells = ut.compress(spell_list, flags) cmc_feasible_sequences = [] for num in range(1, len(feasible_spells) + 1): spell_combos = list(itertools.combinations(feasible_spells, num)) for combo in spell_combos: total = sum([spell.cmc for spell in combo]) if total <= max_avail_cmc: cmc_feasible_sequences.append(combo) return cmc_feasible_sequences
def dismiss_all(self, event=None): """ All unknown annotations are given DIFFERENT new names """ # Delete all original names ibs = self.ibs aid_list = self.all_aid_list is_unknown = ibs.is_aid_unknown(aid_list) aid_list_filtered = ut.compress(aid_list, is_unknown) # Rename annotations ibs.set_annot_names_to_different_new_names(aid_list_filtered) self.update_callback() self.backend_callback() self.show_page()