def shallowcopy(qreq_, qaids=None): """ Creates a copy of qreq with the same qparams object and a subset of the qx and dx objects. used to generate chunks of vsone and vsmany queries Example: >>> # ENABLE_DOCTEST >>> from wbia.algo.smk.match_chips5 import * # NOQA >>> from wbia.algo.smk.smk_pipeline import testdata_smk >>> import wbia >>> wbia, smk, qreq_ = testdata_smk() >>> qreq2_ = qreq_.shallowcopy(qaids=1) >>> assert qreq_.daids is qreq2_.daids, 'should be the same' >>> assert len(qreq_.qaids) != len(qreq2_.qaids), 'should be diff' >>> #assert qreq_.metadata is not qreq2_.metadata """ # qreq2_ = qreq_.__class__() cls = qreq_.__class__ qreq2_ = cls.__new__(cls) qreq2_.__dict__.update(qreq_.__dict__) qaids = ut.ensure_iterable(qaids) assert ut.issubset(qaids, qreq_.qaids), 'not a subset' qreq2_.qaids = qaids return qreq2_
def __init__(repo, url=None, code_dir=None, dpath=None, modname=None, pythoncmd=None): # modname might need to be called egg? import utool as ut if url is not None and '.git@' in url: # parse out specific branch repo.default_branch = url.split('@')[-1] url = '@'.join(url.split('@')[:-1]) else: repo.default_branch = None repo.url = url repo._modname = None if modname is None: modname = [] repo._modname_hints = ut.ensure_iterable(modname) repo.dpath = None repo.scripts = {} if pythoncmd is None: import sys pythoncmd = sys.executable repo.pythoncmd = pythoncmd if dpath is None and repo.url is not None and code_dir is not None: dpath = join(code_dir, repo.reponame) if dpath is not None: repo.dpath = util_path.unixpath(dpath)
def view_file_in_directory(fpaths): import utool as ut fpaths = ut.ensure_iterable(fpaths) fnames = [basename(f) for f in fpaths] dpaths = [dirname(f) for f in fpaths] dpath_to_fnames = ut.group_items(fnames, dpaths) for dpath, fnames in dpath_to_fnames.items(): ut.view_directory(dpath, fnames[0], verbose=False)
def _exec_filter(flags, tags_list, fields, op, compare): if fields is not None: fields = ut.ensure_iterable(fields) if ignore_case: fields = [f.lower() for f in fields] flags_ = _flag_tags(tags_list, fields, op, compare) logic_func(flags, flags_, out=flags) return flags
def get_prefered_browser(pref_list=[], fallback=True): r""" Args: browser_preferences (list): (default = []) fallback (bool): uses default if non of preferences work (default = True) CommandLine: python -m utool.util_grabdata --test-get_prefered_browser Ignore: import webbrowser webbrowser._tryorder pref_list = ['chrome', 'firefox', 'google-chrome'] pref_list = ['firefox', 'google-chrome'] Example: >>> # DISABLE_DOCTEST >>> from utool.util_grabdata import * # NOQA >>> browser_preferences = ['firefox', 'chrome', 'safari'] >>> fallback = True >>> browser = get_prefered_browser(browser_preferences, fallback) >>> result = ('browser = %s' % (str(browser),)) >>> print(result) >>> ut.quit_if_noshow() """ import webbrowser import utool as ut pref_list = ut.ensure_iterable(pref_list) error_list = [] # Hack for finding chrome on win32 if ut.WIN32: # http://stackoverflow.com/questions/24873302/webbrowser-chrome-exe-does-not-work win32_chrome_fpath = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe' win32_chrome_browsername = win32_chrome_fpath + ' %s' win32_map = { 'chrome': win32_chrome_browsername, 'google-chrome': win32_chrome_browsername, } for browsername, win32_browsername in win32_map.items(): index = ut.listfind(pref_list, browsername) if index is not None and True: # ut.checkpath(win32_browsername): pref_list.insert(index + 1, win32_browsername) for browsername in pref_list: try: browser = webbrowser.get(browsername) return browser except webbrowser.Error as ex: error_list.append(ex) print(str(browsername) + ' failed. Reason: ' + str(ex)) if fallback: browser = webbrowser return browser else: raise AssertionError('No browser meets preferences=%r. error_list=%r' % (pref_list, error_list,))
def imagesets(ibs, gsids=None, text=None): if text is not None: gsids = ibs.get_imageset_imgsetids_from_text(text) if gsids is None: gsids = ibs.get_valid_imgsetids() elif gsids.__class__.__name__ == 'ImageSets': return gsids gsids = ut.ensure_iterable(gsids) return ImageSets(gsids, ibs)
def remove_annot_case_tags(ibs, aid_list, tag_list): if isinstance(tag_list, six.string_types): # Apply single tag to everybody tag_list = [tag_list] * len(aid_list) tags_list = [ut.ensure_iterable(tag) for tag in tag_list] text_list = ibs.get_annot_tag_text(aid_list) orig_tags_list = [[] if note is None else _parse_tags(note) for note in text_list] new_tags_list = [ut.setdiff(t2, t1) for t1, t2 in zip(tags_list, orig_tags_list)] new_text_list = [';'.join(tags) for tags in new_tags_list] ibs.set_annot_tag_text(aid_list, new_text_list)
def dummy_single_func(depc, *row_arg, **kw): config = kw.get('config') param_val = config[config_param] data = [] for row, p in zip(row_arg, parents): p = p.replace('*', '') # print('p = %r' % (p,)) # print('row = %r' % (row,)) if not p.startswith(depc.root): native_cols = depc.get_native(p, ut.ensure_iterable(row)) parent_data = '+'.join( ['#'.join(col) for col in native_cols]) else: parent_data = ('root(' + ';'.join( list(map(str, ut.ensure_iterable(row)))) + ')') data += [parent_data] d = '[' + '&'.join(data) + ']' retstr = tablename + '(' + d + ':' + str(param_val) + ')' return (retstr, )
def images(ibs, gids=None, uuids=None, **kwargs): """ Makes an Images object """ if uuids is not None: assert gids is None, 'specify one primary key' gids = ibs.get_image_gids_from_uuid(uuids) if gids is None: gids = ibs.get_valid_gids() elif gids.__class__.__name__ == 'Images': return gids gids = ut.ensure_iterable(gids) return Images(gids, ibs, **kwargs)
def get_annotmatch_prop(ibs, prop, annotmatch_rowids): r""" hacky getter for dynamic properties of annotmatches using notes table Args: prop (str): annotmatch_rowids (?): Returns: list: filtered_aid_list CommandLine: python -m wbia.tag_funcs --exec-get_annotmatch_prop Example: >>> # DISABLE_DOCTEST >>> # Test setting and getting standard keys >>> from wbia.tag_funcs import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='testdb1') >>> prop = 'hard' >>> annotmatch_rowids = ibs._get_all_annotmatch_rowids() >>> flag_list = get_annotmatch_prop(ibs, prop, annotmatch_rowids) >>> flag_list = ('filtered_aid_list = %s' % (str(flag_list),)) >>> subset_rowids = annotmatch_rowids[::2] >>> set_annotmatch_prop(ibs, prop, subset_rowids, [True] * len(subset_rowids)) >>> flag_list2 = get_annotmatch_prop(ibs, prop, annotmatch_rowids) >>> print('flag_list2 = %r' % (flag_list2,)) Example: >>> # DISABLE_DOCTEST >>> # Test setting and getting non-standard keys >>> from wbia.tag_funcs import * # NOQA >>> import wbia >>> ibs = wbia.opendb(defaultdb='testdb1') >>> prop = 'occlusion' >>> annotmatch_rowids = ibs._get_all_annotmatch_rowids() >>> flag_list = get_annotmatch_prop(ibs, prop, annotmatch_rowids) >>> flag_list = ('filtered_aid_list = %s' % (str(flag_list),)) >>> subset_rowids = annotmatch_rowids[1::2] >>> subset_rowids1 = annotmatch_rowids[::2] >>> set_annotmatch_prop(ibs, prop, subset_rowids1, [True] * len(subset_rowids)) >>> set_annotmatch_prop(ibs, 'pose', subset_rowids1, [True] * len(subset_rowids)) >>> flag_list2 = get_annotmatch_prop(ibs, prop, annotmatch_rowids) >>> print('flag_list2 = %r' % (flag_list2,)) """ # if prop.lower() in ANNOTMATCH_PROPS_STANDARD_SET: # return ibs.get_annotmatch_standard_prop(prop, annotmatch_rowids) for prop_ in ut.ensure_iterable(prop): flag1 = prop_.lower() not in ANNOTMATCH_PROPS_OTHER_SET flag2 = prop_.lower() not in ANNOTMATCH_PROPS_OLD_SET if flag1 and flag2: raise NotImplementedError('Unknown prop_=%r' % (prop_, )) return get_annotmatch_other_prop(ibs, prop, annotmatch_rowids)
def annots(ibs, aids=None, uuids=None, **kwargs): """ Makes an Annots object """ if uuids is not None: assert aids is None, 'specify one primary key' aids = ibs.get_annot_aids_from_uuid(uuids) if aids is None: aids = ibs.get_valid_aids() elif aids.__class__.__name__ == 'Annots': return aids aids = ut.ensure_iterable(aids) return Annots(aids, ibs, **kwargs)
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, 'cant sudo on windows' if command == 'short_status': return repo.short_status() command_list = ut.ensure_iterable(command) cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) if not dry: print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ut.color_text(repo.dpath, 'yellow')) verbose = True with repo.chdir_context(): ret = None for count, cmd in enumerate(command_list): if dry: print(cmd) continue if not sudo or ut.WIN32: # ret = os.system(cmd) cmdinfo = ut.cmd2(cmd, verbout=True) out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret']) else: # cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1) out, err, ret = ut.cmd(cmd, sudo=True) if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: if error == 'raise': raise Exception('Failed command %r' % (cmd,)) elif error == 'return': return out else: raise ValueError('unknown flag error=%r' % (error,)) if return_out: return out if not dry: print('L____')
def make_standard_test_scripts(*args): if len(args) == 1: varydict, expt_name, media_name = args[0] else: varydict, expt_name, media_name = args media_names = ut.ensure_iterable(media_name) cmd_fmtstr_list = [] for media_name in media_names: cmd_flaglist = get_results_command(expt_name, media_name) cmd_fmtstr = ''.join(cmd_flaglist) cmd_fmtstr = ' \\\n '.join(cmd_flaglist) cmd_fmtstr_list.append(cmd_fmtstr) fname = 'experiment_' + expt_name + '.sh' return write_formatted_script_lines(cmd_fmtstr_list, [varydict], fname)
def generate_class_images(dream, target_labels): """ import plottool as pt fnum = None kw = dict(init='gauss', niters=500, update_rate=.05, weight_decay=1e-4) target_labels = list(range(model.output_dims)) dream = draw_net.Dream(model, **kw) target_labels = 8 images = list(dream.generate_class_images(target_labels)) vid = vt.make_video(images, 'dynimg.pimj', fps=1, is_color=False, format='PIM1') vid = vt.make_video2(images, 'dynimg') import matplotlib.pyplot as plt ims = [] for img in imgs: im = plt.imshow(img[:, :, 0], interpolation='nearest', cmap='gray') ims.append([im]) import matplotlib.animation as animation fig = plt.figure() ani = animation.ArtistAnimation(fig, ims, interval=50, blit=True, repeat_delay=1000) ani.save('dynamic_images.mp4') ut.startfile('dynamic_images.mp4') plt.show() """ import ibeis_cnn.__THEANO__ as theano from ibeis_cnn.__THEANO__ import tensor as T # NOQA import utool as ut input_shape = dream.model.input_shape b, c, w, h = input_shape was_scalar = not ut.isiterable(target_labels) target_labels = ut.ensure_iterable(target_labels) assert len(target_labels) <= b, 'batch size too small' initial_state = dream._make_init_state() shared_images = theano.shared(initial_state.astype(np.float32)) step_fn = dream._make_objective(shared_images, target_labels) out = dream._postprocess_class_image(shared_images, target_labels, was_scalar) yield out for _ in ut.ProgIter(range(dream.niters), lbl='class dream', bs=True): step_fn() # objective = step_fn() # print('objective = %r' % (objective,)) out = dream._postprocess_class_image(shared_images, target_labels, was_scalar) yield out
def __init__(repo, url=None, code_dir=None, dpath=None, modname=None, pythoncmd=None): # modname might need to be called egg? import utool as ut repo.url = url repo._modname = None if modname is None: modname = [] repo._modname_hints = ut.ensure_iterable(modname) repo.dpath = None repo.scripts = {} repo.pythoncmd = pythoncmd if dpath is None and repo.url is not None and code_dir is not None: dpath = join(code_dir, repo.reponame) if dpath is not None: repo.dpath = util_path.unixpath(dpath)
def get_test_qaids(ibs): """ Gets test annotation_rowids based on command line arguments """ #print('[main_helpers]') test_qaids = [] valid_aids = ibs.get_valid_aids() printDBG('1. valid_aids = %r' % valid_aids[0:5]) #print(utool.dict_str(vars(params.args))) if params.args.qaid is not None: printDBG('Testing qaid=%r' % params.args.qaid) test_qaids.extend(params.args.qaid) if params.args.all_cases: printDBG('Testing all %d cases' % (len(valid_aids),)) printDBG('1. test_qaids = %r' % test_qaids[0:5]) test_qaids.extend(valid_aids) printDBG('2. test_qaids = %r' % test_qaids[0:5]) else: is_hard_list = ibsfuncs.get_annot_is_hard(ibs, valid_aids) hard_aids = utool.filter_items(valid_aids, is_hard_list) printDBG('Testing %d known hard cases' % len(hard_aids)) test_qaids.extend(hard_aids) if params.args.all_gt_cases: has_gt_list = ibs.get_annot_has_groundtruth(valid_aids) hasgt_aids = utool.filter_items(valid_aids, has_gt_list) print('Testing all %d ground-truthed cases' % len(hasgt_aids)) test_qaids.extend(hasgt_aids) # Sample a large pool of query indexes # Filter only the ones you want from the large pool if params.args.index is not None: indexes = utool.ensure_iterable(params.args.index) #printDBG('Chosen indexes=%r' % (indexes,)) #printDBG('test_qaids = %r' % test_qaids[0:5]) _test_qaids = [test_qaids[xx] for xx in indexes] test_qaids = _test_qaids #printDBG('test_qaids = %r' % test_qaids) elif len(test_qaids) == 0 and len(valid_aids) > 0: #printDBG('no hard or gt aids. Defaulting to the first ANNOTATION') test_qaids = valid_aids[0:1] #print('test_qaids = %r' % test_qaids) test_qaids = utool.unique_keep_order2(test_qaids) return test_qaids
def append_annot_case_tags(ibs, aid_list, tag_list): """ Generally appends tags to annotations. Careful not to introduce too many random tags. Maybe we should just let that happen and introduce tag-aliases Note: this is more of a set add rather than a list append TODO: remove """ # Ensure each item is a list #tags_list = [tag if isinstance(tag, list) else [tag] for tag in tag_list] if isinstance(tag_list, six.string_types): # Apply single tag to everybody tag_list = [tag_list] * len(aid_list) tags_list = [ut.ensure_iterable(tag) for tag in tag_list] text_list = ibs.get_annot_tag_text(aid_list) orig_tags_list = [[] if note is None else _parse_tags(note) for note in text_list] new_tags_list = [ut.unique(t1 + t2) for t1, t2 in zip(tags_list, orig_tags_list)] ibs.set_annot_case_tags(aid_list, new_tags_list)
def matches(ibs, ams=None, edges=None, uuid_edges=None, **kwargs): """ Makes an Annots object """ if uuid_edges is not None: assert ams is None, 'specify one primary key' assert edges is None, 'specify one primary key' uuids1, uuids2 = list(zip(*uuid_edges)) aids1 = ibs.get_annot_aids_from_uuid(uuids1) aids2 = ibs.get_annot_aids_from_uuid(uuids2) ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2) if edges is not None: assert ams is None, 'specify one primary key' assert uuid_edges is None, 'specify one primary key' aids1, aids2 = list(zip(*edges)) ams = ibs.get_annotmatch_rowid_from_undirected_superkey(aids1, aids2) if ams is None: ams = ibs._get_all_annotmatch_rowids() elif ams.__class__.__name__ == 'AnnotMatches': return ams ams = ut.ensure_iterable(ams) return AnnotMatches(ams, ibs, **kwargs)
def issue(repo, command, sudo=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, "cant sudo on windows" command_list = ut.ensure_iterable(command) cmdstr = "\n ".join([cmd_ for cmd_ in command_list]) print("+--- *** repocmd(%s) *** " % (cmdstr,)) print("repo=%s" % ut.color_text(repo.dpath, "yellow")) with ut.ChdirContext(repo.dpath, verbose=False): ret = None for count, cmd in enumerate(command_list): if not sudo or ut.WIN32: ret = os.system(cmd) else: out, err, ret = ut.cmd(cmd, sudo=True) verbose = True if verbose > 1: print("ret(%d) = %r" % (count, ret)) if ret != 0: raise Exception("Failed command %r" % (cmd,)) print("L____")
def issue(repo, command, sudo=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, 'cant sudo on windows' command_list = ut.ensure_iterable(command) cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ut.color_text(repo.dpath, 'yellow')) with ut.ChdirContext(repo.dpath, verbose=False): ret = None for count, cmd in enumerate(command_list): if not sudo or ut.WIN32: ret = os.system(cmd) else: out, err, ret = ut.cmd(cmd, sudo=True) verbose = True if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: raise Exception('Failed command %r' % (cmd,)) print('L____')
def build_alias_map(regex_map, tag_vocab): """ Constructs explicit mapping. Order of items in regex map matters. Items at top are given preference. Example: >>> tags_list = [['t1', 't2'], [], ['t3'], ['t4', 't5']] >>> tag_vocab = ut.flat_unique(*tags_list) >>> regex_map = [('t[3-4]', 'A9'), ('t0', 'a0')] >>> unmapped = list(set(tag_vocab) - set(alias_map.keys())) """ import utool as ut import re alias_map = ut.odict([]) for pats, new_tag in reversed(regex_map): pats = ut.ensure_iterable(pats) for pat in pats: flags = [re.match(pat, t) for t in tag_vocab] for old_tag in ut.compress(tag_vocab, flags): alias_map[old_tag] = new_tag identity_map = ut.take_column(regex_map, 1) for tag in ut.filter_Nones(identity_map): alias_map[tag] = tag return alias_map
def make_class_images(dream, target_labels): import ibeis_cnn.__THEANO__ as theano from ibeis_cnn.__THEANO__ import tensor as T # NOQA import utool as ut was_scalar = not ut.isiterable(target_labels) target_labels = ut.ensure_iterable(target_labels) if True: # We are forcing a batch size for this visualization input_shape = (len(target_labels),) + dream.model.input_shape[1:] else: # Maybe some cnn layers cant take variable batches? input_shape = dream.model.input_shape b, c, w, h = input_shape assert len(target_labels) <= b, 'batch size too small' initial_state = dream._make_init_state() # make image a shared variable that you can update if dream.shared_images is None: dream.shared_images = theano.shared(initial_state) else: dream.shared_images.set_value(initial_state) if dream.step_fn is None: dream.step_fn = dream._make_objective(dream.shared_images, target_labels) # Optimize objective via backpropogation for a few iterations for _ in ut.ProgIter(range(dream.niters), lbl='making class model img', bs=True): dream.step_fn() #print('objective = %r' % (objective,)) out = dream._postprocess_class_image(dream.shared_images, target_labels, was_scalar) return out
def export(ibs, aid_pairs=None): """ 3 - 4 different animals 2 views of each matching keypoint coordinates on each annotation """ if aid_pairs is None: if ibs.get_dbname() == 'PZ_MOTHERS': aid_pair_list = MOTHERS_VIEWPOINT_EXPORT_PAIRS if ibs.get_dbname() == 'GZ_ALL': aid_pair_list = GZ_VIEWPOINT_EXPORT_PAIRS ibs.update_query_cfg(ratio_thresh=1.6) export_path = expanduser('~/Dropbox/Assignments/dataset') #utool.view_directory(export_path) # MOTHERS EG: for aid_pair in aid_pair_list: cm_list, qreq_ = ibs.query_chips(aid_pair, aid_pair) #ibeis.viz.show_qres(ibs, qaid2_qres.values()[1]); df2.iup() mrids_list = [] mkpts_list = [] for cm in cm_list: qaid = cm.qaid print('Getting kpts from %r' % qaid) #cm.show_top(ibs) posrid_list = utool.ensure_iterable(cm.get_classified_pos()) mrids_list.extend([(qaid, posrid) for posrid in posrid_list]) mkpts_list.extend(cm.get_matching_keypoints(ibs, posrid_list)) mkey2_kpts = {} for mrids_tup, mkpts_tup in zip(mrids_list, mkpts_list): assert len(mrids_tup) == 2, 'must be a match tuple' mrids_ = np.array(mrids_tup) sortx = mrids_.argsort() mrids_ = mrids_[sortx] mkpts_ = np.array(mkpts_tup)[sortx] if sortx[0] == 0: pass mkey = tuple(mrids_.tolist()) try: kpts_list = mkey2_kpts[mkey] print('append to mkey=%r' % (mkey, )) except KeyError: print('new mkey=%r' % (mkey, )) kpts_list = [] kpts_list.append(mkpts_) mkey2_kpts[mkey] = kpts_list mkeys_list = mkey2_kpts.keys() mkeys_keypoints = mkey2_kpts.values() for mkeys, mkpts_list in zip(mkeys_list, mkeys_keypoints): print(mkeys) print(len(kpts_list)) kpts1_m = np.vstack([mkpts[0] for mkpts in mkpts_list]) kpts2_m = np.vstack([mkpts[1] for mkpts in mkpts_list]) match_lines = [ repr(( tuple(kp1[ktool.LOC_DIMS].tolist()), tuple(kp2[ktool.LOC_DIMS].tolist()), )) + ', ' for kp1, kp2 in zip(kpts1_m, kpts2_m) ] mcpaths_list = ibs.get_annot_chip_fpath(mkeys) fnames_list = list(map(lambda x: split(x)[1], mcpaths_list)) for path in mcpaths_list: utool.copy(path, export_path) header_lines = [ '# Exported keypoint matches (might be duplicates matches)', '# matching_aids = %r' % (mkey, ) ] header_lines += [ '# img%d = %r' % (count, fname) for count, fname in enumerate(fnames_list) ] header_lines += [ '# LINE FORMAT: match_pts = [(img1_xy, img2_xy) ... ]' ] header_text = '\n'.join(header_lines) match_text = '\n'.join(['match_pts = ['] + match_lines + [']']) matchfile_text = '\n'.join([header_text, match_text]) matchfile_name = ('match_aids(%d,%d).txt' % mkey) matchfile_path = join(export_path, matchfile_name) utool.write_to(matchfile_path, matchfile_text) print(header_text) print(utool.truncate_str(match_text, maxlen=500))
def execute_filter(flags, tags_list, fields, op, compare): if fields is not None: fields = ut.ensure_iterable(fields) flags_ = flag_tags(tags_list, fields, op, compare) logic_func(flags, flags_, out=flags) return flags
def filterflags_general_tags( tags_list, has_any=None, has_all=None, has_none=None, min_num=None, max_num=None, any_startswith=None, any_endswith=None, any_match=None, none_match=None, logic='and', ): r""" maybe integrate into utool? Seems pretty general Args: tags_list (list): has_any (None): (default = None) has_all (None): (default = None) min_num (None): (default = None) max_num (None): (default = None) CommandLine: python -m wbia.tag_funcs --exec-filterflags_general_tags python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --helpx python -m wbia.tag_funcs --exec-filterflags_general_tags:0 python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --none_match n python -m wbia.tag_funcs --exec-filterflags_general_tags:0 --has_none=n,o python -m wbia.tag_funcs --exec-filterflags_general_tags:1 python -m wbia.tag_funcs --exec-filterflags_general_tags:2 Example0: >>> # DISABLE_DOCTEST >>> from wbia.tag_funcs import * # NOQA >>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']] >>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list) >>> print('kwargs = %r' % (kwargs,)) >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> print(flags) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example1: >>> # ENABLE_DOCTEST >>> from wbia.tag_funcs import * # NOQA >>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']] >>> has_all = 'n' >>> min_num = 1 >>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example2: >>> # ENABLE_DOCTEST >>> from wbia.tag_funcs import * # NOQA >>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']] >>> kwargs = { >>> 'any_endswith': 'n', >>> 'any_match': None, >>> 'any_startswith': 'n', >>> 'has_all': None, >>> 'has_any': None, >>> 'has_none': None, >>> 'max_num': 3, >>> 'min_num': 1, >>> 'none_match': ['P'], >>> } >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> filtered = ut.compress(tags_list, flags) >>> result = ('result = %s' % (ut.repr2(filtered),)) result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']] """ import re import operator def fix_tags(tags): return {six.text_type(t.lower()) for t in tags} if logic is None: logic = 'and' logic_func = {'and': np.logical_and, 'or': np.logical_or}[logic] default_func = {'and': np.ones, 'or': np.zeros}[logic] tags_list_ = [fix_tags(tags_) for tags_ in tags_list] flags = default_func(len(tags_list_), dtype=np.bool) if min_num is not None: flags_ = [len(tags_) >= min_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if max_num is not None: flags_ = [len(tags_) <= max_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_any is not None: has_any = fix_tags(set(ut.ensure_iterable(has_any))) flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_none is not None: has_none = fix_tags(set(ut.ensure_iterable(has_none))) flags_ = [ len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_ ] logic_func(flags, flags_, out=flags) if has_all is not None: has_all = fix_tags(set(ut.ensure_iterable(has_all))) flags_ = [ len(has_all.intersection(tags_)) == len(has_all) for tags_ in tags_list_ ] logic_func(flags, flags_, out=flags) def check_item(tags_, fields, op, compare): t_flags = [any([compare(t, f) for f in fields]) for t in tags_] num_passed = sum(t_flags) flag = op(num_passed, 0) return flag def flag_tags(tags_list, fields, op, compare): flags = [ check_item(tags_, fields, op, compare) for tags_ in tags_list_ ] return flags def execute_filter(flags, tags_list, fields, op, compare): if fields is not None: fields = ut.ensure_iterable(fields) flags_ = flag_tags(tags_list, fields, op, compare) logic_func(flags, flags_, out=flags) return flags flags = execute_filter(flags, tags_list, any_startswith, operator.gt, six.text_type.startswith) flags = execute_filter(flags, tags_list, any_endswith, operator.gt, six.text_type.endswith) flags = execute_filter(flags, tags_list, any_match, operator.gt, lambda t, f: re.match(f, t)) flags = execute_filter(flags, tags_list, none_match, operator.eq, lambda t, f: re.match(f, t)) return flags
def filterflags_general_tags(tags_list, has_any=None, has_all=None, has_none=None, min_num=None, max_num=None, any_startswith=None, any_endswith=None, any_match=None, none_match=None, logic='and'): r""" maybe integrate into utool? Seems pretty general Args: tags_list (list): has_any (None): (default = None) has_all (None): (default = None) min_num (None): (default = None) max_num (None): (default = None) CommandLine: python -m ibeis.tag_funcs --exec-filterflags_general_tags python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --helpx python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --none_match n python -m ibeis.tag_funcs --exec-filterflags_general_tags:0 --has_none=n,o python -m ibeis.tag_funcs --exec-filterflags_general_tags:1 python -m ibeis.tag_funcs --exec-filterflags_general_tags:2 Example0: >>> # DISABLE_DOCTEST >>> from ibeis.tag_funcs import * # NOQA >>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']] >>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list) >>> print('kwargs = %r' % (kwargs,)) >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> print(flags) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example1: >>> # ENABLE_DOCTEST >>> from ibeis.tag_funcs import * # NOQA >>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']] >>> has_all = 'n' >>> min_num = 1 >>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example2: >>> # ENABLE_DOCTEST >>> from ibeis.tag_funcs import * # NOQA >>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o',], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']] >>> kwargs = { >>> 'any_endswith': 'n', >>> 'any_match': None, >>> 'any_startswith': 'n', >>> 'has_all': None, >>> 'has_any': None, >>> 'has_none': None, >>> 'max_num': 3, >>> 'min_num': 1, >>> 'none_match': ['P'], >>> } >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> filtered = ut.compress(tags_list, flags) >>> result = ('result = %s' % (ut.repr2(filtered),)) result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']] """ import re import operator from ibeis import constants as const def fix_tags(tags): from ibeis import constants as const return {const.__STR__(t.lower()) for t in tags} if logic is None: logic = 'and' logic_func = { 'and': np.logical_and, 'or': np.logical_or, }[logic] default_func = { 'and': np.ones, 'or': np.zeros, }[logic] tags_list_ = [fix_tags(tags_) for tags_ in tags_list] flags = default_func(len(tags_list_), dtype=np.bool) if min_num is not None: flags_ = [len(tags_) >= min_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if max_num is not None: flags_ = [len(tags_) <= max_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_any is not None: has_any = fix_tags(set(ut.ensure_iterable(has_any))) flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_none is not None: has_none = fix_tags(set(ut.ensure_iterable(has_none))) flags_ = [len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_all is not None: has_all = fix_tags(set(ut.ensure_iterable(has_all))) flags_ = [len(has_all.intersection(tags_)) == len(has_all) for tags_ in tags_list_] logic_func(flags, flags_, out=flags) def test_item(tags_, fields, op, compare): t_flags = [any([compare(t, f) for f in fields]) for t in tags_] num_passed = sum(t_flags) flag = op(num_passed, 0) return flag def flag_tags(tags_list, fields, op, compare): flags = [test_item(tags_, fields, op, compare) for tags_ in tags_list_] return flags def execute_filter(flags, tags_list, fields, op, compare): if fields is not None: fields = ut.ensure_iterable(fields) flags_ = flag_tags(tags_list, fields, op, compare) logic_func(flags, flags_, out=flags) return flags flags = execute_filter( flags, tags_list, any_startswith, operator.gt, const.__STR__.startswith) flags = execute_filter( flags, tags_list, any_endswith, operator.gt, const.__STR__.endswith) flags = execute_filter( flags, tags_list, any_match, operator.gt, lambda t, f: re.match(f, t)) flags = execute_filter( flags, tags_list, none_match, operator.eq, lambda t, f: re.match(f, t)) return flags
def demodata_infr(**kwargs): """ kwargs = {} CommandLine: python -m wbia.algo.graph.demo demodata_infr --show python -m wbia.algo.graph.demo demodata_infr --num_pccs=25 python -m wbia.algo.graph.demo demodata_infr --profile --num_pccs=100 Ignore: >>> from wbia.algo.graph.demo import * # NOQA >>> from wbia.algo.graph import demo >>> import networkx as nx >>> kwargs = dict(num_pccs=6, p_incon=.5, size_std=2) >>> kwargs = ut.argparse_dict(kwargs) >>> infr = demo.demodata_infr(**kwargs) >>> pccs = list(infr.positive_components()) >>> assert len(pccs) == kwargs['num_pccs'] >>> nonfull_pccs = [cc for cc in pccs if len(cc) > 1 and nx.is_empty(nx.complement(infr.pos_graph.subgraph(cc)))] >>> expected_n_incon = len(nonfull_pccs) * kwargs['p_incon'] >>> n_incon = len(list(infr.inconsistent_components())) >>> # TODO can test that we our sample num incon agrees with pop mean >>> #sample_mean = n_incon / len(nonfull_pccs) >>> #pop_mean = kwargs['p_incon'] >>> print('status = ' + ut.repr4(infr.status(extended=True))) >>> ut.quit_if_noshow() >>> infr.show(pickable=True, groupby='name_label') >>> ut.show_if_requested() Ignore: kwargs = { 'ccs': [[1, 2, 3], [4, 5]] } """ import networkx as nx import vtool as vt from wbia.algo.graph import nx_utils def kwalias(*args): params = args[0:-1] default = args[-1] for key in params: if key in kwargs: return kwargs[key] return default num_pccs = kwalias('num_pccs', 16) size_mean = kwalias('pcc_size_mean', 'pcc_size', 'size', 5) size_std = kwalias('pcc_size_std', 'size_std', 0) # p_pcc_incon = kwargs.get('p_incon', .1) p_pcc_incon = kwargs.get('p_incon', 0) p_pcc_incomp = kwargs.get('p_incomp', 0) pcc_sizes = kwalias('pcc_sizes', None) pos_redun = kwalias('pos_redun', [1, 2, 3]) pos_redun = ut.ensure_iterable(pos_redun) # number of maximum inconsistent edges per pcc max_n_incon = kwargs.get('n_incon', 3) rng = np.random.RandomState(0) counter = 1 if pcc_sizes is None: pcc_sizes = [ int(randn(size_mean, size_std, rng=rng, a_min=1)) for _ in range(num_pccs) ] else: num_pccs = len(pcc_sizes) if 'ccs' in kwargs: # Overwrites other options pcc_sizes = list(map(len, kwargs['ccs'])) num_pccs = len(pcc_sizes) size_mean = None size_std = 0 new_ccs = [] pcc_iter = list(enumerate(pcc_sizes)) pcc_iter = ut.ProgIter(pcc_iter, enabled=num_pccs > 20, label='make pos-demo') for i, size in pcc_iter: p = 0.1 want_connectivity = rng.choice(pos_redun) want_connectivity = min(size - 1, want_connectivity) # Create basic graph of positive edges with desired connectivity g = nx_utils.random_k_edge_connected_graph(size, k=want_connectivity, p=p, rng=rng) nx.set_edge_attributes(g, name='evidence_decision', values=POSTV) nx.set_edge_attributes(g, name='truth', values=POSTV) # nx.set_node_attributes(g, name='orig_name_label', values=i) assert nx.is_connected(g) # Relabel graph with non-conflicting names if 'ccs' in kwargs: g = nx.relabel_nodes(g, dict(enumerate(kwargs['ccs'][i]))) else: # Make sure nodes do not conflict with others g = nx.relabel_nodes( g, dict(enumerate(range(counter, len(g) + counter + 1)))) counter += len(g) # The probability any edge is inconsistent is `p_incon` # This is 1 - P(all edges consistent) # which means p(edge is consistent) = (1 - p_incon) / N complement_edges = ut.estarmap(nx_utils.e_, nx_utils.complement_edges(g)) if len(complement_edges) > 0: # compute probability that any particular edge is inconsistent # to achieve probability the PCC is inconsistent p_edge_inconn = 1 - (1 - p_pcc_incon)**(1 / len(complement_edges)) p_edge_unrev = 0.1 p_edge_notcomp = 1 - (1 - p_pcc_incomp)**(1 / len(complement_edges)) probs = np.array([p_edge_inconn, p_edge_unrev, p_edge_notcomp]) # if the total probability is greater than 1 the parameters # are invalid, so we renormalize to "fix" it. # if probs.sum() > 1: # warnings.warn('probabilities sum to more than 1') # probs = probs / probs.sum() pcumsum = probs.cumsum() # Determine which mutually exclusive state each complement edge is in # logger.info('pcumsum = %r' % (pcumsum,)) states = np.searchsorted(pcumsum, rng.rand(len(complement_edges))) incon_idxs = np.where(states == 0)[0] if len(incon_idxs) > max_n_incon: logger.info('max_n_incon = %r' % (max_n_incon, )) chosen = rng.choice(incon_idxs, max_n_incon, replace=False) states[np.setdiff1d(incon_idxs, chosen)] = len(probs) grouped_edges = ut.group_items(complement_edges, states) for state, edges in grouped_edges.items(): truth = POSTV if state == 0: # Add in inconsistent edges evidence_decision = NEGTV # TODO: truth could be INCMP or POSTV # new_edges.append((u, v, {'evidence_decision': NEGTV})) elif state == 1: evidence_decision = UNREV # TODO: truth could be INCMP or POSTV # new_edges.append((u, v, {'evidence_decision': UNREV})) elif state == 2: evidence_decision = INCMP truth = INCMP else: continue # Add in candidate edges attrs = { 'evidence_decision': evidence_decision, 'truth': truth } for (u, v) in edges: g.add_edge(u, v, **attrs) new_ccs.append(g) # (list(g.nodes()), new_edges)) pos_g = nx.union_all(new_ccs) assert len(new_ccs) == len(list(nx.connected_components(pos_g))) assert num_pccs == len(new_ccs) # Add edges between the PCCS neg_edges = [] if not kwalias('ignore_pair', False): logger.info('making pairs') pair_attrs_lookup = { 0: { 'evidence_decision': NEGTV, 'truth': NEGTV }, 1: { 'evidence_decision': INCMP, 'truth': INCMP }, 2: { 'evidence_decision': UNREV, 'truth': NEGTV }, # could be incomp or neg } # These are the probabilities that one edge has this state p_pair_neg = kwalias('p_pair_neg', 0.4) p_pair_incmp = kwalias('p_pair_incmp', 0.2) p_pair_unrev = kwalias('p_pair_unrev', 0) # p_pair_neg = 1 cc_combos = ((list(g1.nodes()), list(g2.nodes())) for (g1, g2) in it.combinations(new_ccs, 2)) valid_cc_combos = [(cc1, cc2) for cc1, cc2 in cc_combos if len(cc1) and len(cc2)] for cc1, cc2 in ut.ProgIter(valid_cc_combos, label='make neg-demo'): possible_edges = ut.estarmap(nx_utils.e_, it.product(cc1, cc2)) # probability that any edge between these PCCs is negative n_edges = len(possible_edges) p_edge_neg = 1 - (1 - p_pair_neg)**(1 / n_edges) p_edge_incmp = 1 - (1 - p_pair_incmp)**(1 / n_edges) p_edge_unrev = 1 - (1 - p_pair_unrev)**(1 / n_edges) # Create event space with sizes proportional to probabilities pcumsum = np.cumsum([p_edge_neg, p_edge_incmp, p_edge_unrev]) # Roll dice for each of the edge to see which state it lands on possible_pstate = rng.rand(len(possible_edges)) states = np.searchsorted(pcumsum, possible_pstate) flags = states < len(pcumsum) stateful_states = states.compress(flags) stateful_edges = ut.compress(possible_edges, flags) unique_states, groupxs_list = vt.group_indices(stateful_states) for state, groupxs in zip(unique_states, groupxs_list): # logger.info('state = %r' % (state,)) # Add in candidate edges edges = ut.take(stateful_edges, groupxs) attrs = pair_attrs_lookup[state] for (u, v) in edges: neg_edges.append((u, v, attrs)) logger.info('Made {} neg_edges between PCCS'.format(len(neg_edges))) else: logger.info('ignoring pairs') import wbia G = wbia.AnnotInference._graph_cls() G.add_nodes_from(pos_g.nodes(data=True)) G.add_edges_from(pos_g.edges(data=True)) G.add_edges_from(neg_edges) infr = wbia.AnnotInference.from_netx(G, infer=kwargs.get('infer', True)) infr.verbose = 3 infr.relabel_using_reviews(rectify=False) # fontname = 'Ubuntu' fontsize = 12 fontname = 'sans' splines = 'spline' # splines = 'ortho' # splines = 'line' infr.set_node_attrs('shape', 'circle') infr.graph.graph['ignore_labels'] = True infr.graph.graph['dark_background'] = False infr.graph.graph['fontname'] = fontname infr.graph.graph['fontsize'] = fontsize infr.graph.graph['splines'] = splines infr.set_node_attrs('width', 29) infr.set_node_attrs('height', 29) infr.set_node_attrs('fontsize', fontsize) infr.set_node_attrs('fontname', fontname) infr.set_node_attrs('fixed_size', True) # Set synthetic ground-truth attributes for testing # infr.apply_edge_truth() infr.edge_truth = infr.get_edge_attrs('truth') # Make synthetic verif infr.dummy_verif = DummyVerif(infr) infr.verifiers = {} infr.verifiers['match_state'] = infr.dummy_verif infr.demokw = kwargs return infr
def types(pubman, want): want = set(ut.ensure_iterable(want)) for pub in pubman.pub_list: if pub.type in want: yield pub
def get_training_featscores(qreq_, cm_list, disttype=None, namemode=True, fsvx=slice(None, None, None), threshx=None, thresh=.9, num=None, top_percent=None): """ Returns the flattened set of feature scores between each query and the correct groundtruth annotations as well as the top scoring false annotations. Args: qreq_ (ibeis.QueryRequest): query request object with hyper-parameters cm_list (list): disttype (None): (default = None) namemode (bool): (default = True) fsvx (slice): (default = slice(None, None, None)) threshx (None): (default = None) thresh (float): only used if threshx is specified (default = 0.9) SeeAlso: TestResult.draw_feat_scoresep Returns: tuple: (tp_scores, tn_scores, scorecfg) CommandLine: python -m ibeis.algo.hots.scorenorm --exec-get_training_featscores Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.hots.scorenorm import * # NOQA >>> import ibeis >>> cm_list, qreq_ = ibeis.testdata_cmlist(defaultdb='PZ_MTEST', a=['default:qsize=10']) >>> disttype = None >>> namemode = True >>> fsvx = None >>> threshx = 1 >>> thresh = 0.5 >>> (tp_scores, tn_scores, scorecfg) = get_training_featscores( >>> qreq_, cm_list, disttype, namemode, fsvx, threshx, thresh) >>> result = scorecfg >>> print(result) (lnbnn*fg)[fg > 0.5] lnbnn*fg[fg > 0.5] """ if fsvx is None: fsvx = slice(None, None, None) fsv_col_lbls = None tp_fsvs_list = [] tn_fsvs_list = [] #cm_list = [ cm_list[key] for key in sorted(cm_list.keys()) ] # Train on only positive examples trainable = [ qreq_.ibs.get_annot_has_groundtruth(cm.qaid, daid_list=cm.daid_list) and cm.get_top_nids()[0] == cm.qnid for cm in cm_list ] cm_list_ = ut.compress(cm_list, trainable) print('training using %d chipmatches' % (len(cm_list))) if disttype is None: fsv_col_lbls = cm.fsv_col_lbls train_getter = get_training_fsv else: fsv_col_lbls = ut.ensure_iterable(disttype) # annots = {} # Hack for cached vector lookups ibs = qreq_.ibs data_annots = ut.KeyedDefaultDict(ibs.get_annot_lazy_dict, config2_=qreq_.data_config2_) query_annots = ut.KeyedDefaultDict(ibs.get_annot_lazy_dict, config2_=qreq_.query_config2_) train_getter = partial(get_training_desc_dist, fsv_col_lbls=fsv_col_lbls, qreq_=qreq_, data_annots=data_annots, query_annots=query_annots) for cm in ut.ProgIter(cm_list_, lbl='building train featscores', adjust=True, freq=1): try: tp_fsv, tn_fsv = train_getter( cm, namemode=namemode, top_percent=top_percent) tp_fsvs_list.extend(tp_fsv) tn_fsvs_list.extend(tn_fsv) except UnbalancedExampleException: continue fsv_tp = np.vstack(tp_fsvs_list) fsv_tn = np.vstack(tn_fsvs_list) fsv_col_lbls_ = ut.take(fsv_col_lbls, fsvx) fsv_tp_ = fsv_tp.T[fsvx].T fsv_tn_ = fsv_tn.T[fsvx].T if threshx is not None: tp_scores = fsv_tp_[fsv_tp.T[threshx] > thresh].prod(axis=1) tn_scores = fsv_tn_[fsv_tn.T[threshx] > thresh].prod(axis=1) threshpart = ('[' + fsv_col_lbls[threshx] + ' > ' + str(thresh) + ']') scorecfg = '(%s)%s' % ('*'.join(fsv_col_lbls_), threshpart) else: tp_scores = fsv_tp_.prod(axis=1) tn_scores = fsv_tn_.prod(axis=1) scorecfg = '*'.join(fsv_col_lbls_) return tp_scores, tn_scores, scorecfg
def add_edge2(u, v, *args, **kwargs): v = ut.ensure_iterable(v) u = ut.ensure_iterable(u) for _u, _v in ut.product(u, v): graph.add_edge(_u, _v, *args, **kwargs)
def filterflags_general_tags(tags_list, has_any=None, has_all=None, has_none=None, min_num=None, max_num=None, any_startswith=None, any_endswith=None, in_any=None, any_match=None, none_match=None, logic='and', ignore_case=True): r""" maybe integrate into utool? Seems pretty general Args: tags_list (list): has_any (None): (default = None) has_all (None): (default = None) min_num (None): (default = None) max_num (None): (default = None) Notes: in_any should probably be ni_any TODO: make this function more natural CommandLine: python -m utool.util_tags --exec-filterflags_general_tags python -m utool.util_tags --exec-filterflags_general_tags:0 --helpx python -m utool.util_tags --exec-filterflags_general_tags:0 python -m utool.util_tags --exec-filterflags_general_tags:0 --none_match n python -m utool.util_tags --exec-filterflags_general_tags:0 --has_none=n,o python -m utool.util_tags --exec-filterflags_general_tags:1 python -m utool.util_tags --exec-filterflags_general_tags:2 Example0: >>> # ENABLE_DOCTEST >>> from utool.util_tags import * # NOQA >>> import utool as ut >>> tags_list = [['v'], [], ['P'], ['P', 'o'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['q', 'v'], ['n'], ['n'], ['N']] >>> kwargs = ut.argparse_dict(ut.get_kwdefaults2(filterflags_general_tags), type_hint=list) >>> print('kwargs = %r' % (kwargs,)) >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> print(flags) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example1: >>> # ENABLE_DOCTEST >>> from utool.util_tags import * # NOQA >>> import utool as ut >>> tags_list = [['v'], [], ['P'], ['P'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n'], ['N']] >>> has_all = 'n' >>> min_num = 1 >>> flags = filterflags_general_tags(tags_list, has_all=has_all, min_num=min_num) >>> result = ut.compress(tags_list, flags) >>> print('result = %r' % (result,)) Example2: >>> # ENABLE_DOCTEST >>> from utool.util_tags import * # NOQA >>> import utool as ut >>> tags_list = [['vn'], ['vn', 'no'], ['P'], ['P'], ['n', 'o'], [], ['n', 'N'], ['e', 'i', 'p', 'b', 'n'], ['n'], ['n', 'nP'], ['NP']] >>> kwargs = { >>> 'any_endswith': 'n', >>> 'any_match': None, >>> 'any_startswith': 'n', >>> 'has_all': None, >>> 'has_any': None, >>> 'has_none': None, >>> 'max_num': 3, >>> 'min_num': 1, >>> 'none_match': ['P'], >>> } >>> flags = filterflags_general_tags(tags_list, **kwargs) >>> filtered = ut.compress(tags_list, flags) >>> result = ('result = %s' % (ut.repr2(filtered),)) result = [['vn', 'no'], ['n', 'o'], ['n', 'N'], ['n'], ['n', 'nP']] """ import numpy as np import utool as ut def _fix_tags(tags): if ignore_case: return set([]) if tags is None else {six.text_type(t.lower()) for t in tags} else: return set([]) if tags is None else {six.text_type() for t in tags} if logic is None: logic = 'and' logic_func = { 'and': np.logical_and, 'or': np.logical_or, }[logic] default_func = { 'and': np.ones, 'or': np.zeros, }[logic] tags_list_ = [_fix_tags(tags_) for tags_ in tags_list] flags = default_func(len(tags_list_), dtype=np.bool) if min_num is not None: flags_ = [len(tags_) >= min_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if max_num is not None: flags_ = [len(tags_) <= max_num for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_any is not None: has_any = _fix_tags(set(ut.ensure_iterable(has_any))) flags_ = [len(has_any.intersection(tags_)) > 0 for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_none is not None: has_none = _fix_tags(set(ut.ensure_iterable(has_none))) flags_ = [len(has_none.intersection(tags_)) == 0 for tags_ in tags_list_] logic_func(flags, flags_, out=flags) if has_all is not None: has_all = _fix_tags(set(ut.ensure_iterable(has_all))) flags_ = [len(has_all.intersection(tags_)) == len(has_all) for tags_ in tags_list_] logic_func(flags, flags_, out=flags) def _test_item(tags_, fields, op, compare): t_flags = [any([compare(t, f) for f in fields]) for t in tags_] num_passed = sum(t_flags) flag = op(num_passed, 0) return flag def _flag_tags(tags_list, fields, op, compare): flags = [_test_item(tags_, fields, op, compare) for tags_ in tags_list_] return flags def _exec_filter(flags, tags_list, fields, op, compare): if fields is not None: fields = ut.ensure_iterable(fields) if ignore_case: fields = [f.lower() for f in fields] flags_ = _flag_tags(tags_list, fields, op, compare) logic_func(flags, flags_, out=flags) return flags flags = _exec_filter( flags, tags_list, any_startswith, operator.gt, six.text_type.startswith) flags = _exec_filter( flags, tags_list, in_any, operator.gt, operator.contains) flags = _exec_filter( flags, tags_list, any_endswith, operator.gt, six.text_type.endswith) flags = _exec_filter( flags, tags_list, any_match, operator.gt, lambda t, f: re.match(f, t)) flags = _exec_filter( flags, tags_list, none_match, operator.eq, lambda t, f: re.match(f, t)) return flags
def _make_rankings( infr, qaids=None, daids=None, prog_hook=None, cfgdict=None, name_method='node', use_cache=None, invalidate_supercache=None, ): # from wbia.algo.graph import graph_iden # TODO: expose other ranking algos like SMK rank_algo = 'LNBNN' infr.print('Exec {} ranking algorithm'.format(rank_algo), 1) ibs = infr.ibs if qaids is None: qaids = infr.aids qaids = ut.ensure_iterable(qaids) if daids is None: daids = infr.aids if cfgdict is None: cfgdict = { # 'can_match_samename': False, 'can_match_samename': True, 'can_match_sameimg': True, # 'augment_queryside_hack': True, 'K': 3, 'Knorm': 3, 'prescore_method': 'csum', 'score_method': 'csum', } cfgdict.update(infr.ranker_params) infr.print('Using LNBNN config = %r' % (cfgdict,)) # hack for using current nids if name_method == 'node': aids = sorted(set(ut.aslist(qaids) + ut.aslist(daids))) custom_nid_lookup = infr.get_node_attrs('name_label', aids) elif name_method == 'edge': custom_nid_lookup = { aid: nid for nid, cc in infr.pos_graph._ccs.items() for aid in cc } elif name_method == 'wbia': custom_nid_lookup = None else: raise KeyError('Unknown name_method={}'.format(name_method)) qreq_ = ibs.new_query_request( qaids, daids, cfgdict=cfgdict, custom_nid_lookup=custom_nid_lookup, verbose=infr.verbose >= 2, ) # cacher = qreq_.get_big_cacher() # if not cacher.exists(): # pass # # import sys # # sys.exit(1) cm_list = qreq_.execute( prog_hook=prog_hook, use_cache=use_cache, invalidate_supercache=invalidate_supercache, ) infr._set_vsmany_info(qreq_, cm_list) edges = set(infr._cm_breaking(cm_list, review_cfg={'ranks_top': 5})) return edges
def __delitem__(self, index): import utool as ut index = ut.ensure_iterable(index) ut.delete_items_by_index(self.found_fpath_list, index) ut.delete_items_by_index(self.found_lines_list, index) ut.delete_items_by_index(self.found_lxs_list, index)
def export(ibs, aid_pairs=None): """ 3 - 4 different animals 2 views of each matching keypoint coordinates on each annotation """ if aid_pairs is None: if ibs.get_dbname() == 'PZ_MOTHERS': aid_pair_list = MOTHERS_VIEWPOINT_EXPORT_PAIRS if ibs.get_dbname() == 'GZ_ALL': aid_pair_list = GZ_VIEWPOINT_EXPORT_PAIRS ibs.update_query_cfg(ratio_thresh=1.6) export_path = expanduser('~/Dropbox/Assignments/dataset') #utool.view_directory(export_path) # MOTHERS EG: for aid_pair in aid_pair_list: qaid2_qres = ibs.query_intra_encounter(aid_pair) #ibeis.viz.show_qres(ibs, qaid2_qres.values()[1]); df2.iup() mrids_list = [] mkpts_list = [] for qaid, qres in six.iteritems(qaid2_qres): print('Getting kpts from %r' % qaid) #qres.show_top(ibs) posrid_list = utool.ensure_iterable(qres.get_classified_pos()) mrids_list.extend([(qaid, posrid) for posrid in posrid_list]) mkpts_list.extend(qres.get_matching_keypoints(ibs, posrid_list)) mkey2_kpts = {} for mrids_tup, mkpts_tup in zip(mrids_list, mkpts_list): assert len(mrids_tup) == 2, 'must be a match tuple' mrids_ = np.array(mrids_tup) sortx = mrids_.argsort() mrids_ = mrids_[sortx] mkpts_ = np.array(mkpts_tup)[sortx] if sortx[0] == 0: pass mkey = tuple(mrids_.tolist()) try: kpts_list = mkey2_kpts[mkey] print('append to mkey=%r' % (mkey,)) except KeyError: print('new mkey=%r' % (mkey,)) kpts_list = [] kpts_list.append(mkpts_) mkey2_kpts[mkey] = kpts_list mkeys_list = mkey2_kpts.keys() mkeys_keypoints = mkey2_kpts.values() for mkeys, mkpts_list in zip(mkeys_list, mkeys_keypoints): print(mkeys) print(len(kpts_list)) kpts1_m = np.vstack([mkpts[0] for mkpts in mkpts_list]) kpts2_m = np.vstack([mkpts[1] for mkpts in mkpts_list]) match_lines = [ repr( ( tuple(kp1[ktool.LOC_DIMS].tolist()), tuple(kp2[ktool.LOC_DIMS].tolist()), ) ) + ', ' for kp1, kp2 in zip(kpts1_m, kpts2_m)] mcpaths_list = ibs.get_annot_cpaths(mkeys) fnames_list = map(lambda x: split(x)[1], mcpaths_list) for path in mcpaths_list: utool.copy(path, export_path) header_lines = ['# Exported keypoint matches (might be duplicates matches)', '# matching_aids = %r' % (mkey,)] header_lines += ['# img%d = %r' % (count, fname) for count, fname in enumerate(fnames_list)] header_lines += ['# LINE FORMAT: match_pts = [(img1_xy, img2_xy) ... ]'] header_text = '\n'.join(header_lines) match_text = '\n'.join(['match_pts = ['] + match_lines + [']']) matchfile_text = '\n'.join([header_text, match_text]) matchfile_name = ('match_aids(%d,%d).txt' % mkey) matchfile_path = join(export_path, matchfile_name) utool.write_to(matchfile_path, matchfile_text) print(header_text) print(utool.truncate_str(match_text, maxlen=500))