def update_registry(drive): print('Updating registered files in %r' % (drive,)) # Update existing files fpath_exists_list = list(map(exists, ut.ProgIter(drive.fpath_list, 'checkexist fpath', freq=1000))) dpath_exists_list = list(map(exists, ut.ProgIter(drive.dpath_list, 'checkexist dpath', freq=1000))) if all(fpath_exists_list): print('No change in file structure') else: print('%d/%d files no longer exist' % ( len(drive.fpath_list) - sum(fpath_exists_list), len(drive.fpath_list))) removed_fpaths = ut.compress(drive.fpath_list, ut.not_list(fpath_exists_list)) print('removed_fpaths = %s' % (ut.repr2(removed_fpaths),)) if all(dpath_exists_list): print('No change in dpath structure') else: print('%d/%d dirs no longer exist' % ( len(drive.dpath_list) - sum(dpath_exists_list), len(drive.dpath_list))) removed_dpaths = ut.compress( drive.dpath_list, ut.not_list(dpath_exists_list)) print('removed_dpaths = %s' % (ut.repr2(removed_dpaths),)) drive.fpath_list = ut.compress(drive.fpath_list, fpath_exists_list) drive.dpath_list = ut.compress(drive.dpath_list, dpath_exists_list) drive.cache.save('fpath_list', drive.fpath_list) drive.cache.save('dpath_list', drive.dpath_list)
def _debug_repr_model(model): cpd_code_list = [_debug_repr_cpd(cpd) for cpd in model.cpds] code_fmt = ut.codeblock( ''' import numpy as np import pgmpy import pgmpy.inference import pgmpy.factors import pgmpy.models {cpds} cpd_list = {nodes} input_graph = {edges} model = pgmpy.models.BayesianModel(input_graph) model.add_cpds(*cpd_list) infr = pgmpy.inference.BeliefPropagation(model) ''') code = code_fmt.format( cpds='\n'.join(cpd_code_list), nodes=ut.repr2(sorted(model.nodes()), strvals=True), edges=ut.repr2(sorted(model.edges()), nl=1), ) ut.print_code(code) ut.copy_text_to_clipboard(code)
def on_pick(event, infr=None): import wbia.plottool as pt logger.info('ON PICK: %r' % (event, )) artist = event.artist plotdat = pt.get_plotdat_dict(artist) if plotdat: if 'node' in plotdat: all_node_data = ut.sort_dict(plotdat['node_data'].copy()) visual_node_data = ut.dict_subset(all_node_data, infr.visual_node_attrs, None) node_data = ut.delete_dict_keys(all_node_data, infr.visual_node_attrs) node = plotdat['node'] node_data['degree'] = infr.graph.degree(node) node_label = infr.pos_graph.node_label(node) logger.info('visual_node_data: ' + ut.repr2(visual_node_data, nl=1)) logger.info('node_data: ' + ut.repr2(node_data, nl=1)) ut.cprint('node: ' + ut.repr2(plotdat['node']), 'blue') logger.info('(pcc) node_label = %r' % (node_label, )) logger.info('artist = %r' % (artist, )) elif 'edge' in plotdat: all_edge_data = ut.sort_dict(plotdat['edge_data'].copy()) logger.info(infr.repr_edge_data(all_edge_data)) ut.cprint('edge: ' + ut.repr2(plotdat['edge']), 'blue') logger.info('artist = %r' % (artist, )) else: logger.info('???: ' + ut.repr2(plotdat)) logger.info(ut.get_timestamp())
def _debug_repr_model(model): cpd_code_list = [_debug_repr_cpd(cpd) for cpd in model.cpds] code_fmt = ut.codeblock(''' import numpy as np import pgmpy import pgmpy.inference import pgmpy.factors import pgmpy.models {cpds} cpd_list = {nodes} input_graph = {edges} model = pgmpy.models.BayesianModel(input_graph) model.add_cpds(*cpd_list) infr = pgmpy.inference.BeliefPropagation(model) ''') code = code_fmt.format( cpds='\n'.join(cpd_code_list), nodes=ut.repr2(sorted(model.nodes()), strvals=True), edges=ut.repr2(sorted(model.edges()), nl=1), ) ut.print_code(code) ut.copy_text_to_clipboard(code)
def api_remote_wbia(remote_wbia_url, remote_api_func, remote_wbia_port=5001, **kwargs): import requests if GLOBAL_APP_ENABLED and GLOBAL_APP is None: raise ValueError('Flask has not been initialized') api_name = remote_api_func.__name__ route_list = list(GLOBAL_APP.url_map.iter_rules(api_name)) assert len(route_list) == 1, 'More than one route resolved' route = route_list[0] api_route = route.rule assert api_route.startswith('/api/'), 'Must be an API route' method_list = sorted(list(route.methods - set(['HEAD', 'OPTIONS']))) remote_api_method = method_list[0].upper() assert api_route is not None, 'Route could not be found' args = (remote_wbia_url, remote_wbia_port, api_route) remote_api_url = 'http://%s:%s%s' % args headers = {'Authorization': get_url_authorization(remote_api_url)} for key in kwargs.keys(): value = kwargs[key] if isinstance(value, (tuple, list, set)): value = str(list(value)) kwargs[key] = value logger.info('[REMOTE] %s' % ('-' * 80,)) logger.info('[REMOTE] Calling remote IBEIS API: %r' % (remote_api_url,)) logger.info('[REMOTE] \tMethod: %r' % (remote_api_method,)) if ut.DEBUG2 or ut.VERBOSE: logger.info('[REMOTE] \tHeaders: %s' % (ut.repr2(headers),)) logger.info('[REMOTE] \tKWArgs: %s' % (ut.repr2(kwargs),)) # Make request to server try: if remote_api_method == 'GET': req = requests.get(remote_api_url, headers=headers, data=kwargs, verify=False) elif remote_api_method == 'POST': req = requests.post( remote_api_url, headers=headers, data=kwargs, verify=False ) elif remote_api_method == 'PUT': req = requests.put(remote_api_url, headers=headers, data=kwargs, verify=False) elif remote_api_method == 'DELETE': req = requests.delete( remote_api_url, headers=headers, data=kwargs, verify=False ) else: message = '_api_result got unsupported method=%r' % (remote_api_method,) raise KeyError(message) except requests.exceptions.ConnectionError as ex: message = '_api_result could not connect to server %s' % (ex,) raise IOError(message) response = req.text converted = ut.from_json(value) response = converted.get('response', None) logger.info('[REMOTE] got response') if ut.DEBUG2: logger.info('response = %s' % (response,)) return response
def update_registry(drive): print('Updating registered files in %r' % (drive, )) # Update existing files fpath_exists_list = list( map(exists, ut.ProgIter(drive.fpath_list, 'checkexist fpath', freq=1000))) dpath_exists_list = list( map(exists, ut.ProgIter(drive.dpath_list, 'checkexist dpath', freq=1000))) if all(fpath_exists_list): print('No change in file structure') else: print('%d/%d files no longer exist' % (len(drive.fpath_list) - sum(fpath_exists_list), len(drive.fpath_list))) removed_fpaths = ut.compress(drive.fpath_list, ut.not_list(fpath_exists_list)) print('removed_fpaths = %s' % (ut.repr2(removed_fpaths), )) if all(dpath_exists_list): print('No change in dpath structure') else: print('%d/%d dirs no longer exist' % (len(drive.dpath_list) - sum(dpath_exists_list), len(drive.dpath_list))) removed_dpaths = ut.compress(drive.dpath_list, ut.not_list(dpath_exists_list)) print('removed_dpaths = %s' % (ut.repr2(removed_dpaths), )) drive.fpath_list = ut.compress(drive.fpath_list, fpath_exists_list) drive.dpath_list = ut.compress(drive.dpath_list, dpath_exists_list) drive.cache.save('fpath_list', drive.fpath_list) drive.cache.save('dpath_list', drive.dpath_list)
def _print_previous_loop_statistics(infr, count): # Print stats about what happend in the this loop history = infr.metrics_list[-count:] recover_blocks = ut.group_items([ (k, sum(1 for i in g)) for k, g in it.groupby(ut.take_column(history, 'recovering')) ]).get(True, []) infr.print(( 'Recovery mode entered {} times, ' 'made {} recovery decisions.').format( len(recover_blocks), sum(recover_blocks)), color='green') testaction_hist = ut.dict_hist(ut.take_column(history, 'test_action')) infr.print( 'Test Action Histogram: {}'.format( ut.repr4(testaction_hist, si=True)), color='yellow') if infr.params['inference.enabled']: action_hist = ut.dict_hist( ut.emap(frozenset, ut.take_column(history, 'action'))) infr.print( 'Inference Action Histogram: {}'.format( ub.repr2(action_hist, si=True)), color='yellow') infr.print( 'Decision Histogram: {}'.format(ut.repr2(ut.dict_hist( ut.take_column(history, 'pred_decision') ), si=True)), color='yellow') infr.print( 'User Histogram: {}'.format(ut.repr2(ut.dict_hist( ut.take_column(history, 'user_id') ), si=True)), color='yellow')
def make_wbia_cell_list(ibs): cell_template_list = get_default_cell_template_list(ibs) autogen_str = '# python -m wbia autogen_ipynb --launch --dbdir %r' % (ibs.get_dbdir()) # autogen_str = ut.make_autogen_str() dbname = ibs.get_dbname() dbdir = ibs.dbdir default_acfgstr = ut.get_argval('-a', type_=str, default='default:is_known=True') asreport = ut.get_argflag('--asreport') default_pcfgstr_list = ut.get_argval(('-t', '-p'), type_=list, default='default') default_pcfgstr = ut.repr3(default_pcfgstr_list, nobr=True) if asreport: annotconfig_list_body = ut.codeblock(ut.repr2(default_acfgstr)) pipeline_list_body = ut.codeblock(default_pcfgstr) else: annotconfig_list_body = ut.codeblock( ut.repr2(default_acfgstr) + '\n' + ut.codeblock( """ #'default:has_any=(query,),dpername=1,exclude_reference=True', #'default:is_known=True', #'default:is_known=True,minqual=good,require_timestamp=True,dcrossval_enc=1,view=left' #'default:qsame_imageset=True,been_adjusted=True,excluderef=True,qsize=10,dsize=20', #'default:require_timestamp=True,min_timedelta=3600', #'default:species=primary', #'unctrl:been_adjusted=True', #'timectrl:', #'timectrl:view=primary,minqual=good', #'default:minqual=good,require_timestamp=True,view=left,dcrossval_enc=1,joinme=1', #'default:minqual=good,require_timestamp=True,view=right,dcrossval_enc=1,joinme=1', #'default:minqual=ok,require_timestamp=True,view=left,dcrossval_enc=1,joinme=2', #'default:minqual=ok,require_timestamp=True,view=right,dcrossval_enc=1,joinme=2', """ ) ) pipeline_list_body = ut.codeblock( default_pcfgstr + '\n' + ut.codeblock( """ #'default', #'default:K=1,AI=False,QRH=True', #'default:K=1,RI=True,AI=False', #'default:K=1,adapteq=True', #'default:fg_on=[True,False]', """ ) ) locals_ = locals() _format = partial(ut.format_cells, locals_=locals_) cell_list = ut.flatten(map(_format, cell_template_list)) return cell_list
def filter_duplicate_acfgs(expanded_aids_list, acfg_list, acfg_name_list, verbose=None): """ Removes configs with the same expanded aids list CommandLine: # The following will trigger this function: wbia -m wbia get_annotcfg_list:0 -a timectrl timectrl:view=left --db PZ_MTEST """ from wbia.expt import annotation_configs if verbose is None: verbose = ut.VERBOSE acfg_list_ = [] expanded_aids_list_ = [] seen_ = ut.ddict(list) for acfg, (qaids, daids) in zip(acfg_list, expanded_aids_list): key = (ut.hashstr_arr27(qaids, 'qaids'), ut.hashstr_arr27(daids, 'daids')) if key in seen_: seen_[key].append(acfg) continue else: seen_[key].append(acfg) expanded_aids_list_.append((qaids, daids)) acfg_list_.append(acfg) if verbose: duplicate_configs = dict( [(key_, val_) for key_, val_ in seen_.items() if len(val_) > 1] ) if len(duplicate_configs) > 0: logger.info('The following configs produced duplicate annnotation configs') for key, val in duplicate_configs.items(): # Print the difference between the duplicate configs _tup = annotation_configs.compress_acfg_list_for_printing(val) nonvaried_compressed_dict, varied_compressed_dict_list = _tup logger.info('+--') logger.info('key = %r' % (key,)) logger.info( 'duplicate_varied_cfgs = %s' % (ut.repr2(varied_compressed_dict_list),) ) logger.info( 'duplicate_nonvaried_cfgs = %s' % (ut.repr2(nonvaried_compressed_dict),) ) logger.info('L__') if verbose >= 1: logger.info( '[harn.help] parsed %d / %d unique annot configs' % (len(acfg_list_), len(acfg_list)) ) if verbose > 2: logger.info('[harn.help] parsed from: %r' % (acfg_name_list,)) return expanded_aids_list_, acfg_list_
def print_dataset_info(data, labels, key): labelhist = {key: len(val) for key, val in ut.group_items(labels, labels).items()} stats_dict = ut.get_stats(data.ravel()) ut.delete_keys(stats_dict, ['shape', 'nMax', 'nMin']) print('[dataset] Dataset Info: ') print('[dataset] * Data:') print('[dataset] %s_data(shape=%r, dtype=%r)' % (key, data.shape, data.dtype)) print('[dataset] %s_memory(data) = %r' % (key, ut.get_object_size_str(data),)) print('[dataset] %s_stats(data) = %s' % (key, ut.repr2(stats_dict, precision=2),)) print('[dataset] * Labels:') print('[dataset] %s_labels(shape=%r, dtype=%r)' % (key, labels.shape, labels.dtype)) print('[dataset] %s_label histogram = %s' % (key, ut.repr2(labelhist)))
def repr_edge_data(infr, all_edge_data, visual=True): visual_edge_data = { k: v for k, v in all_edge_data.items() if k in infr.visual_edge_attrs } edge_data = ut.delete_dict_keys(all_edge_data.copy(), infr.visual_edge_attrs) lines = [] if visual: lines += [('visual_edge_data: ' + ut.repr2(visual_edge_data, nl=1)) ] lines += [('edge_data: ' + ut.repr2(edge_data, nl=1))] return '\n'.join(lines)
def report_partitioning_statistics(new_reduced_joint): # compute partitioning statistics import vtool as vt vals, idxs = vt.group_indices(new_reduced_joint.values.ravel()) #groupsize = list(map(len, idxs)) #groupassigns = ut.unflat_vecmap(new_reduced_joint.assignment, idxs) all_states = new_reduced_joint._row_labels(asindex=True) clusterstats = [tuple(sorted(list(ut.dict_hist(a).values()))) for a in all_states] grouped_vals = ut.group_items(new_reduced_joint.values.ravel(), clusterstats) #probs_assigned_to_clustertype = [( # sorted(np.unique(np.array(b).round(decimals=5)).tolist())[::-1], a) # for a, b in grouped_vals.items()] probs_assigned_to_clustertype = [( ut.dict_hist(np.array(b).round(decimals=5)), a) for a, b in grouped_vals.items()] sortx = ut.argsort([max(c[0].keys()) for c in probs_assigned_to_clustertype]) probs_assigned_to_clustertype = ut.take(probs_assigned_to_clustertype, sortx) # This list of 2-tuples with the first item being the unique # probabilies that are assigned to a cluster type along with the number # of times they were assigned. A cluster type is the second item. Every # number represents how many annotations were assigned to a specific # label. The length of that list is the number of total labels. For # all low scores you will see [[{somenum: 1}, {0: 800}], [1, 1, 1, ... 1]] # indicating that that the assignment of everyone to a different label happend once # where the probability was somenum and a 800 times where the probability was 0. #print(sorted([(b, a) for a, b in ut.map_dict_vals(sum, x)]).items()) #z = sorted([(b, a) for a, b in ut.map_dict_vals(sum, grouped_vals).items()]) print(ut.repr2(probs_assigned_to_clustertype, nl=2, precision=2, sorted_=True))
def print_annot_info(): logger.info('[interact_chip] Annotation Info = ' + ut.repr2(annot_info, nl=4)) logger.info('config2_ = %r' % (config2_, )) if config2_ is not None: logger.info('config2_.__dict__ = %s' % (ut.repr3(config2_.__dict__), ))
def draw_precision_recall_curve_(recall_range_, p_interp_curve, title_pref=None, fnum=1): import wbia.plottool as pt if recall_range_ is None: recall_range_ = np.array([]) p_interp_curve = np.array([]) fig = pt.figure(fnum=fnum, docla=True, doclf=True) # NOQA if recall_range_ is None: ave_p = np.nan else: ave_p = p_interp_curve.sum() / p_interp_curve.size pt.plot2( recall_range_, p_interp_curve, marker='o--', x_label='recall', y_label='precision', unitbox=True, flipx=False, color='r', title='Interplated Precision Vs Recall\n' + 'avep = %r' % ave_p, ) logger.info('Interplated Precision') logger.info(ut.repr2(list(zip(recall_range_, p_interp_curve))))
def preload_commands(dbdir, **kwargs): """ Preload commands work with command line arguments and global caches """ #print('[main_cmd] preload_commands') if params.args.dump_argv: print(ut.repr2(vars(params.args), sorted_=False)) if params.args.dump_global_cache: ut.global_cache_dump() # debug command, dumps to stdout if params.args.set_workdir is not None: sysres.set_workdir(params.args.set_workdir) if params.args.get_workdir: print(' Current work dir = %s' % sysres.get_workdir()) if params.args.logdir is not None: sysres.set_logdir(params.args.logdir) if params.args.get_logdir: print(' Current local log dir = %s' % (sysres.get_logdir_local(),)) print(' Current global log dir = %s' % (sysres.get_logdir_global(),)) if params.args.view_logdir: ut.view_directory(sysres.get_logdir_local()) ut.view_directory(sysres.get_logdir_global()) if params.args.view_logdir_local: ut.view_directory(sysres.get_logdir_local()) if params.args.view_logdir_global: ut.view_directory(sysres.get_logdir_local()) if ut.get_argflag('--vwd'): vwd() if ut.get_argflag('--vdq'): print('got arg --vdq') vdq(dbdir) if kwargs.get('delete_ibsdir', False): ibsfuncs.delete_ibeis_database(dbdir) if params.args.preload_exit: print('[main_cmd] preload exit') sys.exit(0)
def make_option_dict(options, shortcuts=True): """ helper for popup menu callbacks """ keys = ut.take_column(options, 0) values = ut.take_column(options, 1) if shortcuts: shortcut_keys = [ key[key.find('&') + 1] if '&' in key else None for key in keys ] try: ut.assert_unique(shortcut_keys, name='shortcut_keys', ignore=[None]) except AssertionError: print('shortcut_keys = %r' % (shortcut_keys, )) print('options = %r' % (ut.repr2(options), )) raise shortcut_dict = { sc_key: val # sc_key: (make_option_dict(val, shortcuts=True) # if isinstance(val, list) else val) for (sc_key, val) in zip(shortcut_keys, values) if sc_key is not None and not isinstance(val, list) } return shortcut_dict else: ut.assert_unique(keys, name='option_keys') fulltext_dict = { key: (make_option_dict(val, shortcuts=False) if isinstance( val, list) else val) for (key, val) in zip(keys, values) if key is not None } return fulltext_dict
def load_gztest(ibs): r""" CommandLine: python -m wbia.algo.hots.special_query --test-load_gztest Example: >>> # DISABLE_DOCTEST >>> from wbia.algo.hots.devcases import * # NOQA >>> import wbia >>> ibs = wbia.opendb('GZ_ALL') """ from os.path import join from wbia.algo.hots import match_chips4 as mc4 dir_ = ut.get_module_dir(mc4) eval_text = ut.read_from(join(dir_, 'GZ_TESTTUP.txt')) testcases = eval(eval_text) count_dict = ut.count_dict_vals(testcases) logger.info(ut.repr2(count_dict)) testtup_list = ut.flatten( ut.dict_take_list( testcases, [ 'vsone_wins', 'vsmany_outperformed', 'vsmany_dominates', 'vsmany_wins' ], )) qaid_list = [testtup.qaid_t for testtup in testtup_list] visual_uuids = ibs.get_annot_visual_uuids(qaid_list) visual_uuids
def find_close_incorrect_match(ibs, qaids): use_cache = False save_qcache = False cfgdict_vsmany = dict( index_method='single', pipeline_root='vsmany', ) qres_vsmany_list, qreq_vsmany_ = ibs.query_chips( qaids, ibs.get_valid_aids(), cfgdict=cfgdict_vsmany, return_request=True, use_cache=use_cache, save_qcache=save_qcache, verbose=True, ) qres_vsmany = qres_vsmany_list[0] qres_vsmany.ishow_top(ibs) top_aids = qres_vsmany.get_top_aids() top_nids = ibs.get_annot_nids(top_aids) qaid = qaids[0] qnid = ibs.get_annot_nids(qaid) is_groundfalse = [nid != qnid for nid in top_nids] top_gf_aids = ut.compress(top_aids, is_groundfalse) # top_gt_aids = ut.filterfalse_items(top_aids, is_groundfalse) top_gf_vuuids = ibs.get_annot_visual_uuids(top_gf_aids) qvuuid = ibs.get_annot_visual_uuids(qaid) gf_mapping = {qvuuid: top_gf_vuuids[0:1]} logger.info('gf_mapping = ' + ut.repr2(gf_mapping)) pass
def make_ibeis_cell_list(ibs): cell_template_list = get_default_cell_template_list(ibs) autogen_str = make_autogen_str() dbname = ibs.get_dbname() #if ut.get_argflag('--hacktestscore'): # annotconfig_list_body = ut.codeblock( # ''' # 'timectrl', # ''' # ) #else: default_acfgstr = ut.get_argval('-a', type_=str, default='default:is_known=True') annotconfig_list_body = ut.codeblock( ut.repr2(default_acfgstr) + '\n' + ut.codeblock(''' # See ibeis/expt/annotation_configs.py for names of annot configuration options #'default:has_any=(query,),dpername=1,exclude_reference=True', #'default:is_known=True', #'default:qsame_encounter=True,been_adjusted=True,excluderef=True' #'default:qsame_encounter=True,been_adjusted=True,excluderef=True,qsize=10,dsize=20', #'default:require_timestamp=True,min_timedelta=3600', #'default:species=primary', #'timectrl:', #'timectrl:been_adjusted=True,dpername=3', #'timectrl:qsize=10,dsize=20', #'unctrl:been_adjusted=True', ''') ) #if ut.get_argflag('--hacktestscore'): # pipeline_list_body = ut.codeblock( # ''' # # See ibeis/algo/Config.py for names of pipeline config options # 'default:lnbnn_on=True,bar_l2_on=False,normonly_on=False,fg_on=True', # 'default:lnbnn_on=False,bar_l2_on=True,normonly_on=False,fg_on=True', # 'default:lnbnn_on=False,bar_l2_on=False,normonly_on=True,fg_on=True', # 'default:lnbnn_on=True,bar_l2_on=False,normonly_on=False,fg_on=False', # 'default:lnbnn_on=False,bar_l2_on=True,normonly_on=False,fg_on=False', # 'default:lnbnn_on=False,bar_l2_on=False,normonly_on=True,fg_on=False', # ''' # ) #elif True: default_pcfgstr_list = ut.get_argval(('-t', '-p'), type_=list, default='default') default_pcfgstr = ut.repr3(default_pcfgstr_list, nobr=True) pipeline_list_body = ut.codeblock( default_pcfgstr + '\n' + ut.codeblock(''' #'default', #'default:K=1', #'default:K=1,AI=False', #'default:K=1,AI=False,QRH=True', #'default:K=1,RI=True,AI=False', #'default:K=1,adapteq=True', #'default:fg_on=[True,False]', ''') ) locals_ = locals() _format = partial(format_cells, locals_=locals_) cell_list = ut.flatten(map(_format, cell_template_list)) return cell_list
def _test_int_byte_conversion(): import itertools as it import utool as ut inputs = list(it.chain( range(0, 10), (2 ** i for i in range(0, 256, 32)), (2 ** i + 1 for i in range(0, 256, 32)), )) for int_0 in inputs: print('---') print('int_0 = %s' % (ut.repr2(int_0),)) bytes_ = _int_to_bytes(int_0) int_ = _bytes_to_int(bytes_) print('bytes_ = %s' % (ut.repr2(bytes_),)) print('int_ = %s' % (ut.repr2(int_),)) assert int_ == int_0
def checkinfo_wrapper(*args, **kwargs): suggested_fix = '' funcname = get_funcname(func) packagename = funcname.replace('_version', '') pipname_ = pipname if pipname is not None else packagename try: infodict = func(*args, **kwargs) except ImportError as ex: infodict = module_stdinfo_dict(None, name=pipname_) suggested_fix = 'pip install ' + pipname_ if not sys.platform.startswith('win32'): suggested_fix = 'sudo ' + suggested_fix return False, 'None', target, infodict, ut.formatex( ex), suggested_fix except Exception as ex: infodict = module_stdinfo_dict(None, name=pipname_) return False, 'None', target, infodict, ut.formatex( ex), 'Some unknown error in ' + packagename current_version = infodict['__version__'] # Build status text msg = ut.repr2(infodict, strvals=True) msg += '\n' + '%s: %r >= (target=%r)?' % (funcname, current_version, target) statustext = ut.msgblock(infodict['__name__'], msg) # Check if passed passed = version_ge_target(current_version, target) # Suggest possible fix if not passed: suggested_fix = 'pip install ' + infodict[ '__name__'] + ' --upgrade' if not sys.platform.startswith('win32'): suggested_fix = 'sudo ' + suggested_fix return passed, current_version, target, infodict, statustext, suggested_fix
def figure_clicked(self, event=None): from wbia.viz import viz_helpers as vh import wbia.guitool as gt ax = event.inaxes if ih.clicked_inside_axis(event): viztype = vh.get_ibsdat(ax, 'viztype') if viztype == 'chip': aid = vh.get_ibsdat(ax, 'aid') print('... aid=%r' % aid) if event.button == 3: # right-click from wbia.viz.interact import interact_chip height = self.fig.canvas.geometry().height() qpoint = gt.newQPoint(event.x, height - event.y) if self.qreq_ is None: config2_ = None else: if aid in self.qreq_.qaids: config2_ = self.qreq_.query_config2_ else: config2_ = self.qreq_.data_config2_ callback_list = interact_chip.build_annot_context_options( self.ibs, aid, refresh_func=self.show_page, config2_=config2_ ) gt.popup_menu(self.fig.canvas, qpoint, callback_list) # interact_chip.show_annot_context_menu( # self.ibs, aid, self.fig.canvas, qpoint, refresh_func=self.show_page) # self.show_page() # ibs.print_annotation_table() print(ut.repr2(event.__dict__))
def _debug_repr_cpd(cpd): import re import utool as ut code_fmt = ut.codeblock( ''' {variable} = pgmpy.factors.TabularCPD( variable={variable_repr}, variable_card={variable_card_repr}, values={get_cpd_repr}, evidence={evidence_repr}, evidence_card={evidence_card_repr}, ) ''') keys = ['variable', 'variable_card', 'values', 'evidence', 'evidence_card'] dict_ = ut.odict(zip(keys, [getattr(cpd, key) for key in keys])) # HACK dict_['values'] = cpd.get_cpd() r = ut.repr2(dict_, explicit=True, nobraces=True, nl=True) print(r) # Parse props that are needed for this fmtstr fmt_keys = [match.groups()[0] for match in re.finditer('{(.*?)}', code_fmt)] need_reprs = [key[:-5] for key in fmt_keys if key.endswith('_repr')] need_keys = [key for key in fmt_keys if not key.endswith('_repr')] # Get corresponding props # Call methods if needbe tmp = [(prop, getattr(cpd, prop)) for prop in need_reprs] tmp = [(x, y()) if ut.is_funclike(y) else (x, y) for (x, y) in tmp] fmtdict = dict(tmp) fmtdict = ut.map_dict_vals(ut.repr2, fmtdict) fmtdict = ut.map_dict_keys(lambda x: x + '_repr', fmtdict) tmp2 = [(prop, getattr(cpd, prop)) for prop in need_keys] fmtdict.update(dict(tmp2)) code = code_fmt.format(**fmtdict) return code
def get_cfgstr(nnindexer, noquery=False): r""" returns string which uniquely identified configuration and support data Args: noquery (bool): if True cfgstr is only relevant to building the index. No search params are returned (default = False) Returns: str: flann_cfgstr CommandLine: python -m wbia.algo.hots.neighbor_index --test-get_cfgstr Example: >>> # DISABLE_DOCTEST >>> from wbia.algo.hots.neighbor_index import * # NOQA >>> import wbia >>> cfgdict = dict(fg_on=False) >>> qreq_ = wbia.testdata_qreq_(defaultdb='testdb1', p='default:fg_on=False') >>> qreq_.load_indexer() >>> nnindexer = qreq_.indexer >>> noquery = True >>> flann_cfgstr = nnindexer.get_cfgstr(noquery) >>> result = ('flann_cfgstr = %s' % (str(flann_cfgstr),)) >>> print(result) flann_cfgstr = _FLANN((algo=kdtree,seed=42,t=8,))_VECS((11260,128)gj5nea@ni0%f3aja) """ flann_cfgstr_list = [] use_params_hash = True use_data_hash = True if use_params_hash: flann_defaults = vt.get_flann_params( nnindexer.flann_params['algorithm']) # flann_params_clean = flann_defaults.copy() flann_params_clean = ut.sort_dict(flann_defaults) ut.update_existing(flann_params_clean, nnindexer.flann_params) if noquery: ut.delete_dict_keys(flann_params_clean, ['checks']) shortnames = dict(algorithm='algo', checks='chks', random_seed='seed', trees='t') short_params = ut.odict([ (shortnames.get(key, key), str(val)[0:7]) for key, val in six.iteritems(flann_params_clean) ]) flann_valsig_ = ut.repr2(short_params, nl=False, explicit=True, strvals=True) flann_valsig_ = flann_valsig_.lstrip('dict').replace(' ', '') # flann_valsig_ = str(list(flann_params.values())) # flann_valsig = ut.remove_chars(flann_valsig_, ', \'[]') flann_cfgstr_list.append('_FLANN(' + flann_valsig_ + ')') if use_data_hash: vecs_hashstr = ut.hashstr_arr(nnindexer.idx2_vec, '_VECS') flann_cfgstr_list.append(vecs_hashstr) flann_cfgstr = ''.join(flann_cfgstr_list) return flann_cfgstr
def make_ibeis_cell_list(ibs): cell_template_list = get_default_cell_template_list(ibs) autogen_str = ut.make_autogen_str() dbname = ibs.get_dbname() default_acfgstr = ut.get_argval('-a', type_=str, default='default:is_known=True') asreport = ut.get_argflag('--asreport') default_pcfgstr_list = ut.get_argval(('-t', '-p'), type_=list, default='default') default_pcfgstr = ut.repr3(default_pcfgstr_list, nobr=True) if asreport: annotconfig_list_body = ut.codeblock( ut.repr2(default_acfgstr) ) pipeline_list_body = ut.codeblock( default_pcfgstr ) else: annotconfig_list_body = ut.codeblock( ut.repr2(default_acfgstr) + '\n' + ut.codeblock(''' # See ibeis/expt/annotation_configs.py for names of annot configuration options #'default:has_any=(query,),dpername=1,exclude_reference=True', #'default:is_known=True', #'default:qsame_imageset=True,been_adjusted=True,excluderef=True,qsize=10,dsize=20', #'default:require_timestamp=True,min_timedelta=3600', #'default:species=primary', #'timectrl:', #'unctrl:been_adjusted=True', ''') ) pipeline_list_body = ut.codeblock( default_pcfgstr + '\n' + ut.codeblock(''' #'default', #'default:K=1,AI=False,QRH=True', #'default:K=1,RI=True,AI=False', #'default:K=1,adapteq=True', #'default:fg_on=[True,False]', ''') ) locals_ = locals() _format = partial(ut.format_cells, locals_=locals_) cell_list = ut.flatten(map(_format, cell_template_list)) return cell_list
def biggest_files(drive): print('Biggest Files in %r' % (drive,)) sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(drive.fpath_bytes_list, sel) biggest_files = ut.take(drive.fpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_files)) print(ut.repr2(biginfo_list, strvals=True))
def wrp_cache_invalidator(self, *args, **kwargs): # the class must have a table_cache property colscache_ = self.table_cache[tblname] colnames_ = list( six.iterkeys(colscache_)) if colnames is None else colnames if DEBUG_API_CACHE: indenter = ut.Indenter('[%s]' % (tblname, )) indenter.start() logger.info('+------') logger.info( 'INVALIDATING tblname=%r, colnames=%r, rowidx=%r, force=%r' % (tblname, colnames, rowidx, force)) logger.info('self = %r' % (self, )) logger.info('args = %r' % (args, )) logger.info('kwargs = %r' % (kwargs, )) logger.info('colscache_ = ' + ut.repr2(colscache_, truncate=1)) # Clear the cache of any specified colname # when the invalidator is called if rowidx is None: for colname in colnames_: kwargs_cache_ = colscache_[colname] # We dont know the rowsids so clear everything for cache_ in six.itervalues(kwargs_cache_): cache_.clear() else: rowid_list = args[rowidx] for colname in colnames_: kwargs_cache_ = colscache_[colname] # We know the rowids to delete # iterate over all getter kwargs values for cache_ in six.itervalues(kwargs_cache_): ut.delete_dict_keys(cache_, rowid_list) # Preform set/delete action if DEBUG_API_CACHE: logger.info('After:') logger.info('colscache_ = ' + ut.repr2(colscache_, truncate=1)) logger.info('L__________') writer_result = writer_func(self, *args, **kwargs) if DEBUG_API_CACHE: indenter.stop() return writer_result
def biggest_files(drive): print('Biggest Files in %r' % (drive, )) sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(drive.fpath_bytes_list, sel) biggest_files = ut.take(drive.fpath_list, sel) biginfo_list = list( zip(map(ut.byte_str2, biggest_nbytes), biggest_files)) print(ut.repr2(biginfo_list, strvals=True))
def drive_test_script(ibs): r""" Test script where we drive around and take pictures of animals both in a given database and not in a given databse to make sure the system works. CommandLine: python -m wbia.viz.viz_image --test-drive_test_script python -m wbia.viz.viz_image --test-drive_test_script --db PZ_MTEST --show python -m wbia.viz.viz_image --test-drive_test_script --db GIR_Tanya --show python -m wbia.viz.viz_image --test-drive_test_script --db GIR_Master0 --show python -m wbia.viz.viz_image --test-drive_test_script --db PZ_Master0 --show python -m wbia.viz.viz_image --test-drive_test_script --db PZ_FlankHack --show python -m wbia.viz.viz_image --test-drive_test_script --db PZ_FlankHack --show python -m wbia.viz.viz_image --test-drive_test_script --dbdir /raid/work2/Turk/GIR_Master --show Example: >>> # DISABLE_DOCTEST >>> from wbia.viz.viz_image import * # NOQA >>> import wbia >>> ibs = wbia.opendb() >>> drive_test_script(ibs) """ import wbia aid_list = wbia.testdata_aids(a='default:pername=1') logger.info('Running with (annot) aid_list = %r' % (aid_list)) gid_list = ibs.get_annot_gids(aid_list) logger.info('Running with (image) gid_list = %r' % (gid_list)) avuuid_list = ibs.get_annot_visual_uuids(aid_list) guuid_list = ibs.get_image_uuids(gid_list) logger.info('Running with annot_visual_uuid_list = %s' % (ut.repr2(zip(aid_list, avuuid_list)))) logger.info('Running with image_uuid_list = %s' % (ut.repr2(zip(gid_list, guuid_list)))) for gid, aid in ut.ProgressIter(zip(gid_list, aid_list), lbl='progress '): logger.info('\ngid, aid, nid = %r, %r, %r' % ( gid, aid, ibs.get_annot_nids(aid), )) show_image(ibs, gid, annote=False, rich_title=True) pt.show_if_requested()
def inspect_deck(deck): def get_card_tags(card, deck): tags = [] stats = card.mana_source_stats(deck) if stats is not None: tags.append("land") if len(stats[1]) > 0: tags.append("tapland") else: tags.append("untapland") return tags # ------------ print("len(deck) = %r" % (len(deck),)) tags_list = [get_card_tags(card, deck) for card in deck.card_list] print("Deck Counts:") print(ut.repr2(ut.dict_hist(ut.flatten(tags_list)), nl=True)) hand = deck.sample_hand() manastats_list = [card.mana_source_stats(deck) for card in hand] print(ut.list_str([card.name + ": " + text_type(stats) for card, stats in zip(hand, manastats_list)])) tags_list = [get_card_tags(card, deck) for card in hand] print("Hand Counts") print(ut.repr2(ut.dict_hist(ut.flatten(tags_list)), nl=True)) valid_tags = ["land", "tapland", "untapland"] x = {tag: [] for tag in valid_tags} for _ in range(500): hand = deck.sample_hand() tags_list = [get_card_tags(card, deck) for card in hand] taghist = ut.dict_hist(ut.flatten(tags_list)) for key, val in x.items(): val.append(taghist.get(key, 0)) print("Monte Stats:") for key, val in list(x.items()): print("%15s: %s" % (key, ut.repr2(ut.get_stats(val), precision=2))) def hand_stats(): # [card.types for card in hand] # [card.rrr() for card in hand] [card.mana_source_stats(deck) for card in hand] card.types
def _make_deploy_metadata(self, task_key=None): pblm = self.pblm if pblm.samples is None: pblm.setup() if task_key is None: task_key = pblm.primary_task_key # task_keys = list(pblm.samples.supported_tasks()) clf_key = pblm.default_clf_key data_key = pblm.default_data_key # Save the classifie data_info = pblm.feat_extract_info[data_key] feat_extract_config, feat_dims = data_info samples = pblm.samples labels = samples.subtasks[task_key] edge_hashid = samples.edge_set_hashid() label_hashid = samples.task_label_hashid(task_key) tasksamp_hashid = samples.task_sample_hashid(task_key) annot_hashid = ut.hashid_arr(samples._unique_annots.visual_uuids, 'annots') # species = pblm.infr.ibs.get_primary_database_species( # samples._unique_annots.aid) species = '+'.join(sorted(set(samples._unique_annots.species))) metadata = { 'tasksamp_hashid': tasksamp_hashid, 'edge_hashid': edge_hashid, 'label_hashid': label_hashid, 'annot_hashid': annot_hashid, 'class_hist': labels.make_histogram(), 'class_names': labels.class_names, 'data_info': data_info, 'task_key': task_key, 'species': species, 'data_key': data_key, 'clf_key': clf_key, 'n_dims': len(feat_dims), # 'aid_pairs': samples.aid_pairs, } meta_cfgstr = ut.repr2(metadata, kvsep=':', itemsep='', si=True) hashid = ut.hash_data(meta_cfgstr)[0:16] deploy_fname = self.fname_fmtstr.format(hashid=hashid, ** metadata) + '.cPkl' deploy_metadata = metadata.copy() deploy_metadata['hashid'] = hashid deploy_metadata['fname'] = deploy_fname return deploy_metadata, deploy_fname
def on_contextMenuRequested(widget, index, pos): logger.info('context request') if widget.api is not None: logger.info(ut.repr2(widget.api.get_available_colnames())) # HACK test # widget.api.add_column_names(['qx2_gt_rank', 'qx2_gf_rank', 'qx2_gt_raw_score', 'qx2_gf_raw_score']) widget.refresh_headers() # widget.change_headers(widget.api.make_headers()) if VERBOSE_ITEM_WIDGET: logger.info('context request')
def make_ibeis_cell_list(ibs): cell_template_list = get_default_cell_template_list(ibs) autogen_str = ut.make_autogen_str() dbname = ibs.get_dbname() default_acfgstr = ut.get_argval('-a', type_=str, default='default:is_known=True') asreport = ut.get_argflag('--asreport') default_pcfgstr_list = ut.get_argval(('-t', '-p'), type_=list, default='default') default_pcfgstr = ut.repr3(default_pcfgstr_list, nobr=True) if asreport: annotconfig_list_body = ut.codeblock(ut.repr2(default_acfgstr)) pipeline_list_body = ut.codeblock(default_pcfgstr) else: annotconfig_list_body = ut.codeblock( ut.repr2(default_acfgstr) + '\n' + ut.codeblock(''' # See ibeis/expt/annotation_configs.py for names of annot configuration options #'default:has_any=(query,),dpername=1,exclude_reference=True', #'default:is_known=True', #'default:qsame_imageset=True,been_adjusted=True,excluderef=True,qsize=10,dsize=20', #'default:require_timestamp=True,min_timedelta=3600', #'default:species=primary', #'timectrl:', #'unctrl:been_adjusted=True', ''')) pipeline_list_body = ut.codeblock(default_pcfgstr + '\n' + ut.codeblock(''' #'default', #'default:K=1,AI=False,QRH=True', #'default:K=1,RI=True,AI=False', #'default:K=1,adapteq=True', #'default:fg_on=[True,False]', ''')) locals_ = locals() _format = partial(ut.format_cells, locals_=locals_) cell_list = ut.flatten(map(_format, cell_template_list)) return cell_list
def main(): if True: import pandas as pd pd.options.display.max_rows = 1000 pd.options.display.width = 1000 basis = { #'n_clusters': [10, 100, 1000, 2000][::-1], #'n_features': [4, 32, 128, 512][::-1], #'per_cluster': [1, 10, 100, 200][::-1], 'n_clusters': [10, 100, 500][::-1], 'n_features': [32, 128][::-1], 'per_cluster': [1, 10, 20][::-1], 'asint': [True, False], } vals = [] for kw in ut.ProgIter(ut.all_dict_combinations(basis), lbl='gridsearch', bs=False, adjust=False, freq=1): print('kw = ' + ut.repr2(kw)) exec(ut.execstr_dict(kw)) centers1, new_speed = test_kmeans_plus_plus_speed(fix=True, **kw) centers2, old_speed = test_kmeans_plus_plus_speed(fix=False, **kw) import utool with utool.embed_on_exception_context: assert np.all(centers1 == centers2), 'new code disagrees' kw['new_speed'] = new_speed kw['old_speed'] = old_speed vals.append(kw) print('---------') df = pd.DataFrame.from_dict(vals) df['percent_change'] = 100 * (df['old_speed'] - df['new_speed']) / df['old_speed'] df = df.reindex_axis(list(basis.keys()) + ['new_speed', 'old_speed', 'percent_change'], axis=1) df['absolute_change'] = (df['old_speed'] - df['new_speed']) print(df.sort('absolute_change', ascending=False)) #print(df) print(df['percent_change'][df['absolute_change'] > .1].mean()) #print(df.loc[df['percent_change'].argsort()[::-1]]) else: new_speed = test_kmeans_plus_plus_speed() try: profile.dump_stats('out.lprof') profile.print_stats(stripzeros=True) except Exception: pass print('new_speed = %r' % (new_speed, ))
def _image_view(sel_aids=sel_aids, **_kwargs): try: viz.show_image(ibs, gid, sel_aids=sel_aids, fnum=self.fnum, **_kwargs) df2.set_figtitle('Image View') except TypeError as ex: ut.printex(ex, ut.repr2(_kwargs)) raise
def csv_format(r): text = ut.repr2(r, precision=precision) #text = six.text_type(r) # Check if needs escape escape_chars = ['"', ' ', ','] if any([c in text for c in escape_chars]): # escape quotes with quotes text = text.replace('"', '""') # encapsulate with quotes text = '"' + text + '"' return text
def _grep_dpath(dpath): grep_tup = ut.grep([pat], dpath_list=[dpath], exclude_patterns=['*.pyc'], verbose=False) reflags = 0 (found_fpath_list, found_lines_list, found_lxs_list) = grep_tup regex_list = [pat] _exprs_flags = [ut.util_regex.extend_regex2(expr, reflags) for expr in regex_list] extended_regex_list = ut.take_column(_exprs_flags, 0) grep_result = ut.GrepResult(found_fpath_list, found_lines_list, found_lxs_list, extended_regex_list, reflags=reflags) text = '\n'.join([ 'Greping Directory "{}"'.format(dpath), 'tofind_list={}'.format(ut.repr2(extended_regex_list)), grep_result.make_resultstr(colored=False), '=============', 'found_fpath_list = {}'.format(ut.repr2(found_fpath_list, nl=1)) ]) return text
def expand_closure_source(funcname, func): source = ut.get_func_sourcecode(func) closure_vars = [(k, v.cell_contents) for k, v in zip(func.func_code.co_freevars, func.func_closure)] source = ut.unindent(source) import re for k, v in closure_vars: source = re.sub('\\b' + k + '\\b', ut.repr2(v), source) source = re.sub(r'def .*\(self', 'def ' + funcname + '(self', source) source = ut.indent(source.strip(), ' ') + '\n' return source
def rcv_multipart_json(sock, num=2, print=print): """ helper """ # note that the first two parts will be ['Controller.ROUTER', 'Client.<id_>'] # these are needed for the reply to propagate up to the right client multi_msg = sock.recv_multipart() if VERBOSE_JOBS: print('----') print('RCV Json: %s' % (ut.repr2(multi_msg, truncate=True),)) idents = multi_msg[:num] request_json = multi_msg[num] request = ut.from_json(request_json) return idents, request
def biggest_dirs(drive): print('Biggest Dirs in %r' % (drive,)) dpath_list = drive.dpath_list fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list) unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list, fidxs_list) dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list)) sortx = ut.list_argsort(dpath_nbytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(dpath_nbytes_list, sel) biggest_dpaths = ut.take(dpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths)) print(ut.repr2(biginfo_list, strvals=True)) pass
def get_job_result(jobiface, jobid): with ut.Indenter('[client %d] ' % (jobiface.id_)): if jobiface.verbose >= 1: print = partial(ut.colorprint, color='teal') print('----') print('Request result of jobid=%r' % (jobid,)) pair_msg = dict(action='job_result', jobid=jobid) jobiface.collect_deal_sock.send_json(pair_msg) if jobiface.verbose >= 3: print('... waiting for collector reply') reply = jobiface.collect_deal_sock.recv_json() if jobiface.verbose >= 2: print('got reply = %s' % (ut.repr2(reply, truncate=True),)) return reply
def pandas_repr(df): import utool as ut args = [ df.values, ] kwargs = [ ('columns', df.columns.values.tolist()), ('index', df.index.values.tolist()), ] header = 'pd.DataFrame(' footer = ')' arg_parts = [ ut.hz_str(' ', ut.repr2(arg)) for arg in args if arg is not None ] kwarg_parts = [ ut.hz_str(' {}={}'.format(key, ut.repr2(val))) for key, val in kwargs if val is not None ] body = ',\n'.join(arg_parts + kwarg_parts) dfrepr = '\n'.join([header, body, footer]) print(dfrepr) pass
def find_nonunique_names(self): fnames = map(basename, self.rel_fpath_list) duplicate_map = ut.find_duplicate_items(fnames) groups = [] for dupname, idxs in duplicate_map.items(): uuids = self.get_prop('uuids', idxs) fpaths = self.get_prop('abs', idxs) groups = ut.group_items(fpaths, uuids) if len(groups) > 1: if all(x == 1 for x in map(len, groups.values())): # All groups are different, this is an simpler case print(ut.repr2(groups, nl=3)) else: # Need to handle the multi-item groups first pass
def __nice__(card): if not hasattr(card, 'state'): card.state = [] body = card.name if 'mana_cost' in card.nice_attrs and card.mana_cost: body += ' (' + card.mana_cost + ')' if 'state' in card.nice_attrs and len(card.state) > 0: state_nice = { # 'tapped': TAPPED } body += ' ' + ut.repr2([state_nice.get(s, s) for s in card.state]) if 'rarity' in card.nice_attrs and card.rarity: body += ' ' + card.rarity return body
def find_module_callers(): """ TODO: attempt to build a call graph between module functions to make it easy to see what can be removed and what cannot. """ import utool as ut from os.path import normpath mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_analyzer.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_all.py') mod_fpath = ut.truepath('~/code/ibeis/ibeis/expt/results_organizer.py') module = ut.import_module_from_fpath(mod_fpath) user_profile = ut.ensure_user_profile() doctestables = list(ut.iter_module_doctestable(module, include_builtin=False)) grepkw = {} grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['verbose'] = True usage_map = {} for funcname, func in doctestables: print('Searching for funcname = %r' % (funcname,)) found_fpath_list, found_lines_list, found_lxs_list = ut.grep([funcname], **grepkw) used_in = (found_fpath_list, found_lines_list, found_lxs_list) usage_map[funcname] = used_in external_usage_map = {} for funcname, used_in in usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in isexternal_flag = [normpath(fpath) != normpath(mod_fpath) for fpath in found_fpath_list] ext_used_in = (ut.compress(found_fpath_list, isexternal_flag), ut.compress(found_lines_list, isexternal_flag), ut.compress(found_lxs_list, isexternal_flag)) external_usage_map[funcname] = ext_used_in for funcname, used_in in external_usage_map.items(): (found_fpath_list, found_lines_list, found_lxs_list) = used_in print('Calling modules: \n' + ut.repr2(ut.unique_ordered(ut.flatten([used_in[0] for used_in in external_usage_map.values()])), nl=True))
def get_short_infostr(ibs): """ Returns printable database information Args: ibs (IBEISController): ibeis controller object Returns: str: infostr CommandLine: python -m ibeis.other.dbinfo --test-get_short_infostr Example: >>> # ENABLE_DOCTEST >>> from ibeis.other.dbinfo import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb('testdb1') >>> infostr = get_short_infostr(ibs) >>> result = str(infostr) >>> print(result) dbname = 'testdb1' num_images = 13 num_annotations = 13 num_names = 7 """ dbname = ibs.get_dbname() #workdir = ut.unixpath(ibs.get_workdir()) num_images = ibs.get_num_images() num_annotations = ibs.get_num_annotations() num_names = ibs.get_num_names() #workdir = %r infostr = ut.codeblock(''' dbname = %s num_images = %r num_annotations = %r num_names = %r ''' % (ut.repr2(dbname), num_images, num_annotations, num_names)) return infostr
def test_model(num_annots, num_names, score_evidence=[], name_evidence=[], other_evidence={}, noquery=False, verbose=None, **kwargs): if verbose is None: verbose = ut.VERBOSE method = kwargs.pop('method', None) model = make_name_model(num_annots, num_names, verbose=verbose, **kwargs) if verbose: model.print_priors(ignore_ttypes=['match', 'score']) model, evidence, soft_evidence = update_model_evidence( model, name_evidence, score_evidence, other_evidence) if verbose and len(soft_evidence) != 0: model.print_priors(ignore_ttypes=['match', 'score'], title='Soft Evidence', color='green') #if verbose: # ut.colorprint('\n --- Soft Evidence ---', 'white') # for ttype, cpds in model.ttype2_cpds.items(): # if ttype != 'match': # for fs_ in ut.ichunks(cpds, 4): # ut.colorprint(ut.hz_str([f._cpdstr('psql') for f in fs_]), # 'green') if verbose: ut.colorprint('\n --- Inference ---', 'red') if (len(evidence) > 0 or len(soft_evidence) > 0) and not noquery: evidence = model._ensure_internal_evidence(evidence) query_vars = [] query_vars += ut.list_getattr(model.ttype2_cpds['name'], 'variable') #query_vars += ut.list_getattr(model.ttype2_cpds['match'], 'variable') query_vars = ut.setdiff(query_vars, evidence.keys()) #query_vars = ut.setdiff(query_vars, soft_evidence.keys()) query_results = cluster_query(model, query_vars, evidence, soft_evidence, method) else: query_results = {} factor_list = query_results['factor_list'] if verbose: if verbose: print('+--------') semtypes = [model.var2_cpd[f.variables[0]].ttype for f in factor_list] for type_, factors in ut.group_items(factor_list, semtypes).items(): print('Result Factors (%r)' % (type_,)) factors = ut.sortedby(factors, [f.variables[0] for f in factors]) for fs_ in ut.ichunks(factors, 4): ut.colorprint(ut.hz_str([f._str('phi', 'psql') for f in fs_]), 'yellow') print('MAP assignments') top_assignments = query_results.get('top_assignments', []) tmp = [] for lbl, val in top_assignments: tmp.append('%s : %.4f' % (ut.repr2(lbl), val)) print(ut.align('\n'.join(tmp), ' :')) print('L_____\n') showkw = dict(evidence=evidence, soft_evidence=soft_evidence, **query_results) pgm_viz.show_model(model, **showkw) return (model, evidence, query_results)
def triangulatePoint(self, xL, xR, mode=None): """ Args: xL (ndarray): [2 x N] array of N 2D points in left cam pixels xR (ndarray): [2 x N] array of N 2D points in right cam pixels """ if mode is None: mode = self.mode if mode.lower() == 'opencv': if False: hxL = np.zeros((xL.shape[1], 1, 2)) hxR = np.zeros((xL.shape[1], 1, 2)) for i in range(xL.shape[1]): hxL[i, 0, 0] = xL[0, i] hxL[i, 0, 1] = xL[1, i] hxR[i, 0, 0] = xR[0, i] hxR[i, 0, 1] = xR[1, i] else: # Much easier way to do above code hxL = xL.T[:, None, :] hxR = xR.T[:, None, :] KL = self.calData['cameraMatrixL'] KR = self.calData['cameraMatrixR'] kcR = self.calData['distCoeffsR'] kcL = self.calData['distCoeffsL'] R, T = self.calData['R'], self.calData['T'] # THIS SEEMS WRONG PL = np.hstack((np.eye(3), np.zeros((3, 1)))) PR = np.hstack((R, T)) # PL = KL.dot(np.hstack((np.eye(3), np.zeros((3, 1))))) # PR = KR.dot(np.hstack((R, T))) imgptsL = cv2.undistortPoints(hxL, KL, kcL) imgptsR = cv2.undistortPoints(hxR, KR, kcR) import utool as ut print('imgptsL =\n{}'.format(ut.repr2(imgptsL[:, 0, :].T, precision=3))) print('imgptsR =\n{}'.format(ut.repr2(imgptsR[:, 0, :].T, precision=3))) XL = cv2.triangulatePoints(PL, PR, imgptsL, imgptsR) XL /= XL[3] XL = XL[0:3] XR = np.dot(R, XL) + T elif mode == 'matlab': '''% [XL,XR] = stereo_triangulation(xL,xR,om,T,fc_left,cc_left,kc_left,alpha_c_left,fc_right,cc_right,kc_right,alpha_c_right), % % Function that computes the position of a set on N points given the left and right image projections. % The cameras are assumed to be calibrated, intrinsically, and extrinsically. % % Input: % xL: 2xN matrix of pixel coordinates in the left image % xR: 2xN matrix of pixel coordinates in the right image % om,T: rotation vector and translation vector between right and left cameras (output of stereo calibration) % fc_left,cc_left,...: intrinsic parameters of the left camera (output of stereo calibration) % fc_right,cc_right,...: intrinsic parameters of the right camera (output of stereo calibration) % % Output: % % XL: 3xN matrix of coordinates of the points in the left camera reference frame % XR: 3xN matrix of coordinates of the points in the right camera reference frame % % Note: XR and XL are related to each other through the rigid motion equation: XR = R * XL + T, where R = rodrigues(om) % For more information, visit http://www.vision.caltech.edu/bouguetj/calib_doc/htmls/example5.html % % % (c) Jean-Yves Bouguet - Intel Corporation - April 9th, 2003 ''' #--- Normalize the image projection according to the intrinsic parameters of the left and right cameras xt = self.normalizePixel( xL, self.calData['fc_left'], self.calData['cc_left'], self.calData['kc_left'], self.calData['alpha_c_left']) xtt = self.normalizePixel( xR, self.calData['fc_right'], self.calData['cc_right'], self.calData['kc_right'], self.calData['alpha_c_right']) import utool as ut print('xt =\n{}'.format(ut.repr2(xt, precision=3))) print('xtt =\n{}'.format(ut.repr2(xtt, precision=3))) T = self.calData['T'] R = self.calData['R'] #R=np.array([[0.982261529882744, -0.0231885098133093, 0.186076811895112],[0.0236366236373557, 0.999720597968969, -0.00018978828345735],[-0.186020420748467, 0.00458464930006644, 0.98253518209546]]) #--- Extend the normalized projections in homogeneous coordinates g = np.array(np.ones((1, xt.shape[1]))) xt = np.vstack((xt, g)) xtt = np.vstack((xtt, g)) #--- Number of points: N = xt.shape[1] #--- Triangulation of the rays in 3D space: u = np.dot(R, xt) n_xt2 = (xt * xt).sum(axis=0) n_xtt2 = (xtt * xtt).sum(axis=0) T_vect = np.tile(T, (1, N)) DD = n_xt2 * n_xtt2 - (u * xtt).sum(axis=0)**2 dot_uT = (u * T_vect).sum(axis=0) dot_xttT = (xtt * T_vect).sum(axis=0) dot_xttu = (u * xtt).sum(axis=0) NN1 = dot_xttu * dot_xttT - n_xtt2 * dot_uT NN2 = n_xt2 * dot_xttT - dot_uT * dot_xttu Zt = NN1 / DD Ztt = NN2 / DD X1 = xt * np.tile(Zt, (3, 1)) X2 = np.dot(R.T, (xtt * np.tile(Ztt, (3, 1)) - T_vect)) #--- Left coordinates: XL = 0.5 * (X1 + X2) #--- Right coordinates: XR = np.dot(R, XL) + T_vect import utool as ut print('XL =\n{}'.format(ut.repr2(XL, precision=3))) print('XR =\n{}'.format(ut.repr2(XR, precision=3))) print('----') return (XL, XR)
def get_statestr(player): statestr = ut.repr2(player.infodict2(), nl=2, strvals=True) return statestr
def draw_cards(player, num=1): cards = player.deck.draw_card(num) if player.verbose >= 2: print("Player draws %d cards: %s" % (num, ut.repr2(mtgobjs.CardGroup(cards).infohist, nl=1))) player.hand.extend(cards)
def cluster_query(model, query_vars=None, evidence=None, soft_evidence=None, method=None, operation='maximize'): """ CommandLine: python -m ibeis.algo.hots.bayes --exec-cluster_query --show ParamGrid: >>> param_grid = dict( >>> #method=['approx', 'bf', 'bp'], >>> method=['approx', 'bp'], >>> ) >>> combos = ut.all_dict_combinations(param_grid) >>> index = 0 >>> keys = 'method'.split(', ') >>> method, = ut.dict_take(combos[index], keys) Setup: >>> from ibeis.algo.hots.bayes import * # NOQA >>> verbose = True >>> other_evidence = {} >>> name_evidence = [1, None, None, 0] >>> score_evidence = [2, 0, 2] >>> special_names = ['fred', 'sue', 'tom', 'paul'] >>> model = make_name_model( >>> num_annots=4, num_names=4, num_scores=3, verbose=True, mode=1, >>> special_names=special_names) >>> method = None >>> model, evidence, soft_evidence = update_model_evidence( >>> model, name_evidence, score_evidence, other_evidence) >>> evidence = model._ensure_internal_evidence(evidence) >>> query_vars = ut.list_getattr(model.ttype2_cpds['name'], 'variable') Example: >>> # DISABLE_DOCTEST >>> query_results = cluster_query(model, query_vars, evidence, >>> method=method) >>> print(ut.repr2(query_results['top_assignments'], nl=1)) >>> ut.quit_if_noshow() >>> pgm_ext.show_model(model, evidence=evidence, **query_results) >>> ut.show_if_requested() """ evidence = model._ensure_internal_evidence(evidence) if query_vars is None: query_vars = model.nodes() orig_query_vars = query_vars # NOQA query_vars = ut.setdiff(query_vars, list(evidence.keys())) if method is None: method = ut.get_argval('--method', type_=str, default='bp') reduced_joint = compute_reduced_joint(model, query_vars, evidence, method, operation) new_reduced_joint = collapse_factor_labels(model, reduced_joint, evidence) if False: report_partitioning_statistics(new_reduced_joint) # FIXME: are these max marginals? max_marginals = {} for i, var in enumerate(query_vars): one_out = query_vars[:i] + query_vars[i + 1:] max_marginals[var] = new_reduced_joint.marginalize(one_out, inplace=False) # max_marginals[var] = joint2.maximize(one_out, inplace=False) factor_list = max_marginals.values() # Now find the most likely state reduced_variables = new_reduced_joint.variables new_state_idxs = np.array(new_reduced_joint._row_labels(asindex=True)) new_values = new_reduced_joint.values.ravel() sortx = new_values.argsort()[::-1] sort_new_state_idxs = new_state_idxs.take(sortx, axis=0) sort_new_values = new_values.take(sortx) sort_new_states = list(zip(*[ ut.dict_take(model.statename_dict[var], idx) for var, idx in zip(reduced_variables, sort_new_state_idxs.T)])) # Better map assignment based on knowledge of labels map_assign = dict(zip(reduced_variables, sort_new_states[0])) sort_reduced_rowstr_lbls = [ ut.repr2(dict(zip(reduced_variables, lbls)), explicit=True, nobraces=True, strvals=True) for lbls in sort_new_states ] top_assignments = list(zip(sort_reduced_rowstr_lbls[:4], sort_new_values)) if len(sort_new_values) > 3: top_assignments += [('other', 1 - sum(sort_new_values[:4]))] query_results = { 'factor_list': factor_list, 'top_assignments': top_assignments, 'map_assign': map_assign, 'method': method, } print('query_results = %s' % (ut.repr3(query_results, nl=2),)) return query_results
def annotate_matches3(ibs, aid_list, bbox_list, offset_list, name_fm_list, name_fs_list, qreq_=None, **kwargs): """ TODO: use this as the main function. """ # TODO Use this function when you clean show_matches in_image = kwargs.get('in_image', False) #show_query = kwargs.get('show_query', True) draw_border = kwargs.get('draw_border', True) draw_lbl = kwargs.get('draw_lbl', True) notitle = kwargs.get('notitle', False) # List of annotation scores for each annot in the name #printDBG('[viz] annotate_matches3()') #truth = ibs.get_match_truth(aid1, aid2) #name_equality = ( # np.array(ibs.get_annot_nids(aid_list[1:])) == ibs.get_annot_nids(aid_list[0]) #).tolist() #truth = 1 if all(name_equality) else (2 if any(name_equality) else 0) #truth_color = vh.get_truth_color(truth) ## Build title #score = kwargs.pop('score', None) #rawscore = kwargs.pop('rawscore', None) #aid2_raw_rank = kwargs.pop('aid2_raw_rank', None) #print(kwargs) #title = vh.get_query_text(ibs, None, aid2, truth, qaid=aid1, **kwargs) # Build xlbl ax = pt.gca() ph.set_plotdat(ax, 'viztype', 'multi_match') ph.set_plotdat(ax, 'qaid', aid_list[0]) ph.set_plotdat(ax, 'num_matches', len(aid_list) - 1) ph.set_plotdat(ax, 'aid_list', aid_list[1:]) for count, aid in enumerate(aid_list, start=1): ph.set_plotdat(ax, 'aid%d' % (count,), aid) #name_equality = (ibs.get_annot_nids(aid_list[0]) == # np.array(ibs.get_annot_nids(aid_list[1:]))) #truth = 1 if np.all(name_equality) else (2 if np.any(name_equality) else 0) truth = get_multitruth(ibs, aid_list) if any(ibs.is_aid_unknown(aid_list[1:])) or ibs.is_aid_unknown(aid_list[0]): truth = ibs.const.TRUTH_UNKNOWN truth_color = vh.get_truth_color(truth) name_annot_scores = kwargs.get('name_annot_scores', None) if len(aid_list) == 2: # HACK; generalize to multple annots title = vh.get_query_text(ibs, None, aid_list[1], truth, qaid=aid_list[0], **kwargs) if not notitle: pt.set_title(title, ax) if draw_lbl: # Build labels nid_list = ibs.get_annot_nids(aid_list, distinguish_unknowns=False) name_list = ibs.get_annot_names(aid_list) lbls_list = [[] for _ in range(len(aid_list))] if kwargs.get('show_name', False): for count, (lbls, name) in enumerate(zip(lbls_list, name_list)): lbls.append(ut.repr2((name))) if kwargs.get('show_nid', True): for count, (lbls, nid) in enumerate(zip(lbls_list, nid_list)): # only label the first two images with nids LABEL_ALL_NIDS = False if count <= 1 or LABEL_ALL_NIDS: #lbls.append(vh.get_nidstrs(nid)) lbls.append(('q' if count == 0 else '') + vh.get_nidstrs(nid)) if kwargs.get('show_aid', True): for count, (lbls, aid) in enumerate(zip(lbls_list, aid_list)): lbls.append(('q' if count == 0 else '') + vh.get_aidstrs(aid)) if (kwargs.get('show_annot_score', True) and name_annot_scores is not None): max_digits = kwargs.get('score_precision', None) for (lbls, score) in zip(lbls_list[1:], name_annot_scores): lbls.append(ut.num_fmt(score, max_digits=max_digits)) lbl_list = [' : '.join(lbls) for lbls in lbls_list] else: lbl_list = [None] * len(aid_list) #pt.set_title(title, ax) # Plot annotations over images if in_image: in_image_bbox_list = vh.get_bboxes(ibs, aid_list, offset_list) in_image_theta_list = ibs.get_annot_thetas(aid_list) # HACK! #if show_query: # pt.draw_bbox(bbox1, bbox_color=pt.ORANGE, lbl=lbl1, theta=theta1) bbox_color = pt.ORANGE bbox_color = truth_color if draw_border else pt.ORANGE for bbox, theta, lbl in zip(in_image_bbox_list, in_image_theta_list, lbl_list): pt.draw_bbox(bbox, bbox_color=bbox_color, lbl=lbl, theta=theta) pass else: xy, w, h = pt.get_axis_xy_width_height(ax) #theta2 = 0 #if xywh2 is None: # #xywh2 = (xy[0], xy[1], w, h) # # weird when sidebyside is off y seems to be inverted # xywh2 = (0, 0, w, h) #if not show_query and xywh1 is None: # data_config2 = None if qreq_ is None else # qreq_.get_external_data_config2() # kpts2 = ibs.get_annot_kpts([aid2], config2_=data_config2)[0] # #pt.draw_kpts2(kpts2.take(fm.T[1], axis=0)) # # Draw any selected matches # #sm_kw = dict(rect=True, colors=pt.BLUE) # pt.plot_fmatch(None, xywh2, None, kpts2, fm, fs=fs, **kwargs) #if draw_border: # pt.draw_border(ax, truth_color, 4, offset=offset2) if draw_border: pt.draw_border(ax, color=truth_color, lw=4) if draw_lbl: # Custom user lbl for chips 1 and 2 #if show_query: # (x1, y1, w1, h1) = xywh1 # pt.absolute_lbl(x1 + w1, y1, lbl1) for bbox, lbl in zip(bbox_list, lbl_list): (x, y, w, h) = bbox pt.absolute_lbl(x + w, y, lbl) # No matches draw a red box if True: no_matches = name_fm_list is None or all([True if fm is None else len(fm) == 0 for fm in name_fm_list]) if no_matches: xy, w, h = pt.get_axis_xy_width_height(ax) #axes_bbox = (xy[0], xy[1], w, h) if draw_border: pass
def print_status(self): print('is_down = ' + ut.repr2(self.is_down)) print('is_drag = ' + ut.repr2(self.is_drag))
def segmentation_example(): import vigra import opengm import sklearn import sklearn.mixture import numpy as np from vigra import graphs import matplotlib as mpl import plottool as pt pt.ensure_pylab_qt4() # load image and convert to LAB img_fpath = str(ut.grab_test_imgpath(str('lena.png'))) img = vigra.impex.readImage(img_fpath) imgLab = vigra.colors.transform_RGB2Lab(img) superpixelDiameter = 15 # super-pixel size slicWeight = 15.0 # SLIC color - spatial weight labels, nseg = vigra.analysis.slicSuperpixels(imgLab, slicWeight, superpixelDiameter) labels = vigra.analysis.labelImage(labels) - 1 # get 2D grid graph and RAG gridGraph = graphs.gridGraph(img.shape[0:2]) rag = graphs.regionAdjacencyGraph(gridGraph, labels) # Node Features nodeFeatures = rag.accumulateNodeFeatures(imgLab) nodeFeaturesImg = rag.projectNodeFeaturesToGridGraph(nodeFeatures) nodeFeaturesImg = vigra.taggedView(nodeFeaturesImg, "xyc") nodeFeaturesImgRgb = vigra.colors.transform_Lab2RGB(nodeFeaturesImg) nCluster = 5 g = sklearn.mixture.GMM(n_components=nCluster) g.fit(nodeFeatures[:, :]) clusterProb = g.predict_proba(nodeFeatures) # https://github.com/opengm/opengm/blob/master/src/interfaces/python/examples/tutorial/Irregular%20Factor%20Graphs.ipynb # https://github.com/opengm/opengm/blob/master/src/interfaces/python/examples/tutorial/Hard%20and%20Soft%20Constraints.ipynb clusterProbImg = rag.projectNodeFeaturesToGridGraph( clusterProb.astype(np.float32)) clusterProbImg = vigra.taggedView(clusterProbImg, "xyc") ndim_data = clusterProbImg.reshape((-1, nCluster)) pca = sklearn.decomposition.PCA(n_components=3) print(ndim_data.shape) pca.fit(ndim_data) print(ut.repr2(pca.explained_variance_ratio_, precision=2)) oldshape = (clusterProbImg.shape[0:2] + (-1,)) clusterProgImg3 = pca.transform(ndim_data).reshape(oldshape) print(clusterProgImg3.shape) # graphical model with as many variables # as superpixels, each has 3 states gm = opengm.gm(np.ones(rag.nodeNum, dtype=opengm.label_type) * nCluster) # convert probabilites to energies probs = np.clip(clusterProb, 0.00001, 0.99999) costs = -1.0 * np.log(probs) # add ALL unaries AT ONCE fids = gm.addFunctions(costs) gm.addFactors(fids, np.arange(rag.nodeNum)) # add a potts function beta = 40.0 # strength of potts regularizer regularizer = opengm.pottsFunction([nCluster] * 2, 0.0, beta) fid = gm.addFunction(regularizer) # get variable indices of adjacent superpixels # - or "u" and "v" node id's for edges uvIds = rag.uvIds() uvIds = np.sort(uvIds, axis=1) # add all second order factors at once gm.addFactors(fid, uvIds) # get super-pixels with slic on LAB image Inf = opengm.inference.BeliefPropagation parameter = opengm.InfParam(steps=10, damping=0.5, convergenceBound=0.001) inf = Inf(gm, parameter=parameter) class PyCallback(object): def __init__(self,): self.labels = [] def begin(self, inference): print("begin of inference") def end(self, inference): self.labels.append(inference.arg()) def visit(self, inference): gm = inference.gm() labelVector = inference.arg() print("energy %r" % (gm.evaluate(labelVector),)) self.labels.append(labelVector) callback = PyCallback() visitor = inf.pythonVisitor(callback, visitNth=1) inf.infer(visitor) pt.imshow(clusterProgImg3.swapaxes(0, 1)) # plot superpixels cmap = mpl.colors.ListedColormap(np.random.rand(nseg, 3)) pt.imshow(labels.swapaxes(0, 1).squeeze(), cmap=cmap) pt.imshow(nodeFeaturesImgRgb) cmap = mpl.colors.ListedColormap(np.random.rand(nCluster, 3)) for arg in callback.labels: arg = vigra.taggedView(arg, "n") argImg = rag.projectNodeFeaturesToGridGraph(arg.astype(np.uint32)) argImg = vigra.taggedView(argImg, "xy") # plot superpixels pt.imshow(argImg.swapaxes(0, 1).squeeze(), cmap=cmap)
def execstr_dict(dict_, local_name=None, exclude_list=None, explicit=False): """ returns execable python code that declares variables using keys and values execstr_dict Args: dict_ (dict): local_name (str): optional: local name of dictionary. Specifying this is much safer exclude_list (list): Returns: str: execstr --- the executable string that will put keys from dict into local vars CommandLine: python -m utool.util_dbg --test-execstr_dict Example: >>> # UNSTABLE_DOCTEST >>> from utool.util_dbg import * # NOQA >>> my_dictionary = {'a': True, 'b': False} >>> execstr = execstr_dict(my_dictionary) >>> exec(execstr) >>> assert 'a' in vars() and 'b' in vars(), 'execstr failed' >>> assert b is False and a is True, 'execstr failed' >>> result = execstr >>> print(result) a = my_dictionary['a'] b = my_dictionary['b'] Example: >>> # ENABLE_DOCTEST >>> from utool.util_dbg import * # NOQA >>> import utool as ut >>> my_dictionary = {'a': True, 'b': False} >>> execstr = execstr_dict(my_dictionary) >>> locals_ = locals() >>> exec(execstr, locals_) >>> a, b = ut.dict_take(locals_, ['a', 'b']) >>> assert 'a' in locals_ and 'b' in locals_, 'execstr failed' >>> assert b is False and a is True, 'execstr failed' >>> result = execstr >>> print(result) a = my_dictionary['a'] b = my_dictionary['b'] Example: >>> # ENABLE_DOCTEST >>> from utool.util_dbg import * # NOQA >>> import utool as ut >>> my_dictionary = {'a': True, 'b': False} >>> execstr = execstr_dict(my_dictionary, explicit=True) >>> result = execstr >>> print(result) a = True b = False """ import utool as ut if explicit: expr_list = [] for (key, val) in sorted(dict_.items()): assert isinstance(key, six.string_types), 'keys must be strings' expr_list.append('%s = %s' % (key, ut.repr2(val),)) execstr = '\n'.join(expr_list) #print(execstr) return execstr else: if local_name is None: # Magic way of getting the local name of dict_ local_name = get_varname_from_locals(dict_, get_parent_locals()) try: if exclude_list is None: exclude_list = [] assert isinstance(exclude_list, list) exclude_list.append(local_name) expr_list = [] assert isinstance(dict_, dict), 'incorrect type type(dict_)=%r, dict_=%r' % (type(dict), dict_) for (key, val) in sorted(dict_.items()): assert isinstance(key, six.string_types), 'keys must be strings' if not is_valid_varname(key): continue if not any((fnmatch.fnmatch(key, pat) for pat in exclude_list)): expr = '%s = %s[%s]' % (key, local_name, ut.repr2(key)) expr_list.append(expr) execstr = '\n'.join(expr_list) #print(execstr) return execstr except Exception as ex: locals_ = locals() ut.printex(ex, key_list=['locals_']) raise
def write_default_repo_config(): import utool CONFIG_DICT = utool.get_default_repo_config() config_str = utool.repr2(CONFIG_DICT, strvals=True, newlines=True, recursive=True) print(config_str)
def helloworld(time_=0, *args, **kwargs): time.sleep(time_) retval = ('HELLO time_=%r ' % (time_,)) + ut.repr2((args, kwargs)) return retval
def get_toy_data_1vM(num_annots, num_names=None, **kwargs): r""" Args: num_annots (int): num_names (int): (default = None) Kwargs: initial_aids, initial_nids, nid_sequence, seed Returns: tuple: (pair_list, feat_list) CommandLine: python -m ibeis.algo.hots.demobayes --exec-get_toy_data_1vM --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots.demobayes import * # NOQA >>> num_annots = 1000 >>> num_names = 40 >>> get_toy_data_1vM(num_annots, num_names) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ import vtool as vt tup_ = get_toy_annots(num_annots, num_names, **kwargs) aids, nids, aids1, nids1, all_aids, all_nids = tup_ rng = vt.ensure_rng(None) # Test a simple SVM classifier nid2_nexemp = ut.dict_hist(nids1) aid2_nid = dict(zip(aids, nids)) ut.fix_embed_globals() #def add_to_globals(globals_, subdict): # globals_.update(subdict) unique_nids = list(nid2_nexemp.keys()) def annot_to_class_feats2(aid, aid2_nid, top=None): pair_list = [] score_list = [] nexemplar_list = [] for nid in unique_nids: label = (aid2_nid[aid] == nid) num_exemplars = nid2_nexemp.get(nid, 0) if num_exemplars == 0: continue params = toy_params[label] mu, sigma = ut.dict_take(params, ['mu', 'sigma']) score_ = rng.normal(mu, sigma, size=num_exemplars).max() score = np.clip(score_, 0, np.inf) pair_list.append((aid, nid)) score_list.append(score) nexemplar_list.append(num_exemplars) rank_list = ut.argsort(score_list, reverse=True) feat_list = np.array([score_list, rank_list, nexemplar_list]).T sortx = np.argsort(rank_list) feat_list = feat_list.take(sortx, axis=0) pair_list = np.array(pair_list).take(sortx, axis=0) if top is not None: feat_list = feat_list[:top] pair_list = pair_list[0:top] return pair_list, feat_list toclass_features = [annot_to_class_feats2(aid, aid2_nid, top=5) for aid in aids] aidnid_pairs = np.vstack(ut.get_list_column(toclass_features, 0)) feat_list = np.vstack(ut.get_list_column(toclass_features, 1)) score_list = feat_list.T[0:1].T lbl_list = [aid2_nid[aid] == nid for aid, nid in aidnid_pairs] from sklearn import svm #clf1 = svm.LinearSVC() print('Learning classifiers') clf3 = svm.SVC() clf3.fit(feat_list, lbl_list) clf1 = svm.LinearSVC() clf1.fit(score_list, lbl_list) # Score new annots against the training database tup_ = get_toy_annots(num_annots * 2, num_names, initial_aids=all_aids, initial_nids=all_nids) aids, nids, aids1, nids1, all_aids, all_nids = tup_ aid2_nid = dict(zip(aids, nids)) toclass_features = [annot_to_class_feats2(aid, aid2_nid) for aid in aids] aidnid_pairs = np.vstack(ut.get_list_column(toclass_features, 0)) feat_list = np.vstack(ut.get_list_column(toclass_features, 1)) lbl_list = np.array([aid2_nid[aid] == nid for aid, nid in aidnid_pairs]) print('Running tests') score_list = feat_list.T[0:1].T tp_feat_list = feat_list[lbl_list] tn_feat_list = feat_list[~lbl_list] tp_lbls = lbl_list[lbl_list] tn_lbls = lbl_list[~lbl_list] print('num tp: %d' % len(tp_lbls)) print('num fp: %d' % len(tn_lbls)) tp_score_list = score_list[lbl_list] tn_score_list = score_list[~lbl_list] print('tp_feat' + ut.repr3(ut.get_stats(tp_feat_list, axis=0), precision=2)) print('tp_feat' + ut.repr3(ut.get_stats(tn_feat_list, axis=0), precision=2)) print('tp_score' + ut.repr2(ut.get_stats(tp_score_list), precision=2)) print('tp_score' + ut.repr2(ut.get_stats(tn_score_list), precision=2)) tp_pred3 = clf3.predict(tp_feat_list) tn_pred3 = clf3.predict(tn_feat_list) print((tp_pred3.sum(), tp_pred3.shape)) print((tn_pred3.sum(), tn_pred3.shape)) tp_score3 = clf3.score(tp_feat_list, tp_lbls) tn_score3 = clf3.score(tn_feat_list, tn_lbls) tp_pred1 = clf1.predict(tp_score_list) tn_pred1 = clf1.predict(tn_score_list) print((tp_pred1.sum(), tp_pred1.shape)) print((tn_pred1.sum(), tn_pred1.shape)) tp_score1 = clf1.score(tp_score_list, tp_lbls) tn_score1 = clf1.score(tn_score_list, tn_lbls) print('tp score with rank = %r' % (tp_score3,)) print('tn score with rank = %r' % (tn_score3,)) print('tp score without rank = %r' % (tp_score1,)) print('tn score without rank = %r' % (tn_score1,)) toy_data = {} return toy_data