def update_registry(drive): print('Updating registered files in %r' % (drive,)) # Update existing files fpath_exists_list = list(map(exists, ut.ProgIter(drive.fpath_list, 'checkexist fpath', freq=1000))) dpath_exists_list = list(map(exists, ut.ProgIter(drive.dpath_list, 'checkexist dpath', freq=1000))) if all(fpath_exists_list): print('No change in file structure') else: print('%d/%d files no longer exist' % ( len(drive.fpath_list) - sum(fpath_exists_list), len(drive.fpath_list))) removed_fpaths = ut.compress(drive.fpath_list, ut.not_list(fpath_exists_list)) print('removed_fpaths = %s' % (ut.list_str(removed_fpaths),)) if all(dpath_exists_list): print('No change in dpath structure') else: print('%d/%d dirs no longer exist' % ( len(drive.dpath_list) - sum(dpath_exists_list), len(drive.dpath_list))) removed_dpaths = ut.compress( drive.dpath_list, ut.not_list(dpath_exists_list)) print('removed_dpaths = %s' % (ut.list_str(removed_dpaths),)) drive.fpath_list = ut.compress(drive.fpath_list, fpath_exists_list) drive.dpath_list = ut.compress(drive.dpath_list, dpath_exists_list) drive.cache.save('fpath_list', drive.fpath_list) drive.cache.save('dpath_list', drive.dpath_list)
def test_sver_wrapper2(): r""" CommandLine: python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 --no-c --quiet python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 --rebuild-sver Example: >>> # ENABLE_DOCTEST >>> from vtool.sver_c_wrapper import * # NOQA >>> result = test_sver_wrapper2() >>> print(result) Ignore: C (Serial): unique cases affine inliers: [ '[ 4 25 33 36 37 53]', ] unique cases homog inliers: [ '[]', ] C (Parallel) unique cases affine inliers: [ '[ 4 25 33 36 37 53]', '[10 19 25 29 36 39 53]', ] unique cases homog inliers: [ '[10 43 53]', '[]', ] Python: unique cases affine inliers: [ '[10 19 25 29 36 39 53]', ] unique cases homog inliers: [ '[10 43 53]', ] """ import vtool import vtool.tests.testdata_nondeterm_sver kpts1, kpts2, fm, xy_thresh, scale_thresh, ori_thresh, dlen_sqrd2, min_nInliers, match_weights, full_homog_checks = vtool.tests.testdata_nondeterm_sver.testdata_nondeterm_sver() inliers_list = [] homog_inliers_list = [] for x in range(10): sv_tup = vtool.spatially_verify_kpts( kpts1, kpts2, fm, xy_thresh, scale_thresh, ori_thresh, dlen_sqrd2, min_nInliers, match_weights=match_weights, full_homog_checks=full_homog_checks, returnAff=True) aff_inliers = sv_tup[3] inliers_list.append(str(aff_inliers)) homog_inliers_list.append(str(sv_tup[0])) #print(sv_tup[0]) #print(sv_tup[3]) print('unique cases affine inliers: ' + ut.list_str(list(set(inliers_list)))) print('unique cases homog inliers: ' + ut.list_str(list(set(homog_inliers_list))))
def print_database_structure(cur): import utool as ut tablename_list = ut.get_tablenames(cur) colinfos_list = [ut.get_table_columninfo_list(cur, tablename) for tablename in tablename_list] numrows_list = [ut.get_table_num_rows(cur, tablename) for tablename in tablename_list] for tablename, colinfo_list, num_rows in ut.sortedby(list(zip(tablename_list, colinfos_list, numrows_list)), numrows_list): print('+-------------') print('tablename = %r' % (tablename,)) print('num_rows = %r' % (num_rows,)) #print(ut.list_str(colinfo_list)) print(ut.list_str(ut.get_primary_columninfo(cur, tablename))) print(ut.list_str(ut.get_nonprimary_columninfo(cur, tablename))) print('+-------------')
def show_function_usage(fname, funcname_list, dpath_list): # Check to see for function usage funcname_list = [r'\b%s\b' % (funcname.strip(),) for funcname in funcname_list if len(funcname) > 0] flagged_funcnames = [] for funcname in funcname_list: found_filestr_list, found_lines_list, found_lxs_list = ut.grep([funcname], dpath_list=dpath_list) total = 0 for lines in found_lines_list: total += len(lines) funcname_ = funcname.replace('\\b', '') print(funcname_ + ' ' + str(total)) if total == 1: flagged_funcnames.append(funcname_) # See where external usage is isexternal_list = [fname == fname_ for fname_ in found_filestr_list] external_filestr_list = ut.compress(found_filestr_list, isexternal_list) external_lines_list = ut.compress(found_lines_list, isexternal_list) #external_lxs_list = ut.compress(found_lxs_list, isexternal_list) if len(external_filestr_list) == 0: print(' no external usage') else: for filename, lines in zip(external_filestr_list, external_lines_list): print(' * filename=%r' % (filename,)) print(ut.list_str(lines)) #print(ut.list_str(list(zip(external_filestr_list, external_lines_list)))) print('----------') print('flagged:') print('\n'.join(flagged_funcnames))
def parse_latex_comments_for_commmands(): r""" CommandLine: python -m ibeis.scripts.gen_cand_expts --exec-parse_latex_comments_for_commmands Example: >>> # SCRIPT >>> from ibeis.scripts.gen_cand_expts import * # NOQA >>> parse_latex_comments_for_commmands() """ fname = ut.get_argval('--fname', type_=str, default='figdefexpt.tex') text = ut.read_from(ut.truepath('~/latex/crall-candidacy-2015/' + fname)) #text = ut.read_from(ut.truepath('~/latex/crall-candidacy-2015/figdefindiv.tex')) lines = text.split('\n') cmd_list = [''] in_comment = True for line in lines: if line.startswith('% ---'): # Keep separators toadd = line.replace('%', '#') if not (len(cmd_list) > 1 and cmd_list[-1].startswith('# ---')): cmd_list[-1] += (toadd) else: cmd_list.append(toadd) cmd_list.append('') if line.strip().startswith(r'\begin{comment}'): in_comment = True continue if in_comment: line = line.strip() if line == '' or line.startswith('#') or line.startswith('%'): in_comment = False else: cmd_list[-1] = cmd_list[-1] + line if not line.strip().endswith('\\'): cmd_list[-1] = cmd_list[-1] + ' $@' #cmd_list.append('') #cmd_list.append('#--') cmd_list.append('') in_comment = False else: cmd_list[-1] = cmd_list[-1] + '\n' cmd_list = [cmd.replace('--render', '').replace('--diskshow', '') for cmd in cmd_list] # formatting cmd_list2 = [] for cmd in cmd_list: #cmd = cmd.replace(' -t ', ' \\\n -t ') #cmd = cmd.replace('--db', '\\\n --db') #cmd = cmd.replace('python -m ibeis.dev', './dev.py') cmd = cmd.replace('python -m ibeis.dev -e', 'ibeis -e') cmd_list2.append(cmd) cmd_list = cmd_list2 print('cmd_list = %s' % (ut.list_str(cmd_list),)) from os.path import splitext script_fname = 'regen_' + splitext(fname)[0] + '.sh' fname, script, line_list = write_script_lines(cmd_list, script_fname)
def biggest_files(drive): print('Biggest Files in %r' % (drive,)) sortx = ut.list_argsort(drive.fpath_bytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(drive.fpath_bytes_list, sel) biggest_files = ut.take(drive.fpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_files)) print(ut.list_str(biginfo_list, strvals=True))
def get_csv_results(gridsearch, max_lines=None, score_lbl='score_diff'): """ Make csv text describing results Args: max_lines (int): add top num lines to the csv. No limit if None. score_lbl (str): score label to sort by Returns: str: result data in csv format CommandLine: python -m utool.util_gridsearch --test-get_csv_results Example: >>> # DISABLE_DOCTEST >>> from utool.util_gridsearch import * # NOQA >>> import utool as ut >>> import plottool as pt >>> # build test data >>> score_lbl = 'score_diff' >>> gridsearch = testdata_grid_search() >>> csvtext = gridsearch.get_csv_results(10, score_lbl) >>> print(csvtext) >>> result = ut.hashstr(csvtext) >>> print(result) 60yptleiwo@lk@24 """ import utool as ut collbl_tup = gridsearch.get_sorted_columns_and_labels(score_lbl) (score_name_sorted, param_name_sorted, score_list_sorted, param_vals_sorted) = collbl_tup # Build CSV column_lbls = score_name_sorted + param_name_sorted column_list = score_list_sorted + param_vals_sorted if max_lines is not None: column_list = [ut.listclip(col, max_lines) for col in column_list] header_raw_fmtstr = ut.codeblock( ''' import utool as ut from utool import DimensionBasis title = 'Grid Search Results CSV' label = {label} grid_basis = {grid_basis_str} ''') fmtdict = dict( grid_basis_str=ut.list_str(gridsearch.grid_basis), label=gridsearch.label ) header_raw = header_raw_fmtstr.format(**fmtdict) header = ut.indent(header_raw, '# >>> ') precision = 3 csvtext = ut.make_csv_table(column_list, column_lbls, header, precision=precision) return csvtext
def read_csv(csv_fpath): import utool as ut csv_text = ut.read_from(csv_fpath) csv_lines = csv_text.split('\n') print(ut.list_str(csv_lines[0:2])) csv_data = [[field.strip('"').strip('\r') for field in line.split(',')] for line in csv_lines if len(line) > 0] csv_header = csv_data[0] csv_data = csv_data[1:] return csv_data, csv_header
def jagged_stats_info(arr_, lbl, col_lbls): arr = ut.recursive_replace(arr_, np.inf, np.nan) # Treat infinite as nan stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True) sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls) sel_col_lbls = ut.take(col_lbls, sel_indices) statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True) stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw) sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw) sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str return stat_str, sel_stat_str
def autogen_argparse_block(extra_args=[]): """ SHOULD TURN ANY REGISTERED ARGS INTO A A NEW PARSING CONFIG FILE FOR BETTER --help COMMANDS import utool as ut __REGISTERED_ARGS__ = ut.util_arg.__REGISTERED_ARGS__ Args: extra_args (list): (default = []) CommandLine: python -m utool.util_arg --test-autogen_argparse_block Example: >>> # DISABLE_DOCTEST >>> import utool as ut >>> extra_args = [] >>> result = ut.autogen_argparse_block(extra_args) >>> print(result) """ #import utool as ut # NOQA #__REGISTERED_ARGS__ # TODO FINISHME grouped_args = [] # Group similar a args for argtup in __REGISTERED_ARGS__: argstr_list, type_, default, help_ = argtup argstr_set = set(argstr_list) # <MULTIKEY_SETATTR> # hack in multikey setattr n**2 yuck found = False for index, (keyset, vals) in enumerate(grouped_args): if len(keyset.intersection(argstr_set)) > 0: # update keyset.update(argstr_set) vals.append(argtup) found = True break if not found: new_keyset = argstr_set new_vals = [argtup] grouped_args.append((new_keyset, new_vals)) # </MULTIKEY_SETATTR> # DEBUG multi_groups = [] for keyset, vals in grouped_args: if len(vals) > 1: multi_groups.append(vals) if len(multi_groups) > 0: import utool as ut print('Following arg was specified multiple times') print(ut.list_str(multi_groups, newlines=2))
def on_contextMenuRequested(widget, index, pos): print('context request') if widget.api is not None: print(ut.list_str(widget.api.get_available_colnames())) # HACK test #widget.api.add_column_names(['qx2_gt_rank', 'qx2_gf_rank', 'qx2_gt_raw_score', 'qx2_gf_raw_score']) widget.refresh_headers() #widget.change_headers(widget.api.make_headers()) if VERBOSE_ITEM_WIDGET: print('context request') pass
def jagged_stats_info(arr_, lbl, col_lbls): arr = ut.recursive_replace(arr_, np.inf, np.nan) # Treat infinite as nan stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True) sel_stat_dict, sel_indices = ut.find_interesting_stats( stat_dict, col_lbls) sel_col_lbls = ut.take(col_lbls, sel_indices) statstr_kw = dict(precision=3, newlines=True, lbl=lbl, align=True) stat_str = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw) sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw) sel_stat_str = 'sel_col_lbls = %s' % ( ut.list_str(sel_col_lbls), ) + '\n' + sel_stat_str return stat_str, sel_stat_str
def biggest_dirs(drive): print('Biggest Dirs in %r' % (drive,)) dpath_list = drive.dpath_list fidxs_list = ut.dict_take(drive.dpath_to_fidx, dpath_list) unflat_dpath_bytes_list = ut.list_unflat_take(drive.fpath_bytes_list, fidxs_list) dpath_nbytes_list = list(map(sum, unflat_dpath_bytes_list)) sortx = ut.list_argsort(dpath_nbytes_list)[::-1] sel = sortx[0:10] biggest_nbytes = ut.take(dpath_nbytes_list, sel) biggest_dpaths = ut.take(dpath_list, sel) biginfo_list = list(zip(map(ut.byte_str2, biggest_nbytes), biggest_dpaths)) print(ut.list_str(biginfo_list, strvals=True)) pass
def drive_test_script(ibs): r""" Test script where we drive around and take pictures of animals both in a given database and not in a given databse to make sure the system works. CommandLine: python -m ibeis.viz.viz_image --test-drive_test_script python -m ibeis.viz.viz_image --test-drive_test_script --db PZ_MTEST --show python -m ibeis.viz.viz_image --test-drive_test_script --db GIR_Tanya --show python -m ibeis.viz.viz_image --test-drive_test_script --db GIR_Master0 --show python -m ibeis.viz.viz_image --test-drive_test_script --db PZ_Master0 --show python -m ibeis.viz.viz_image --test-drive_test_script --db PZ_FlankHack --show python -m ibeis.viz.viz_image --test-drive_test_script --db PZ_FlankHack --show python -m ibeis.viz.viz_image --test-drive_test_script --dbdir /raid/work2/Turk/GIR_Master --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.viz.viz_image import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb() >>> drive_test_script(ibs) """ import ibeis aid_list = ibeis.testdata_aids(a='default:pername=1') print('Running with (annot) aid_list = %r' % (aid_list)) gid_list = ibs.get_annot_gids(aid_list) print('Running with (image) gid_list = %r' % (gid_list)) avuuid_list = ibs.get_annot_visual_uuids(aid_list) guuid_list = ibs.get_image_uuids(gid_list) print('Running with annot_visual_uuid_list = %s' % (ut.list_str(zip(aid_list, avuuid_list)))) print('Running with image_uuid_list = %s' % (ut.list_str(zip(gid_list, guuid_list)))) for gid, aid in ut.ProgressIter(zip(gid_list, aid_list), lbl='progress '): print('\ngid, aid, nid = %r, %r, %r' % (gid, aid, ibs.get_annot_nids(aid),)) show_image(ibs, gid, annote=False, rich_title=True) pt.show_if_requested()
def filter_duplicate_acfgs(expanded_aids_list, acfg_list, acfg_name_list, verbose=ut.NOT_QUIET): """ Removes configs with the same expanded aids list CommandLine: # The following will trigger this function: ibeis -e print_acfg -a timectrl timectrl:view=left --db PZ_MTEST """ from ibeis.expt import annotation_configs acfg_list_ = [] expanded_aids_list_ = [] seen_ = ut.ddict(list) for acfg, (qaids, daids) in zip(acfg_list, expanded_aids_list): key = (ut.hashstr_arr27(qaids, 'qaids'), ut.hashstr_arr27(daids, 'daids')) if key in seen_: seen_[key].append(acfg) continue else: seen_[key].append(acfg) expanded_aids_list_.append((qaids, daids)) acfg_list_.append(acfg) if verbose: duplicate_configs = dict([(key_, val_) for key_, val_ in seen_.items() if len(val_) > 1]) if len(duplicate_configs) > 0: print( 'The following configs produced duplicate annnotation configs') for key, val in duplicate_configs.items(): # Print the semantic difference between the duplicate configs _tup = annotation_configs.compress_acfg_list_for_printing(val) nonvaried_compressed_dict, varied_compressed_dict_list = _tup print('+--') print('key = %r' % (key, )) print('duplicate_varied_cfgs = %s' % (ut.list_str(varied_compressed_dict_list), )) print('duplicate_nonvaried_cfgs = %s' % (ut.dict_str(nonvaried_compressed_dict), )) print('L__') print('[harn.help] parsed %d / %d unique annot configs from: %r' % (len(acfg_list_), len(acfg_list), acfg_name_list)) return expanded_aids_list_, acfg_list_
def test_pygist(): print('[pygist] Testing pygist') # Ensure you have test data print('[pygist] Ensuring testdata') datafile = utool.grab_file_url(TEST_MODEL_URL, appname='utool') test_image_dir = utool.grab_zipped_url(TEST_IMAGES_URL, appname='utool') imgpaths = utool.list_images(test_image_dir, fullpath=True) # test image paths outdir = utool.get_app_resource_dir('pygist') # where to put results # Run pygist on test images print('[pygist] Running tests') test_results = pygist.test(imgpaths, outdir=outdir, datafile=datafile) # Print results target_results = [-1, -1, 1, -1, 1, -1, -1, -1, 1, 1, -1, 1, 1] assert target_results == target_results, 'results do not match' print('test_results = %r' % (test_results,)) print(utool.list_str(list(izip(imgpaths, test_results)))) return locals()
def test_pygist(): print('[pygist] Testing pygist') # Ensure you have test data print('[pygist] Ensuring testdata') datafile = utool.grab_file_url(TEST_MODEL_URL, appname='utool') test_image_dir = utool.grab_zipped_url(TEST_IMAGES_URL, appname='utool') imgpaths = utool.list_images(test_image_dir, fullpath=True) # test image paths outdir = utool.get_app_resource_dir('pygist') # where to put results # Run pygist on test images print('[pygist] Running tests') test_results = pygist.test(imgpaths, outdir=outdir, datafile=datafile) # Print results target_results = [-1, -1, 1, -1, 1, -1, -1, -1, 1, 1, -1, 1, 1] assert target_results == target_results, 'results do not match' print('test_results = %r' % (test_results, )) print(utool.list_str(list(izip(imgpaths, test_results)))) return locals()
def draw_precision_recall_curve_(recall_range_, p_interp_curve, title_pref=None, fnum=1): import plottool as pt if recall_range_ is None: recall_range_ = np.array([]) p_interp_curve = np.array([]) fig = pt.figure(fnum=fnum, docla=True, doclf=True) # NOQA if recall_range_ is None: ave_p = np.nan else: ave_p = p_interp_curve.sum() / p_interp_curve.size pt.plot2(recall_range_, p_interp_curve, marker='o--', x_label='recall', y_label='precision', unitbox=True, flipx=False, color='r', title='Interplated Precision Vs Recall\n' + 'avep = %r' % ave_p) print('Interplated Precision') print(ut.list_str(list(zip(recall_range_, p_interp_curve))))
def inspect_deck(deck): def get_card_tags(card, deck): tags = [] stats = card.mana_source_stats(deck) if stats is not None: tags.append("land") if len(stats[1]) > 0: tags.append("tapland") else: tags.append("untapland") return tags # ------------ print("len(deck) = %r" % (len(deck),)) tags_list = [get_card_tags(card, deck) for card in deck.card_list] print("Deck Counts:") print(ut.repr2(ut.dict_hist(ut.flatten(tags_list)), nl=True)) hand = deck.sample_hand() manastats_list = [card.mana_source_stats(deck) for card in hand] print(ut.list_str([card.name + ": " + text_type(stats) for card, stats in zip(hand, manastats_list)])) tags_list = [get_card_tags(card, deck) for card in hand] print("Hand Counts") print(ut.repr2(ut.dict_hist(ut.flatten(tags_list)), nl=True)) valid_tags = ["land", "tapland", "untapland"] x = {tag: [] for tag in valid_tags} for _ in range(500): hand = deck.sample_hand() tags_list = [get_card_tags(card, deck) for card in hand] taghist = ut.dict_hist(ut.flatten(tags_list)) for key, val in x.items(): val.append(taghist.get(key, 0)) print("Monte Stats:") for key, val in list(x.items()): print("%15s: %s" % (key, ut.repr2(ut.get_stats(val), precision=2))) def hand_stats(): # [card.types for card in hand] # [card.rrr() for card in hand] [card.mana_source_stats(deck) for card in hand] card.types
def filter_duplicate_acfgs(expanded_aids_list, acfg_list, acfg_name_list, verbose=ut.NOT_QUIET): """ Removes configs with the same expanded aids list CommandLine: # The following will trigger this function: ibeis -e print_acfg -a timectrl timectrl:view=left --db PZ_MTEST """ from ibeis.expt import annotation_configs acfg_list_ = [] expanded_aids_list_ = [] seen_ = ut.ddict(list) for acfg, (qaids, daids) in zip(acfg_list, expanded_aids_list): key = (ut.hashstr_arr27(qaids, 'qaids'), ut.hashstr_arr27(daids, 'daids')) if key in seen_: seen_[key].append(acfg) continue else: seen_[key].append(acfg) expanded_aids_list_.append((qaids, daids)) acfg_list_.append(acfg) if verbose: duplicate_configs = dict( [(key_, val_) for key_, val_ in seen_.items() if len(val_) > 1]) if len(duplicate_configs) > 0: print('The following configs produced duplicate annnotation configs') for key, val in duplicate_configs.items(): # Print the semantic difference between the duplicate configs _tup = annotation_configs.compress_acfg_list_for_printing(val) nonvaried_compressed_dict, varied_compressed_dict_list = _tup print('+--') print('key = %r' % (key,)) print('duplicate_varied_cfgs = %s' % ( ut.list_str(varied_compressed_dict_list),)) print('duplicate_nonvaried_cfgs = %s' % ( ut.dict_str(nonvaried_compressed_dict),)) print('L__') print('[harn.help] parsed %d / %d unique annot configs from: %r' % ( len(acfg_list_), len(acfg_list), acfg_name_list)) return expanded_aids_list_, acfg_list_
def makeinit(module_path, exclude_modnames=[]): #module_name = basename(module_path) module_name = ut.get_modname_from_modpath(module_path) IMPORT_TUPLES = util_importer.make_import_tuples(module_path, exclude_modnames=exclude_modnames) initstr = util_importer.make_initstr(module_name, IMPORT_TUPLES) regen_command = 'cd %s\n' % (module_path) regen_command += ' makeinit.py' if len(exclude_modnames ) > 0: regen_command += ' -x ' + ' '.join(exclude_modnames) regen_block = (ut.codeblock(''' """ Regen Command: {regen_command} """ ''').format(regen_command=regen_command)) print('### __init__.py ###') print(initstr) print('\nIMPORT_TUPLES = ' + ut.list_str(IMPORT_TUPLES)) print(regen_block)
def precfg_dbs(db_list): r""" Helper to precompute information Runs precfg on multiple databases Args: db_list (list): CommandLine: python -m ibeis.expt.precomputer --exec-precfg_dbs python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist testdb1 PZ_MTEST python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist testdb1 PZ_MTEST --preload -t custom python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=PZ_MTEST,NNP_MasterGIRM_core,PZ_Master0,NNP_Master3,GZ_ALL,PZ_FlankHack --preload --delete-nn-cache #python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=PZ_Master0 -t candidacy1 --preload-chip --controlled --species=primary python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy --preload python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy -t candidacy --preload-chip --species=primary --controlled python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy -t candidacy --preload-chip --species=primary --allgt python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy -t candidacy --preload-feat python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy -t candidacy --preload-featweight python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy -t candidacy --preload python -m ibeis.expt.precomputer --exec-precfg_dbs --dblist=candidacy --delete-nn-cache Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.precomputer import * # NOQA >>> db_list = ut.get_argval('--dblist', type_=list, default=['testdb1']) >>> result = precfg_dbs(db_list) >>> print(result) """ import ibeis.init.main_helpers import ibeis if db_list == ['candidacy']: from ibeis.expt import experiment_configs db_list = experiment_configs.get_candidacy_dbnames() # HACK print('db_list = %s' % (ut.list_str(db_list),)) test_cfg_name_list = ut.get_argval('-t', type_=list, default=[]) for db in db_list: ibs = ibeis.opendb(db=db) ibs, qaids, daids = ibeis.init.main_helpers.testdata_expanded_aids(verbose=False, ibs=ibs) precfg(ibs, qaids, daids, test_cfg_name_list)
def get_text_between_lines(lnum1, lnum2, col1=0, col2=sys.maxint - 1): import vim lines = vim.eval('getline({}, {})'.format(lnum1, lnum2)) import utool as ut lines = ut.ensure_unicode_strlist(lines) try: if len(lines) == 0: pass elif len(lines) == 1: lines[0] = lines[0][col1:col2 + 1] else: lines[0] = lines[0][col1:] lines[-1] = lines[-1][:col2 + 1] text = '\n'.join(lines) except Exception: import utool as ut ut.ENABLE_COLORS = False ut.util_str.ENABLE_COLORS = False ut.util_dbg.COLORED_EXCEPTIONS = False print(ut.list_str(lines)) raise return text
def print_system_users(): r""" prints users on the system On unix looks for /bin/bash users in /etc/passwd CommandLine: python -m utool.util_cplat --test-print_system_users Example: >>> # SCRIPT >>> from utool.util_cplat import * # NOQA >>> result = print_system_users() >>> print(result) """ import utool as ut text = ut.read_from('/etc/passwd') userinfo_text_list = text.splitlines() userinfo_list = [uitext.split(':') for uitext in userinfo_text_list] #print(ut.list_str(sorted(userinfo_list))) bash_users = [tup for tup in userinfo_list if tup[-1] == '/bin/bash'] print(ut.list_str(sorted(bash_users)))
def make_individual_latex_figures(ibs, fpaths_list, flat_case_labels, cfgx2_shortlbl, case_figdir, analysis_fpath_list): # HACK MAKE LATEX CONVINENCE STUFF #print('LATEX HACK') if len(fpaths_list) == 0: print('nothing to render') return RENDER = ut.get_argflag('--render') DUMP_FIGDEF = ut.get_argflag(('--figdump', '--dump-figdef', '--figdef')) if not (DUMP_FIGDEF or RENDER): # HACK return latex_code_blocks = [] latex_block_keys = [] caption_prefix = ut.get_argval('--cappref', type_=str, default='') caption_suffix = ut.get_argval('--capsuf', type_=str, default='') cmdaug = ut.get_argval('--cmdaug', type_=str, default='custom') selected = None for case_idx, (fpaths, labels) in enumerate(zip(fpaths_list, flat_case_labels)): if labels is None: labels = [cmdaug] if len(fpaths) < 4: nCols = len(fpaths) else: nCols = 2 _cmdname = ibs.get_dbname() + ' Case ' + ' '.join(labels) + '_' + str(case_idx) #print('_cmdname = %r' % (_cmdname,)) cmdname = ut.latex_sanitize_command_name(_cmdname) label_str = cmdname if len(caption_prefix) == 0: caption_str = ut.escape_latex('Casetags: ' + ut.list_str(labels, nl=False, strvals=True) + ', db=' + ibs.get_dbname() + '. ') else: caption_str = '' use_sublbls = len(cfgx2_shortlbl) > 1 if use_sublbls: caption_str += ut.escape_latex('Each figure shows a different configuration: ') sublbls = ['(' + chr(97 + count) + ') ' for count in range(len(cfgx2_shortlbl))] else: #caption_str += ut.escape_latex('This figure depicts correct and #incorrect matches from configuration: ') sublbls = [''] * len(cfgx2_shortlbl) def wrap_tt(text): return r'{\tt ' + text + '}' _shortlbls = cfgx2_shortlbl _shortlbls = list(map(ut.escape_latex, _shortlbls)) # Adjust spacing for breaks #tex_small_space = r'' tex_small_space = r'\hspace{0pt}' # Remove query specific config flags in individual results _shortlbls = [re.sub('\\bq[^,]*,?', '', shortlbl) for shortlbl in _shortlbls] # Let config strings be broken over newlines _shortlbls = [re.sub('\\+', tex_small_space + '+' + tex_small_space, shortlbl) for shortlbl in _shortlbls] _shortlbls = [re.sub(', *', ',' + tex_small_space, shortlbl) for shortlbl in _shortlbls] _shortlbls = list(map(wrap_tt, _shortlbls)) cfgx2_texshortlbl = ['\n ' + lbl + shortlbl for lbl, shortlbl in zip(sublbls, _shortlbls)] caption_str += ut.conj_phrase(cfgx2_texshortlbl, 'and') + '.\n ' caption_str = '\n ' + caption_prefix + caption_str + caption_suffix caption_str = caption_str.rstrip() figure_str = ut.get_latex_figure_str(fpaths, nCols=nCols, label_str=label_str, caption_str=caption_str, use_sublbls=None, use_frame=True) latex_block = ut.latex_newcommand(cmdname, figure_str) latex_block = '\n%----------\n' + latex_block latex_code_blocks.append(latex_block) latex_block_keys.append(cmdname) # HACK remove_fpath = ut.truepath('~/latex/crall-candidacy-2015') + '/' latex_fpath = join(case_figdir, 'latex_cases.tex') if selected is not None: selected_keys = selected else: selected_keys = latex_block_keys selected_blocks = ut.dict_take(dict(zip(latex_block_keys, latex_code_blocks)), selected_keys) figdef_block = '\n'.join(selected_blocks) figcmd_block = '\n'.join(['\\' + key for key in latex_block_keys]) selected_block = figdef_block + '\n\n' + figcmd_block # HACK: need full paths to render selected_block_renderable = selected_block selected_block = selected_block.replace(remove_fpath, '') if RENDER: ut.render_latex_text(selected_block_renderable) if DUMP_FIGDEF: ut.writeto(latex_fpath, selected_block) #if NOT DUMP AND NOT RENDER: # print('STANDARD LATEX RESULTS') # cmdname = ibs.get_dbname() + 'Results' # latex_block = ut.get_latex_figure_str2(analysis_fpath_list, cmdname, nCols=1) # ut.print_code(latex_block, 'latex') if DUMP_FIGDEF or RENDER: ut.print_code(selected_block, 'latex')
def makeinit(mod_dpath, exclude_modnames=[], use_star=False): r""" Args: mod_dpath (str): exclude_modnames (list): (default = []) use_star (bool): (default = False) Returns: str: init_codeblock CommandLine: python -m utool.util_autogen makeinit --modname=ibeis.algo Example: >>> # SCRIPT >>> from utool.util_autogen import * # NOQA >>> import utool as ut >>> modname = ut.get_argval('--modname', str, default=None) >>> mod_dpath = (os.getcwd() if modname is None else >>> ut.get_modpath(modname, prefer_pkg=True)) >>> mod_dpath = ut.unixpath(mod_dpath) >>> mod_fpath = join(mod_dpath, '__init__.py') >>> exclude_modnames = ut.get_argval(('--exclude', '-x'), list, default=[]) >>> use_star = ut.get_argflag('--star') >>> init_codeblock = makeinit(mod_dpath, exclude_modnames, use_star) >>> ut.dump_autogen_code(mod_fpath, init_codeblock) """ from utool._internal import util_importer import utool as ut module_name = ut.get_modname_from_modpath(mod_dpath) IMPORT_TUPLES = util_importer.make_import_tuples(mod_dpath, exclude_modnames=exclude_modnames) initstr = util_importer.make_initstr(module_name, IMPORT_TUPLES) regen_command = 'cd %s\n' % (mod_dpath) regen_command += ' makeinit.py' regen_command += ' --modname={modname}'.format(modname=module_name) if use_star: regen_command += ' --star' if len(exclude_modnames ) > 0: regen_command += ' -x ' + ' '.join(exclude_modnames) regen_block = (ut.codeblock(''' """ Regen Command: {regen_command} """ ''').format(regen_command=regen_command)) importstar_codeblock = ut.codeblock( ''' """ python -c "import {module_name}" --dump-{module_name}-init python -c "import {module_name}" --update-{module_name}-init """ __DYNAMIC__ = True if __DYNAMIC__: # TODO: import all utool external prereqs. Then the imports will not import # anything that has already in a toplevel namespace # COMMENTED OUT FOR FROZEN __INIT__ # Dynamically import listed util libraries and their members. from utool._internal import util_importer # FIXME: this might actually work with rrrr, but things arent being # reimported because they are already in the modules list import_execstr = util_importer.dynamic_import(__name__, IMPORT_TUPLES) exec(import_execstr) DOELSE = False else: # Do the nonexec import (can force it to happen no matter what if alwyas set # to True) DOELSE = True if DOELSE: # <AUTOGEN_INIT> pass # </AUTOGEN_INIT> '''.format(module_name=module_name) ) ts_line = '# Autogenerated on {ts}'.format(ts=ut.get_timestamp('printable')) init_codeblock_list = ['# -*- coding: utf-8 -*-', ts_line] init_codeblock_list.append(initstr) init_codeblock_list.append('\nIMPORT_TUPLES = ' + ut.list_str(IMPORT_TUPLES)) if use_star: init_codeblock_list.append(importstar_codeblock) init_codeblock_list.append(regen_block) init_codeblock = '\n'.join(init_codeblock_list) return init_codeblock
def fix_conference_title_names(clean_text, key_list=None): """ mass bibtex fixes CommandLine: ./fix_bib.py """ # Find citations from the tex documents if key_list is None: key_list = find_used_citations(testdata_fpaths()) key_list = list(set(key_list)) ignore = ['JP', '?'] for item in ignore: try: key_list.remove(item) except ValueError: pass unknown_confkeys = [] conference_keys = [ 'journal', 'booktitle', ] ignore_confkey = [] bib_database = bibtexparser.loads(clean_text) bibtex_dict = bib_database.get_entry_dict() isect = set(ignore_confkey).intersection( set(constants_tex_fixes.CONFERENCE_TITLE_MAPS.keys())) assert len(isect) == 0, repr(isect) #ut.embed() #conftitle_to_types_hist = ut.ddict(list) type_key = 'ENTRYTYPE' debug_author = ut.get_argval('--debug-author', type_=str, default=None) # ./fix_bib.py --debug_author=Kappes for key in bibtex_dict.keys(): entry = bibtex_dict[key] if debug_author is not None: debug = debug_author in entry.get('author', '') else: debug = False if debug: print(' --- ENTRY ---') print(ut.repr3(entry)) #if type_key not in entry: # #entry[type_key] = entry['ENTRYTYPE'] # ut.embed() # Clip abstrat if 'abstract' in entry: entry['abstract'] = ' '.join(entry['abstract'].split(' ')[0:7]) # Remove Keys remove_keys = [ 'note', 'urldate', 'series', 'publisher', 'isbn', 'editor', 'shorttitle', 'copyright', 'language', 'month', # These will be put back in #'number', #'pages', #'volume', ] entry = ut.delete_dict_keys(entry, remove_keys) # Fix conference names confkeys = list(set(entry.keys()).intersection(set(conference_keys))) #entry = ut.delete_dict_keys(entry, ['abstract']) # TODO: FIX THESE IF NEEDBE #if len(confkeys) == 0: # print(ut.dict_str(entry)) # print(entry.keys()) if len(confkeys) == 1: confkey = confkeys[0] old_confval = entry[confkey] # Remove curly braces old_confval = old_confval.replace('{', '').replace('}', '') if old_confval in ignore_confkey: print(ut.dict_str(entry)) continue new_confval_candiates = [] if old_confval.startswith('arXiv'): continue # for conf_title, patterns in constants_tex_fixes.CONFERENCE_TITLE_MAPS.items(): for conf in constants_tex_fixes.CONFERENCES: if conf.matches(old_confval): conf_title = conf.accro() if debug: print('old_confval = %r' % (old_confval, )) print('conf_title = %r' % (conf_title, )) new_confval = conf_title new_confval_candiates.append(new_confval) if len(new_confval_candiates) == 0: new_confval = None elif len(new_confval_candiates) == 1: new_confval = new_confval_candiates[0] else: assert False, 'double match' if new_confval is None: if key in key_list: unknown_confkeys.append(old_confval) #print(old_confval) else: # Overwrite old confval entry[confkey] = new_confval # Record info about types of conferneces true_confval = entry[confkey].replace('{', '').replace('}', '') # FIX ENTRIES THAT SHOULD BE CONFERENCES if true_confval in constants_tex_fixes.CONFERENCE_LIST: if entry[type_key] == 'inproceedings': pass #print(confkey) #print(ut.dict_str(entry)) elif entry[type_key] == 'article': entry['booktitle'] = entry['journal'] del entry['journal'] #print(ut.dict_str(entry)) elif entry[type_key] == 'incollection': pass else: raise AssertionError('UNKNOWN TYPE: %r' % (entry[type_key], )) if 'booktitle' not in entry: print('DOES NOT HAVE CORRECT CONFERENCE KEY') print(ut.dict_str(entry)) assert 'journal' not in entry, 'should not have journal' #print(entry['type']) entry[type_key] = 'inproceedings' # FIX ENTRIES THAT SHOULD BE JOURNALS if true_confval in constants_tex_fixes.JOURNAL_LIST: if entry[type_key] == 'article': pass elif entry[type_key] == 'inproceedings': pass #print(ut.dict_str(entry)) elif entry[type_key] == 'incollection': pass else: raise AssertionError('UNKNOWN TYPE: %r' % (entry['type'], )) if 'journal' not in entry: print('DOES NOT HAVE CORRECT CONFERENCE KEY') print(ut.dict_str(entry)) assert 'booktitle' not in entry, 'should not have booktitle' #print(entry['type']) #entry['type'] = 'article' #conftitle_to_types_hist[true_confval].append(entry['type']) elif len(confkeys) > 1: raise AssertionError('more than one confkey=%r' % (confkeys, )) # Fix Authors if 'author' in entry: authors = six.text_type(entry['author']) for truename, alias_list in constants_tex_fixes.AUTHOR_NAME_MAPS.items( ): pattern = six.text_type( ut.regex_or([ ut.util_regex.whole_word(alias) for alias in alias_list ])) authors = re.sub(pattern, six.text_type(truename), authors, flags=re.UNICODE) entry['author'] = authors """ article = journal inprocedings = converence paper """ #conftitle_to_types_set_hist = {key: set(val) for key, val in conftitle_to_types_hist.items()} #print(ut.dict_str(conftitle_to_types_set_hist)) print(ut.list_str(sorted(unknown_confkeys))) print('len(unknown_confkeys) = %r' % (len(unknown_confkeys), )) writer = BibTexWriter() writer.contents = ['comments', 'entries'] writer.indent = ' ' writer.order_entries_by = ('type', 'author', 'year') new_bibtex_str = bibtexparser.dumps(bib_database, writer) return new_bibtex_str
def print_acfg_list(acfg_list, expanded_aids_list=None, ibs=None, combined=False, **kwargs): r""" Args: acfg_list (list): expanded_aids_list (list): (default = None) ibs (IBEISController): ibeis controller object(default = None) combined (bool): (default = False) CommandLine: python -m ibeis.expt.annotation_configs --exec-print_acfg_list --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.expt.annotation_configs import * # NOQA >>> import ibeis >>> acfg_list = '?' >>> expanded_aids_list = None >>> ibs = None >>> combined = False >>> result = print_acfg_list(acfg_list, expanded_aids_list, ibs, combined) >>> print(result) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ _tup = compress_acfg_list_for_printing(acfg_list) nonvaried_compressed_dict, varied_compressed_dict_list = _tup ut.colorprint('+=== <Info acfg_list> ===', 'white') #print('Printing acfg_list info. len(acfg_list) = %r' % (len(acfg_list),)) print('non-varied aidcfg = ' + ut.dict_str(nonvaried_compressed_dict)) seen_ = ut.ddict(list) # get default kwkeys for annot info if ibs is not None: annotstats_kw = kwargs.copy() kwkeys = ut.parse_func_kwarg_keys(ibs.get_annot_stats_dict) annotstats_kw.update( ut.argparse_dict(dict(zip(kwkeys, [None] * len(kwkeys))), only_specified=True)) hashid_list = [] for acfgx in range(len(acfg_list)): acfg = acfg_list[acfgx] title = ('q_cfgname=' + acfg['qcfg']['_cfgname'] + ' d_cfgname=' + acfg['dcfg']['_cfgname']) ut.colorprint( '+--- acfg %d / %d -- %s ---- ' % (acfgx + 1, len(acfg_list), title), 'lightgray') print('acfg = ' + ut.dict_str(varied_compressed_dict_list[acfgx], strvals=True)) if expanded_aids_list is not None: qaids, daids = expanded_aids_list[acfgx] key = (ut.hashstr_arr27(qaids, 'qaids'), ut.hashstr_arr27(daids, 'daids')) if key not in seen_: if ibs is not None: seen_[key].append(acfgx) stats_, locals_ = ibs.get_annotconfig_stats( qaids, daids, verbose=False, combined=combined, **annotstats_kw) hashids = (stats_['qaid_stats']['qhashid'], stats_['daid_stats']['dhashid']) hashid_list.append(hashids) stats_str2 = ut.dict_str(stats_, strvals=True, newlines=True, explicit=False, nobraces=False) print('annot_config_stats = ' + stats_str2) else: dupindex = seen_[key] print('DUPLICATE of index %r' % (dupindex, )) dupdict = varied_compressed_dict_list[dupindex[0]] print('DUP OF acfg = ' + ut.dict_str(dupdict, strvals=True)) print('hashid summary = ' + ut.list_str(hashid_list, nl=1)) ut.colorprint('L___ </Info acfg_list> ___', 'white')
def list_distinctivness_cache(): global_distinctdir = sysres.get_global_distinctiveness_modeldir() print(ut.list_str(ut.ls(global_distinctdir)))
def inject_instance(self, classtype=None, allow_override=False, verbose=VERBOSE_CLASS, strict=True): """ Injects an instance (self) of type (classtype) with all functions registered to (classtype) call this in the __init__ class function Args: self: the class instance classtype: key for a class, preferably the class type itself, but it doesnt have to be SeeAlso: make_class_method_decorator Example: >>> # DOCTEST_DISABLE >>> utool.make_class_method_decorator(InvertedIndex)(smk_debug.invindex_dbgstr) >>> utool.inject_instance(invindex) """ import utool as ut if verbose: print('[util_class] begin inject_instance') try: if classtype is None: # Probably should depricate this block of code # It tries to do too much classtype = self.__class__ if classtype == 'ibeis.gui.models_and_views.IBEISTableView': # HACK HACK HACK from guitool.__PYQT__ import QtGui classtype = QtGui.QAbstractItemView if len(__CLASSTYPE_ATTRIBUTES__[classtype]) == 0: print('[utool] Warning: no classes of type %r are registered' % (classtype,)) print('[utool] type(self)=%r, self=%r' % (type(self), self)), print('[utool] Checking to see if anybody else was registered...') print('[utool] __CLASSTYPE_ATTRIBUTES__ = ' + ut.list_str(__CLASSTYPE_ATTRIBUTES__.keys())) for classtype_, _ in six.iteritems(__CLASSTYPE_ATTRIBUTES__): isinstance(self, classtype_) classtype = classtype_ print('[utool] Warning: using subclass=%r' % (classtype_,)) break func_list = __CLASSTYPE_ATTRIBUTES__[classtype] if verbose or util_arg.VERBOSE: print('[util_class] injecting %d methods\n with classtype=%r\n into %r' % (len(func_list), classtype, self,)) for func in func_list: if VERBOSE_CLASS: print('[util_class] * injecting %r' % (func,)) method_name = None # Allow user to register tuples for aliases if isinstance(func, tuple): func, method_name = func inject_func_as_method(self, func, method_name=method_name, allow_override=allow_override) if verbose: print('[util_class] Running postinject functions on %r' % (self,)) for func in __CLASSTYPE_POSTINJECT_FUNCS__[classtype]: func(self) if verbose: print('[util_class] Finished injecting instance self=%r' % (self,)) except Exception as ex: ut.printex(ex, 'ISSUE WHEN INJECTING %r' % (classtype,), iswarning=not strict) if strict: raise
def test_sver_wrapper2(): r""" CommandLine: python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 --no-c --quiet python -m vtool.sver_c_wrapper --test-test_sver_wrapper2 --rebuild-sver Example: >>> # ENABLE_DOCTEST >>> from vtool.sver_c_wrapper import * # NOQA >>> result = test_sver_wrapper2() >>> print(result) Ignore: C (Serial): unique cases affine inliers: [ '[ 4 25 33 36 37 53]', ] unique cases homog inliers: [ '[]', ] C (Parallel) unique cases affine inliers: [ '[ 4 25 33 36 37 53]', '[10 19 25 29 36 39 53]', ] unique cases homog inliers: [ '[10 43 53]', '[]', ] Python: unique cases affine inliers: [ '[10 19 25 29 36 39 53]', ] unique cases homog inliers: [ '[10 43 53]', ] """ import vtool import vtool.tests.testdata_nondeterm_sver kpts1, kpts2, fm, xy_thresh, scale_thresh, ori_thresh, dlen_sqrd2, min_nInliers, match_weights, full_homog_checks = vtool.tests.testdata_nondeterm_sver.testdata_nondeterm_sver( ) inliers_list = [] homog_inliers_list = [] for x in range(10): sv_tup = vtool.spatially_verify_kpts( kpts1, kpts2, fm, xy_thresh, scale_thresh, ori_thresh, dlen_sqrd2, min_nInliers, match_weights=match_weights, full_homog_checks=full_homog_checks, returnAff=True) aff_inliers = sv_tup[3] inliers_list.append(str(aff_inliers)) homog_inliers_list.append(str(sv_tup[0])) #print(sv_tup[0]) #print(sv_tup[3]) print('unique cases affine inliers: ' + ut.list_str(list(set(inliers_list)))) print('unique cases homog inliers: ' + ut.list_str(list(set(homog_inliers_list))))
def augment_nnindexer_experiment(): """ References: http://answers.opencv.org/question/44592/flann-index-training-fails-with-segfault/ CommandLine: utprof.py -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_MTEST --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --show python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --show python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 --nosave-flann --no-api-cache --nocache-uuids python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_MTEST --show python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show # RUNS THE SEGFAULTING CASE python -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show # Debug it gdb python run -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --show gdb python run -m ibeis.algo.hots._neighbor_experiment --test-augment_nnindexer_experiment --db PZ_Master0 --diskshow --adjust=.1 --save "augment_experiment_{db}.png" --dpath='.' --dpi=180 --figsize=9,6 Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.hots._neighbor_experiment import * # NOQA >>> # execute function >>> augment_nnindexer_experiment() >>> # verify results >>> ut.show_if_requested() """ import ibeis # build test data #ibs = ibeis.opendb('PZ_MTEST') ibs = ibeis.opendb(defaultdb='PZ_Master0') if ibs.get_dbname() == 'PZ_MTEST': initial = 1 addition_stride = 4 max_ceiling = 100 elif ibs.get_dbname() == 'PZ_Master0': initial = 128 #addition_stride = 64 #addition_stride = 128 addition_stride = 256 max_ceiling = 10000 #max_ceiling = 4000 #max_ceiling = 2000 #max_ceiling = 600 else: assert False all_daids = ibs.get_valid_aids(species='zebra_plains') qreq_ = ibs.new_query_request(all_daids, all_daids) max_num = min(max_ceiling, len(all_daids)) # Clear Caches ibs.delete_flann_cachedir() neighbor_index_cache.clear_memcache() neighbor_index_cache.clear_uuid_cache(qreq_) # Setup all_randomize_daids_ = ut.deterministic_shuffle(all_daids[:]) # ensure all features are computed #ibs.get_annot_vecs(all_randomize_daids_, ensure=True) #ibs.get_annot_fgweights(all_randomize_daids_, ensure=True) nnindexer_list = [] addition_lbl = 'Addition' _addition_iter = list(range(initial + 1, max_num, addition_stride)) addition_iter = iter(ut.ProgressIter(_addition_iter, lbl=addition_lbl, freq=1, autoadjust=False)) time_list_addition = [] #time_list_reindex = [] addition_count_list = [] tmp_cfgstr_list = [] #for _ in range(80): # next(addition_iter) try: memtrack = ut.MemoryTracker(disable=False) for count in addition_iter: aid_list_ = all_randomize_daids_[0:count] # Request an indexer which could be an augmented version of an existing indexer. with ut.Timer(verbose=False) as t: memtrack.report('BEFORE AUGMENT') nnindexer_ = neighbor_index_cache.request_augmented_ibeis_nnindexer(qreq_, aid_list_) memtrack.report('AFTER AUGMENT') nnindexer_list.append(nnindexer_) addition_count_list.append(count) time_list_addition.append(t.ellapsed) tmp_cfgstr_list.append(nnindexer_.cfgstr) print('===============\n\n') print(ut.list_str(time_list_addition)) print(ut.list_str(list(map(id, nnindexer_list)))) print(ut.list_str(tmp_cfgstr_list)) print(ut.list_str(list([nnindxer.cfgstr for nnindxer in nnindexer_list]))) IS_SMALL = False if IS_SMALL: nnindexer_list = [] reindex_label = 'Reindex' # go backwards for reindex _reindex_iter = list(range(initial + 1, max_num, addition_stride))[::-1] reindex_iter = ut.ProgressIter(_reindex_iter, lbl=reindex_label) time_list_reindex = [] #time_list_reindex = [] reindex_count_list = [] for count in reindex_iter: print('\n+===PREDONE====================\n') # check only a single size for memory leaks #count = max_num // 16 + ((x % 6) * 1) #x += 1 aid_list_ = all_randomize_daids_[0:count] # Call the same code, but force rebuilds memtrack.report('BEFORE REINDEX') with ut.Timer(verbose=False) as t: nnindexer_ = neighbor_index_cache.request_augmented_ibeis_nnindexer( qreq_, aid_list_, force_rebuild=True, memtrack=memtrack) memtrack.report('AFTER REINDEX') ibs.print_cachestats_str() print('[nnindex.MEMCACHE] size(NEIGHBOR_CACHE) = %s' % ( ut.get_object_size_str(neighbor_index_cache.NEIGHBOR_CACHE.items()),)) print('[nnindex.MEMCACHE] len(NEIGHBOR_CACHE) = %s' % ( len(neighbor_index_cache.NEIGHBOR_CACHE.items()),)) print('[nnindex.MEMCACHE] size(UUID_MAP_CACHE) = %s' % ( ut.get_object_size_str(neighbor_index_cache.UUID_MAP_CACHE),)) print('totalsize(nnindexer) = ' + ut.get_object_size_str(nnindexer_)) memtrack.report_type(neighbor_index_cache.NeighborIndex) ut.print_object_size_tree(nnindexer_, lbl='nnindexer_') if IS_SMALL: nnindexer_list.append(nnindexer_) reindex_count_list.append(count) time_list_reindex.append(t.ellapsed) #import cv2 #import matplotlib as mpl #print(mem_top.mem_top(limit=30, width=120, # #exclude_refs=[cv2.__dict__, mpl.__dict__] # )) print('L___________________\n\n\n') print(ut.list_str(time_list_reindex)) if IS_SMALL: print(ut.list_str(list(map(id, nnindexer_list)))) print(ut.list_str(list([nnindxer.cfgstr for nnindxer in nnindexer_list]))) except KeyboardInterrupt: print('\n[train] Caught CRTL+C') resolution = '' from six.moves import input while not (resolution.isdigit()): print('\n[train] What do you want to do?') print('[train] 0 - Continue') print('[train] 1 - Embed') print('[train] ELSE - Stop network training') resolution = input('[train] Resolution: ') resolution = int(resolution) # We have a resolution if resolution == 0: print('resuming training...') elif resolution == 1: ut.embed() import plottool as pt next_fnum = iter(range(0, 1)).next # python3 PY3 pt.figure(fnum=next_fnum()) if len(addition_count_list) > 0: pt.plot2(addition_count_list, time_list_addition, marker='-o', equal_aspect=False, x_label='num_annotations', label=addition_lbl + ' Time') if len(reindex_count_list) > 0: pt.plot2(reindex_count_list, time_list_reindex, marker='-o', equal_aspect=False, x_label='num_annotations', label=reindex_label + ' Time') pt.set_figtitle('Augmented indexer experiment') pt.legend()
def get_data_list(): r""" CommandLine: python ~/code/ibeis/_installers/ibeis_pyinstaller_data_helper.py --test-get_data_list Example: >>> # ENABLE_DOCTEST >>> from ibeis_pyinstaller_data_helper import * # NOQA >>> result = get_data_list() >>> DATATUP_LIST, BINARYTUP_LIST, iconfile = result >>> print('DATATUP_LIST = ' + ut.list_str(DATATUP_LIST)) >>> print('BINARYTUP_LIST = ' + ut.list_str(BINARYTUP_LIST)) >>> print(len(DATATUP_LIST)) >>> print(len(BINARYTUP_LIST)) >>> print(iconfile) """ # Build data before running analysis for quick debugging DATATUP_LIST = [] BINARYTUP_LIST = [] #import pyhesaff #pyhesaff.HESAFF_CLIB.__LIB_FPATH__ #import pyrf #pyrf.RF_CLIB.__LIB_FPATH__ # Hesaff libhesaff_fname = 'libhesaff' + LIB_EXT libhesaff_src = realpath( join(root_dir, '..', 'hesaff', 'pyhesaff', libhesaff_fname)) libhesaff_dst = join(ibsbuild, 'pyhesaff', 'lib', libhesaff_fname) DATATUP_LIST.append((libhesaff_dst, libhesaff_src)) # PyRF libpyrf_fname = 'libpyrf' + LIB_EXT libpyrf_src = realpath(join(root_dir, '..', 'pyrf', 'pyrf', libpyrf_fname)) libpyrf_dst = join(ibsbuild, 'pyrf', 'lib', libpyrf_fname) DATATUP_LIST.append((libpyrf_dst, libpyrf_src)) # FLANN libflann_fname = 'libflann' + LIB_EXT #try: # #import pyflann # #pyflann.__file__ # #join(dirname(dirname(pyflann.__file__)), 'build') #except ImportError as ex: # print('PYFLANN IS NOT IMPORTABLE') # raise #if WIN32 or LINUX: # FLANN #libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) #libflann_dst = join(ibsbuild, libflann_fname) #elif APPLE: # # libflann_src = '/pyflann/lib/libflann.dylib' # # libflann_dst = join(ibsbuild, libflann_fname) # libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) # libflann_dst = join(ibsbuild, libflann_fname) # This path is when pyflann was built using setup.py develop libflann_src = realpath( join(root_dir, '..', 'flann', 'build', 'lib', libflann_fname)) libflann_dst = join(ibsbuild, 'pyflann', 'lib', libflann_fname) DATATUP_LIST.append((libflann_dst, libflann_src)) # VTool vtool_libs = ['libsver'] for libname in vtool_libs: lib_fname = libname + LIB_EXT vtlib_src = realpath(join(root_dir, '..', 'vtool', 'vtool', lib_fname)) vtlib_dst = join(ibsbuild, 'vtool', lib_fname) DATATUP_LIST.append((vtlib_dst, vtlib_src)) linux_lib_dpaths = [ '/usr/lib/x86_64-linux-gnu', '/usr/lib', '/usr/local/lib' ] # OpenMP if APPLE: # BSDDB, Fix for the modules that PyInstaller needs and (for some reason) # are not being added by PyInstaller libbsddb_src = '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload/_bsddb.so' libbsddb_dst = join(ibsbuild, '_bsddb.so') DATATUP_LIST.append((libbsddb_dst, libbsddb_src)) #libgomp_src = '/opt/local/lib/libgomp.dylib' libgomp_src = '/opt/local/lib/gcc48/libgomp.dylib' BINARYTUP_LIST.append(('libgomp.1.dylib', libgomp_src, 'BINARY')) # very hack libiomp_src = '/Users/bluemellophone/code/libomp_oss/exports/mac_32e/lib.thin/libiomp5.dylib' BINARYTUP_LIST.append(('libiomp5.dylib', libiomp_src, 'BINARY')) if LINUX: libgomp_src = ut.search_in_dirs('libgomp.so.1', linux_lib_dpaths) ut.assertpath(libgomp_src) BINARYTUP_LIST.append(('libgomp.so.1', libgomp_src, 'BINARY')) # MinGW if WIN32: mingw_root = r'C:\MinGW\bin' mingw_dlls = [ 'libgcc_s_dw2-1.dll', 'libstdc++-6.dll', 'libgomp-1.dll', 'pthreadGC2.dll' ] for lib_fname in mingw_dlls: lib_src = join(mingw_root, lib_fname) lib_dst = join(ibsbuild, lib_fname) DATATUP_LIST.append((lib_dst, lib_src)) # We need to add these 4 opencv libraries because pyinstaller does not find them. #OPENCV_EXT = {'win32': '248.dll', # 'darwin': '.2.4.dylib', # 'linux2': '.so.2.4'}[PLATFORM] target_cv_version = '3.0.0' OPENCV_EXT = { 'win32': target_cv_version.replace('.', '') + '.dll', 'darwin': '.' + target_cv_version + '.dylib', 'linux2': '.so.' + target_cv_version }[PLATFORM] missing_cv_name_list = [ 'libopencv_videostab', 'libopencv_superres', 'libopencv_stitching', #'libopencv_gpu', 'libopencv_core', 'libopencv_highgui', 'libopencv_imgproc', ] # Hack to find the appropriate opencv libs for name in missing_cv_name_list: fname = name + OPENCV_EXT src = '' dst = '' if APPLE: src = join('/opt/local/lib', fname) elif LINUX: #src = join('/usr/lib', fname) src, tried = ut.search_in_dirs(fname, linux_lib_dpaths, strict=True, return_tried=True) elif WIN32: if ut.get_computer_name() == 'Ooo': src = join(r'C:/Program Files (x86)/OpenCV/x86/mingw/bin', fname) else: src = join(root_dir, '../opencv/build/bin', fname) dst = join(ibsbuild, fname) # ut.assertpath(src) DATATUP_LIST.append((dst, src)) ################################## # QT Gui dependencies ################################## if APPLE: walk_path = '/opt/local/Library/Frameworks/QtGui.framework/Versions/4/Resources/qt_menu.nib' for root, dirs, files in os.walk(walk_path): for lib_fname in files: toc_src = join(walk_path, lib_fname) toc_dst = join('qt_menu.nib', lib_fname) DATATUP_LIST.append((toc_dst, toc_src)) ################################## # Documentation, Icons, and Web Assets ################################## # Documentation #userguide_dst = join('.', '_docs', 'IBEISUserGuide.pdf') #userguide_src = join(root_dir, '_docs', 'IBEISUserGuide.pdf') #DATATUP_LIST.append((userguide_dst, userguide_src)) # Icon File ICON_EXT = {'darwin': '.icns', 'win32': '.ico', 'linux2': '.ico'}[PLATFORM] iconfile = join('_installers', 'ibsicon' + ICON_EXT) icon_src = join(root_dir, iconfile) icon_dst = join(ibsbuild, iconfile) DATATUP_LIST.append((icon_dst, icon_src)) print('[installer] Checking Data (preweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath( src, verbose=True), 'checkpath for src=%r failed' % (src, ) except Exception as ex: ut.printex( ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise # Web Assets INSTALL_WEB = True and not ut.get_argflag('--noweb') if INSTALL_WEB: web_root = join('ibeis', 'web/') #walk_path = join(web_root, 'static') #static_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # static_data.append((toc_dst, toc_src)) #ut.get_list_column(static_data, 1) == ut.glob(walk_path, '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_src_list = ut.glob(join(web_root, 'static'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_dst_list = [ relpath(src, join(root_dir, 'ibeis')) for src in static_src_list ] static_data = zip(static_dst_list, static_src_list) DATATUP_LIST.extend(static_data) #walk_path = join(web_root, 'templates') #template_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # template_data.append((toc_dst, toc_src)) template_src_list = ut.glob(join(web_root, 'templates'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) template_dst_list = [ relpath(src, join(root_dir, 'ibeis')) for src in template_src_list ] template_data = zip(template_dst_list, template_src_list) DATATUP_LIST.extend(template_data) print('[installer] Checking Data (postweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath( src, verbose=False), 'checkpath for src=%r failed' % (src, ) except Exception as ex: ut.printex( ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise return DATATUP_LIST, BINARYTUP_LIST, iconfile
def ensure_pz_mtest_mergesplit_test(): r""" Make a test database for MERGE and SPLIT cases CommandLine: python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test Example: >>> # SCRIPT >>> from ibeis.init.sysres import * # NOQA >>> ensure_pz_mtest_mergesplit_test() """ import ibeis ibeis.ensure_pz_mtest() workdir = ibeis.sysres.get_workdir() mtest_dbpath = join(workdir, 'PZ_MTEST') source_dbdir = mtest_dbpath dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST') if ut.get_argflag('--reset'): ut.delete(dest_dbdir) if ut.checkpath(dest_dbdir): return copy_ibeisdb(source_dbdir, dest_dbdir) ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST') assert len(ibs.get_valid_aids()) == 119 assert len(ibs.get_valid_nids()) == 41 aid_list = ibs.get_valid_aids() aids_list, nid_list = ibs.group_annots_by_name(aid_list) num_aids = list(map(len, aids_list)) # num cases wanted num_merge = 3 num_split = 1 num_combo = 1 # num inputs needed num_merge_names = num_merge num_split_names = num_split * 2 num_combo_names = num_combo * 3 total_names = num_merge_names + num_split_names + num_combo_names modify_aids = ut.take( aids_list, ut.list_argsort(num_aids, reverse=True)[0:total_names]) merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE') merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE') split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0] combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO') # the first 3 become merge cases #left = 0 #right = left + num_merge for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2): #ibs.get_annot_nids(aids) aids_ = aids[::2] ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_)) ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_)) # the next 2 become split cases #left = right #right = left + num_split_names for aids in modify_aids[3:5]: ibs.set_annot_name_rowids(aids, [split_nid] * len(aids)) #left = right #right = left + num_combo_names # The final 3 are a combination case for aids in modify_aids[5:8]: aids_even = aids[::2] aids_odd = aids[1::2] ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even)) ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd)) final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids) print('final_result = %s' % (ut.list_str(final_result), ))
def build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list): """ helper these list comprehensions replace the prevous for loop they still need to be optimized a little bit (and made clearer) can probably unnest the list comprehensions as well """ """ IGNORE Legacy:: def old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list): fm_nestlist_ = [] fs_nestlist_ = [] daid_nestlist_ = [] for scores, qfxs, dfxs, daids in zip(sparse_list, qfxs_list, dfxs_list, daids_list): for rx, cx, score in zip(scores.row, scores.col, scores.data): _fm = tuple(product(qfxs[rx], dfxs[cx])) _fs = [score / len(_fm)] * len(_fm) _daid = [daids[cx]] * len(_fm) fm_nestlist_.append(_fm) fs_nestlist_.append(_fs) daid_nestlist_.append(_daid) return fm_nestlist_, fs_nestlist_, daid_nestlist_ oldtup_ = old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) fm_nestlist_, fs_nestlist_, daid_nestlist_ = oldtup_ newtup_ = build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) fm_nestlist, fs_nestlist, daid_nestlist = newtup_ assert fm_nestlist == fm_nestlist_ assert fs_nestlist == fs_nestlist_ assert daid_nestlist == daid_nestlist_ 47ms %timeit build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) 59ms %timeit old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) IGNORE """ # FIXME: rewrite double comprehension as a flat comprehension # Build nested feature matches (a single match might have many members) fm_nestlist = [ tuple(product(qfxs[rx], dfxs[cx])) for scores, qfxs, dfxs in zip(sparse_list, qfxs_list, dfxs_list) for rx, cx in zip(scores.row, scores.col) ] nFm_list = [len(fm) for fm in fm_nestlist] #fs_unsplit = (score # for scores in sparse_list # for score in scores.data) #daid_unsplit = (daids[cx] # for scores, daids in zip(sparse_list, daids_list) # for cx in scores.col) # Build nested feature scores fs_unsplit = utool.iflatten( (scores.data for scores in sparse_list)) # Build nested feature matches (a single match might have many members) daid_unsplit = utool.iflatten( (daids.take(scores.col) for scores, daids in zip(sparse_list, daids_list))) # Expand feature scores and daids splitting scores amongst match members fs_nestlist = [ [score / nFm] * nFm for score, nFm in zip(fs_unsplit, nFm_list) ] daid_nestlist = [ [daid] * nFm for daid, nFm in zip(daid_unsplit, nFm_list) ] if DEBUG_SMK: assert len(fm_nestlist) == len(fs_nestlist), 'inconsistent len' assert len(fm_nestlist) == len(nFm_list), 'inconsistent len' assert len(daid_nestlist) == len(fs_nestlist), 'inconsistent len' min_ = min(2, len(nFm_list)) max_ = min(15, len(nFm_list)) print('nFm_list[_min:_max] = ' + utool.list_str(nFm_list[min_:max_])) print('fm_nestlist[_min:_max] = ' + utool.list_str(fm_nestlist[min_:max_])) print('fs_nestlist[_min:_max] = ' + utool.list_str(fs_nestlist[min_:max_])) print('daid_nestlist[_min:_max] = ' + utool.list_str(daid_nestlist[min_:max_])) for fm_, fs_, daid_ in zip(fm_nestlist, fs_nestlist, daid_nestlist): assert len(fm_) == len(fs_), 'inconsistent len' assert len(fm_) == len(daid_), 'inconsistent len' print('[smk_core] checked build_chipmatch correspondence ...ok') return fm_nestlist, fs_nestlist, daid_nestlist
def grep_projects(tofind_list, user_profile=None, verbose=True, new=False, **kwargs): r""" Greps the projects defined in the current UserProfile Args: tofind_list (list): user_profile (None): (default = None) Kwargs: user_profile CommandLine: python -m utool --tf grep_projects grep_projects Example: >>> # DISABLE_DOCTEST >>> from utool.util_project import * # NOQA >>> import utool as ut >>> import sys >>> tofind_list = ut.get_argval('--find', type_=list, >>> default=[sys.argv[-1]]) >>> grep_projects(tofind_list) """ import utool as ut user_profile = ensure_user_profile(user_profile) kwargs = kwargs.copy() colored = kwargs.pop('colored', True) grepkw = {} grepkw['greater_exclude_dirs'] = user_profile.project_exclude_dirs grepkw['exclude_dirs'] = user_profile.project_exclude_dirs grepkw['dpath_list'] = user_profile.project_dpaths grepkw['include_patterns'] = user_profile.project_include_patterns grepkw['exclude_patterns'] = user_profile.project_exclude_patterns grepkw.update(kwargs) msg_list1 = [] msg_list2 = [] print_ = msg_list1.append print_('Greping Projects') print_('tofind_list = %s' % (ut.list_str(tofind_list, nl=True), )) #print_('grepkw = %s' % ut.dict_str(grepkw, nl=True)) if verbose: print('\n'.join(msg_list1)) #with ut.Timer('greping', verbose=True): grep_result = ut.grep(tofind_list, **grepkw) found_fpath_list, found_lines_list, found_lxs_list = grep_result # HACK, duplicate behavior. TODO: write grep print result function reflags = grepkw.get('reflags', 0) _exprs_flags = [ut.extend_regex2(expr, reflags) for expr in tofind_list] extended_regex_list = ut.take_column(_exprs_flags, 0) reflags_list = ut.take_column(_exprs_flags, 1) # HACK # pat = ut.util_regex.regex_or(extended_regex_list) reflags = reflags_list[0] # from utool import util_regex resultstr = ut.make_grep_resultstr(grep_result, extended_regex_list, reflags, colored=colored) msg_list2.append(resultstr) print_ = msg_list2.append #for fpath, lines, lxs in zip(found_fpath_list, found_lines_list, # found_lxs_list): # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, line)) # iter_ = zip(found_fpath_list, found_lines_list, found_lxs_list) # for fpath, lines, lxs in iter_: # print_('----------------------') # print_('found %d line(s) in %r: ' % (len(lines), fpath)) # name = split(fpath)[1] # max_line = len(lines) # ndigits = str(len(str(max_line))) # for (lx, line) in zip(lxs, lines): # line = line.replace('\n', '') # colored_line = ut.highlight_regex( # line.rstrip('\n'), pat, reflags=reflags) # print_(('%s : %' + ndigits + 'd |%s') % (name, lx, colored_line)) print_('====================') print_('found_fpath_list = ' + ut.list_str(found_fpath_list)) print_('') #print_('gvim -o ' + ' '.join(found_fpath_list)) if verbose: print('\n'.join(msg_list2)) msg_list = msg_list1 + msg_list2 if new: return GrepResult(found_fpath_list, found_lines_list, found_lxs_list, extended_regex_list, reflags) else: return msg_list
def findcite(): """ prints info about used and unused citations """ tex_fpath_list = testdata_fpaths() citekey_list = find_used_citations(tex_fpath_list) # Find uncited entries #bibtexparser = ut.tryimport('bibtexparser') bib_fpath = 'My_Library_clean.bib' bibtex_str = ut.read_from(bib_fpath) bib_database = bibtexparser.loads(bibtex_str) bibtex_dict = bib_database.get_entry_dict() for key in bibtex_dict.keys(): entry = bibtex_dict[key] entry = ut.map_dict_keys(six.text_type, entry) entry = ut.map_dict_keys(six.text_type.lower, entry) bibtex_dict[key] = entry print('ALL') ignore = ['JP', '?'] citekey_list = ut.setdiff_ordered(sorted(ut.unique(citekey_list)), ignore) #print(ut.indentjoin(citekey_list)) print('len(citekey_list) = %r' % (len(citekey_list), )) unknown_keys = list(set(citekey_list) - set(bibtex_dict.keys())) unused_keys = list(set(bibtex_dict.keys()) - set(citekey_list)) try: if len(unknown_keys) != 0: print('\nUNKNOWN KEYS:') print(ut.list_str(unknown_keys)) raise AssertionError('unknown keys') except AssertionError as ex: ut.printex(ex, iswarning=True, keys=['unknown_keys']) @ut.argv_flag_dec(indent=' ') def close_keys(): if len(unknown_keys) > 0: bibtex_dict.keys() print('\nDid you mean:') for key in unknown_keys: print('---') print(key) print(ut.closet_words(key, bibtex_dict.keys(), 3)) print('L___') else: print('no unkown keys') close_keys() @ut.argv_flag_dec(indent=' ') def print_unused(): print(ut.indentjoin(ut.sortedby(unused_keys, map(len, unused_keys)))) print('len(unused_keys) = %r' % (len(unused_keys), )) print_unused() all_authors = [] for key in bibtex_dict.keys(): entry = bibtex_dict[key] toremove = ['author', '{', '}', r'\\textbackslash'] author = ut.multi_replace(entry.get('author', ''), toremove, '') authors = author.split(' and ') all_authors.extend(authors) @ut.argv_flag_dec(indent=' ') def author_hist(): #print(all_authors) hist_ = ut.dict_hist(all_authors, ordered=True) hist_[''] = None del hist_[''] print('Author histogram') print(ut.dict_str(hist_)[-1000:]) author_hist() @ut.argv_flag_dec(indent=' ') def unused_important(): important_authors = [ 'hinton', 'chum', 'Jegou', 'zisserman', 'schmid', 'sivic', 'matas', 'lowe', 'perronnin', 'douze', ] for key in unused_keys: entry = bibtex_dict[key] author = entry.get('author', '') #authors = author.split(' and ') hasimportant = any(auth in author.lower() for auth in important_authors) if hasimportant or 'smk' in str(entry).lower(): toremove = [ 'note', 'month', 'type', 'pages', 'urldate', 'language', 'volume', 'number', 'publisher' ] entry = ut.delete_dict_keys(entry, toremove) print( ut.dict_str(entry, strvals=True, key_order=['title', 'author', 'id'])) unused_important()
show_diff = ut.get_argflag('--diff') do_write = ut.get_argflag('--write') need_encoding_fpaths = [] for pat in pattern_items: print('Checking for pattern: %r' % (pat, )) for fpath in fpath_list: pattern = re.escape(pat) found_lines, found_lxs = ut.grepfile(fpath, pattern) # DID NOT FIND ENCODING LINE if len(found_lines) == 0: need_encoding_fpaths.append(fpath) print('The following fpaths need encoding lines: ' + ut.list_str(need_encoding_fpaths, strvals=True)) if do_write or show_diff: for fpath in need_encoding_fpaths: print('\n-----------------\nFound file without encodeing line: ' + fpath) line_list = ut.read_lines_from(fpath) linenum = find_encoding_insert_position(line_list) if linenum is not None: #print(' * linenum = %r' % (linenum,)) new_lines = line_list[:linenum] + [encoding_line + '\n' ] + line_list[linenum:] new_text = ''.join(new_lines) if show_diff: old_text = ''.join(line_list) textdiff = ut.get_textdiff(old_text,
def assert_modules(): """ checkinfo functions return info_dict checkinfo_func CommandLine: python -m ibeis.tests.assert_modules --test-assert_modules Example: >>> # DOCTEST_ENABLE >>> from ibeis.tests.assert_modules import * # NOQA >>> detailed_msg = assert_modules() >>> print(detailed_msg) """ MACHINE_NAME = ut.get_computer_name() machine_info_lines = [] machine_info_lines.append('sys.version = %r ' % (sys.version)) machine_info_lines.append('PATH = ' + ut.list_str(ut.get_path_dirs())) machine_info_lines.append('\n\n\n============================') machine_info_lines.append('Begining assert modules main') machine_info_lines.append('* MACHINE_NAME = %r' % MACHINE_NAME) machine_info_text = '\n'.join(machine_info_lines) print(machine_info_text) statustext_list = [] failed_list = [] fix_list = [] SHOW_STATUS = not ut.get_argflag(('--nostatus', '--nostat')) for checkinfo_wrapper in ASSERT_FUNCS: passed, current_version, target, infodict, statustext, suggested_fix = checkinfo_wrapper() funcname = get_funcname(checkinfo_wrapper) if SHOW_STATUS: statustext_list.append(statustext) if passed: statustext_list.append(funcname + ' ' + str(infodict['__version__']) + ' passed') #statustext_list.append('') else: failed_list.append(funcname + ' FAILED!!!') fix_list.append(suggested_fix) statustext_list.append(funcname + ' FAILED!!!') if SHOW_STATUS: statustext_list.append('') output_text = '\n'.join(statustext_list) failed_text = '\n'.join(failed_list) print(output_text) print(failed_text) check_exist_text = check_modules_exists() print(check_exist_text) fix_text = '' if len(fix_list) > 0: fix_text += ('suggested fixes:\n') fix_text += ('\n'.join(fix_list) + '\n') print(fix_text) detailed_msg = '\n'.join([ machine_info_text, output_text, failed_text, check_exist_text, fix_text, ]) return detailed_msg
def ensure_pz_mtest_mergesplit_test(): r""" Make a test database for MERGE and SPLIT cases CommandLine: python -m ibeis.init.sysres --test-ensure_pz_mtest_mergesplit_test Example: >>> # SCRIPT >>> from ibeis.init.sysres import * # NOQA >>> ensure_pz_mtest_mergesplit_test() """ import ibeis ibeis.ensure_pz_mtest() workdir = ibeis.sysres.get_workdir() mtest_dbpath = join(workdir, 'PZ_MTEST') source_dbdir = mtest_dbpath dest_dbdir = join(workdir, 'PZ_MERGESPLIT_MTEST') if ut.get_argflag('--reset'): ut.delete(dest_dbdir) if ut.checkpath(dest_dbdir): return copy_ibeisdb(source_dbdir, dest_dbdir) ibs = ibeis.opendb('PZ_MERGESPLIT_MTEST') assert len(ibs.get_valid_aids()) == 119 assert len(ibs.get_valid_nids()) == 41 aid_list = ibs.get_valid_aids() aids_list, nid_list = ibs.group_annots_by_name(aid_list) num_aids = list(map(len, aids_list)) # num cases wanted num_merge = 3 num_split = 1 num_combo = 1 # num inputs needed num_merge_names = num_merge num_split_names = num_split * 2 num_combo_names = num_combo * 3 total_names = num_merge_names + num_split_names + num_combo_names modify_aids = ut.take(aids_list, ut.list_argsort(num_aids, reverse=True)[0:total_names]) merge_nids1 = ibs.make_next_nids(num_merge, location_text='XMERGE') merge_nids2 = ibs.make_next_nids(num_merge, location_text='XMERGE') split_nid = ibs.make_next_nids(num_split, location_text='XSPLIT')[0] combo_nids = ibs.make_next_nids(num_combo * 2, location_text='XCOMBO') # the first 3 become merge cases #left = 0 #right = left + num_merge for aids, nid1, nid2 in zip(modify_aids[0:3], merge_nids1, merge_nids2): #ibs.get_annot_nids(aids) aids_ = aids[::2] ibs.set_annot_name_rowids(aids_, [nid1] * len(aids_)) ibs.set_annot_name_rowids(aids_, [nid2] * len(aids_)) # the next 2 become split cases #left = right #right = left + num_split_names for aids in modify_aids[3:5]: ibs.set_annot_name_rowids(aids, [split_nid] * len(aids)) #left = right #right = left + num_combo_names # The final 3 are a combination case for aids in modify_aids[5:8]: aids_even = aids[::2] aids_odd = aids[1::2] ibs.set_annot_name_rowids(aids_even, [combo_nids[0]] * len(aids_even)) ibs.set_annot_name_rowids(aids_odd, [combo_nids[1]] * len(aids_odd)) final_result = ibs.unflat_map(ibs.get_annot_nids, modify_aids) print('final_result = %s' % (ut.list_str(final_result),))
def test_siamese_performance(model, data, labels, flat_metadata, dataname=''): r""" CommandLine: utprof.py -m ibeis_cnn --tf pz_patchmatch --db liberty --test --weights=liberty:current --arch=siaml2_128 --test python -m ibeis_cnn --tf netrun --db liberty --arch=siaml2_128 --test --ensure python -m ibeis_cnn --tf netrun --db liberty --arch=siaml2_128 --test --ensure --weights=new python -m ibeis_cnn --tf netrun --db liberty --arch=siaml2_128 --train --weights=new python -m ibeis_cnn --tf netrun --db pzmtest --weights=liberty:current --arch=siaml2_128 --test # NOQA python -m ibeis_cnn --tf netrun --db pzmtest --weights=liberty:current --arch=siaml2_128 """ import vtool as vt import plottool as pt # TODO: save in model.trainind_dpath/diagnostics/figures ut.colorprint('\n[siam_perf] Testing Siamese Performance', 'white') #epoch_dpath = model.get_epoch_diagnostic_dpath() epoch_dpath = model.arch_dpath ut.vd(epoch_dpath) dataname += ' ' + model.get_history_hashid() + '\n' history_text = ut.list_str(model.era_history, newlines=True) ut.write_to(ut.unixjoin(epoch_dpath, 'era_history.txt'), history_text) #if True: # import matplotlib as mpl # mpl.rcParams['agg.path.chunksize'] = 100000 #data = data[::50] #labels = labels[::50] #from ibeis_cnn import utils #data, labels = utils.random_xy_sample(data, labels, 10000, model.data_per_label_input) FULL = not ut.get_argflag('--quick') fnum_gen = pt.make_fnum_nextgen() ut.colorprint('[siam_perf] Show era history', 'white') fig = model.show_era_loss(fnum=fnum_gen()) pt.save_figure(fig=fig, dpath=epoch_dpath, dpi=180) # hack ut.colorprint('[siam_perf] Show weights image', 'white') fig = model.show_weights_image(fnum=fnum_gen()) pt.save_figure(fig=fig, dpath=epoch_dpath, dpi=180) #model.draw_all_conv_layer_weights(fnum=fnum_gen()) #model.imwrite_weights(1) #model.imwrite_weights(2) # Compute each type of score ut.colorprint('[siam_perf] Building Scores', 'white') test_outputs = model.predict2(model, data) network_output = test_outputs['network_output_determ'] # hack converting network output to distances for non-descriptor networks if len(network_output.shape) == 2 and network_output.shape[1] == 1: cnn_scores = network_output.T[0] elif len(network_output.shape) == 1: cnn_scores = network_output elif len(network_output.shape) == 2 and network_output.shape[1] > 1: assert model.data_per_label_output == 2 vecs1 = network_output[0::2] vecs2 = network_output[1::2] cnn_scores = vt.L2(vecs1, vecs2) else: assert False cnn_scores = cnn_scores.astype(np.float64) # Segfaults with the data passed in is large (AND MEMMAPPED apparently) # Fixed in hesaff implementation SIFT = FULL if SIFT: sift_scores, sift_list = test_sift_patchmatch_scores(data, labels) sift_scores = sift_scores.astype(np.float64) ut.colorprint('[siam_perf] Learning Encoders', 'white') # Learn encoders encoder_kw = { #'monotonize': False, 'monotonize': True, } cnn_encoder = vt.ScoreNormalizer(**encoder_kw) cnn_encoder.fit(cnn_scores, labels) if SIFT: sift_encoder = vt.ScoreNormalizer(**encoder_kw) sift_encoder.fit(sift_scores, labels) # Visualize ut.colorprint('[siam_perf] Visualize Encoders', 'white') viz_kw = dict( with_scores=False, with_postbayes=False, with_prebayes=False, target_tpr=.95, ) inter_cnn = cnn_encoder.visualize( figtitle=dataname + ' CNN scores. #data=' + str(len(data)), fnum=fnum_gen(), **viz_kw) if SIFT: inter_sift = sift_encoder.visualize( figtitle=dataname + ' SIFT scores. #data=' + str(len(data)), fnum=fnum_gen(), **viz_kw) # Save pt.save_figure(fig=inter_cnn.fig, dpath=epoch_dpath) if SIFT: pt.save_figure(fig=inter_sift.fig, dpath=epoch_dpath) # Save out examples of hard errors #cnn_fp_label_indicies, cnn_fn_label_indicies = #cnn_encoder.get_error_indicies(cnn_scores, labels) #sift_fp_label_indicies, sift_fn_label_indicies = #sift_encoder.get_error_indicies(sift_scores, labels) with_patch_examples = FULL if with_patch_examples: ut.colorprint('[siam_perf] Visualize Confusion Examples', 'white') cnn_indicies = cnn_encoder.get_confusion_indicies(cnn_scores, labels) if SIFT: sift_indicies = sift_encoder.get_confusion_indicies(sift_scores, labels) warped_patch1_list, warped_patch2_list = list(zip(*ut.ichunks(data, 2))) samp_args = (warped_patch1_list, warped_patch2_list, labels) _sample = functools.partial(draw_results.get_patch_sample_img, *samp_args) cnn_fp_img = _sample({'fs': cnn_scores}, cnn_indicies.fp)[0] cnn_fn_img = _sample({'fs': cnn_scores}, cnn_indicies.fn)[0] cnn_tp_img = _sample({'fs': cnn_scores}, cnn_indicies.tp)[0] cnn_tn_img = _sample({'fs': cnn_scores}, cnn_indicies.tn)[0] if SIFT: sift_fp_img = _sample({'fs': sift_scores}, sift_indicies.fp)[0] sift_fn_img = _sample({'fs': sift_scores}, sift_indicies.fn)[0] sift_tp_img = _sample({'fs': sift_scores}, sift_indicies.tp)[0] sift_tn_img = _sample({'fs': sift_scores}, sift_indicies.tn)[0] #if ut.show_was_requested(): #def rectify(arr): # return np.flipud(arr) SINGLE_FIG = False if SINGLE_FIG: def dump_img(img_, lbl, fnum): fig, ax = pt.imshow(img_, figtitle=dataname + ' ' + lbl, fnum=fnum) pt.save_figure(fig=fig, dpath=epoch_dpath, dpi=180) dump_img(cnn_fp_img, 'cnn_fp_img', fnum_gen()) dump_img(cnn_fn_img, 'cnn_fn_img', fnum_gen()) dump_img(cnn_tp_img, 'cnn_tp_img', fnum_gen()) dump_img(cnn_tn_img, 'cnn_tn_img', fnum_gen()) dump_img(sift_fp_img, 'sift_fp_img', fnum_gen()) dump_img(sift_fn_img, 'sift_fn_img', fnum_gen()) dump_img(sift_tp_img, 'sift_tp_img', fnum_gen()) dump_img(sift_tn_img, 'sift_tn_img', fnum_gen()) #vt.imwrite(dataname + '_' + 'cnn_fp_img.png', (cnn_fp_img)) #vt.imwrite(dataname + '_' + 'cnn_fn_img.png', (cnn_fn_img)) #vt.imwrite(dataname + '_' + 'sift_fp_img.png', (sift_fp_img)) #vt.imwrite(dataname + '_' + 'sift_fn_img.png', (sift_fn_img)) else: print('Drawing TP FP TN FN') fnum = fnum_gen() pnum_gen = pt.make_pnum_nextgen(4, 2) fig = pt.figure(fnum) pt.imshow(cnn_fp_img, title='CNN FP', fnum=fnum, pnum=pnum_gen()) pt.imshow(sift_fp_img, title='SIFT FP', fnum=fnum, pnum=pnum_gen()) pt.imshow(cnn_fn_img, title='CNN FN', fnum=fnum, pnum=pnum_gen()) pt.imshow(sift_fn_img, title='SIFT FN', fnum=fnum, pnum=pnum_gen()) pt.imshow(cnn_tp_img, title='CNN TP', fnum=fnum, pnum=pnum_gen()) pt.imshow(sift_tp_img, title='SIFT TP', fnum=fnum, pnum=pnum_gen()) pt.imshow(cnn_tn_img, title='CNN TN', fnum=fnum, pnum=pnum_gen()) pt.imshow(sift_tn_img, title='SIFT TN', fnum=fnum, pnum=pnum_gen()) pt.set_figtitle(dataname + ' confusions') pt.adjust_subplots(left=0, right=1.0, bottom=0., wspace=.01, hspace=.05) pt.save_figure(fig=fig, dpath=epoch_dpath, dpi=180, figsize=(9, 18)) with_patch_desc = FULL if with_patch_desc: ut.colorprint('[siam_perf] Visualize Patch Descriptors', 'white') fnum = fnum_gen() fig = pt.figure(fnum=fnum, pnum=(1, 1, 1)) num_rows = 7 pnum_gen = pt.make_pnum_nextgen(num_rows, 3) # Compare actual output descriptors for index in ut.random_indexes(len(sift_list), num_rows): vec_sift = sift_list[index] vec_cnn = network_output[index] patch = data[index] pt.imshow(patch, fnum=fnum, pnum=pnum_gen()) pt.plot_descriptor_signature(vec_cnn, 'cnn vec', fnum=fnum, pnum=pnum_gen()) pt.plot_sift_signature(vec_sift, 'sift vec', fnum=fnum, pnum=pnum_gen()) pt.set_figtitle('Patch Descriptors') pt.adjust_subplots(left=0, right=0.95, bottom=0., wspace=.1, hspace=.15) pt.save_figure(fig=fig, dpath=epoch_dpath, dpi=180, figsize=(9, 18))
def process_batch(X_train, y_train, batch_size, theano_fn, **kwargs): """ compute the loss over all training batches Jon, if you get to this before I do, please fix. -J CommandLine: python -m ibeis_cnn.batch_processing --test-process_batch Example: >>> # DISABLE_DOCTEST >>> from ibeis_cnn.batch_processing import * # NOQA >>> from ibeis_cnn import models >>> model = models.DummyModel(autoinit=True) >>> X_train, y_train = model.make_random_testdata() >>> theano_fn = create_unbuffered_iter_funcs_train2(model) >>> kwargs = {'X_is_cv2_native': False} >>> batch_size = model.batch_size >>> (loss, accu, prob_list, albl_list, pred_list, conf_list) = process_batch(X_train, y_train, batch_size, theano_fn) >>> result = str((loss, accu, prob_list, albl_list, pred_list, conf_list)) >>> print(result) Ignore: Xb, yb = batch_iter.next() assert Xb.shape == (8, 1, 4, 4) yb.shape == (8,) """ batch_output_list = [] # NOQA output_names = [op.variable.name for op in theano_fn.outputs] # NOQA albl_list = [] # [a]ugmented [l]a[b]e[l] list show = False batch_iter = batch_iterator(X_train, y_train, batch_size, **kwargs) for Xb, yb in batch_iter: # Runs a batch through the network and updates the weights. Just returns what it did batch_output = theano_fn(Xb, yb) albl_list.append(yb) batch_output_list.append(batch_output) if show: # Print the network output for the first batch print('--------------') print(ut.list_str(zip(output_names, batch_output))) print('Correct: ', yb) print('--------------') show = False # Convert to numpy array # get outputs of each type def concatenate_hack(sequence, axis=0): # Hack to fix numpy bug. concatenate should do hstacks on 0-dim arrays if len(_output_unstacked) > 0 and len(_output_unstacked[1].shape) == 0: res = np.hstack(_output_unstacked) else: res = np.concatenate(_output_unstacked, axis=axis) return res unstacked_output_gen = ([bop[count] for bop in batch_output_list] for count, name in enumerate(output_names)) stacked_output_list = [concatenate_hack(_output_unstacked, axis=-1) for _output_unstacked in unstacked_output_gen] albl_list = np.hstack(albl_list) # Calculate performance loss_index = ut.listfind(output_names, 'loss_train') if loss_index is not None: loss_list = stacked_output_list[loss_index] loss = np.mean(loss_list) pred_index = ut.listfind(output_names, 'prediction') if pred_index is not None: pred_list = stacked_output_list[pred_index] accu = np.mean(np.equal(albl_list, pred_list)) # Return return loss, accu, prob_list, albl_list, pred_list, conf_list
def process_batch(model, X, y, theano_fn, fix_output=False, buffered=False, show=False, spatial=False, showprog=True, **kwargs): """ Compute the loss over all training batches. Passes data to function that splits it into batches and appropriately preproecsses the data. Then this function sends that data to theano. Then the results are packaged up nicely before returning. CommandLine: python -m ibeis_cnn --tf process_batch --verbose python -m ibeis_cnn --tf process_batch:0 --verbose python -m ibeis_cnn --tf process_batch:1 --verbose Example0: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.batch_processing import * # NOQA >>> from ibeis_cnn import models >>> model = models.DummyModel(batch_size=128) >>> X, y = model.make_random_testdata(num=2000, seed=None) >>> model.init_arch() >>> theano_fn = model.build_predict_func() >>> kwargs = {'X_is_cv2_native': False, 'showprog': True, ... 'randomize_batch_order': True} >>> outputs_ = process_batch(model, X, y, theano_fn, **kwargs) >>> result = ut.dict_str(outputs_) >>> print(result) Example0: >>> # ENABLE_DOCTEST >>> from ibeis_cnn.batch_processing import * # NOQA >>> from ibeis_cnn import models >>> model = models.SiameseL2(batch_size=128, data_shape=(32, 32, 1), ... strict_batch_size=True) >>> X, y = model.make_random_testdata(num=2000, seed=None) >>> model.init_arch() >>> theano_fn = model.build_predict_func() >>> kwargs = {'X_is_cv2_native': False, 'showprog': True, ... 'randomize_batch_order': True} >>> outputs_ = process_batch(model, X, y, theano_fn, **kwargs) >>> result = ut.dict_str(outputs_) >>> print(result) Ignore: Xb, yb = batch_iter.next() assert Xb.shape == (8, 1, 4, 4) yb.shape == (8,) Ignore: X, y = model.make_random_testdata(num=2000, seed=None) kwargs = {'X_is_cv2_native': False, 'showprog': True, 'randomize_batch_order': True, 'time_thresh': .5, } print('Testing Unbuffered') batch_iter = batch_iterator(model, X, y, lbl=theano_fn.name, **kwargs) for Xb, yb in ut.ProgressIter(batch_iter, lbl=':EXEC FG'): [ut.is_prime(346373) for _ in range(2)] # Notice how the progress iters are not interlaced like # they are in the unbuffered version import sys sys.stdout.flush() print('Testing Buffered') sys.stdout.flush() batch_iter2 = batch_iterator(model, X, y, lbl=theano_fn.name, **kwargs) batch_iter2 = ut.buffered_generator(batch_iter2, buffer_size=4) print('Iterating') for Xb, yb in ut.ProgressIter(batch_iter2, lbl=':EXEC FG'): [ut.is_prime(346373) for _ in range(2)] """ import vtool as vt batch_output_list = [] output_names = [ str(outexpr.variable) if outexpr.variable.name is None else outexpr.variable.name for outexpr in theano_fn.outputs ] # augmented label list batch_target_list = [] show = VERBOSE_BATCH or show # Break data into generated batches # generated data with explicit iteration batch_iter = batch_iterator(model, X, y, **kwargs) if buffered: batch_iter = ut.buffered_generator(batch_iter) if showprog: bs = VERBOSE_BATCH < 1 num_batches = (X.shape[0] + model.batch_size - 1) // model.batch_size # progress iterator should be outside of this function batch_iter = ut.ProgressIter(batch_iter, nTotal=num_batches, lbl=theano_fn.name, freq=10, bs=bs, adjust=True) if y is None: # Labels are not known, only one argument for Xb, yb in batch_iter: pass batch_output = theano_fn(Xb) batch_output_list.append(batch_output) else: # TODO: sliced batches for Xb, yb in batch_iter: # Runs a batch through the network and updates the weights. Just # returns what it did batch_output = theano_fn(Xb, yb) batch_output_list.append(batch_output) batch_target_list.append(yb) if show: # Print the network output for the first batch print('--------------') print(ut.list_str(zip(output_names, batch_output))) print('Correct: ', yb) print('--------------') show = False # get outputs of each type unstacked_output_gen = ([bop[count] for bop in batch_output_list] for count, name in enumerate(output_names)) if spatial: unstacked_output_gen = list(unstacked_output_gen) stacked_output_list = [[] for _ in range(len(unstacked_output_gen))] for index, output in enumerate(unstacked_output_gen): output = np.vstack(output) stacked_output_list[index] = output else: stacked_output_list = [ vt.safe_cat(_output_unstacked, axis=0) # concatenate_hack(_output_unstacked, axis=0) for _output_unstacked in unstacked_output_gen ] outputs_ = dict(zip(output_names, stacked_output_list)) if y is not None: auglbl_list = np.hstack(batch_target_list) outputs_['auglbl_list'] = auglbl_list if fix_output: # batch iteration may wrap-around returned data. slice off the padding num_inputs = X.shape[0] / model.data_per_label_input num_outputs = num_inputs * model.data_per_label_output for key in six.iterkeys(outputs_): outputs_[key] = outputs_[key][0:num_outputs] encoder = getattr(model, 'encoder', None) if encoder is not None and 'predictions' in outputs_: pred = outputs_['predictions'] outputs_['labeled_predictions'] = encoder.inverse_transform(pred) return outputs_
def build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list): """ helper these list comprehensions replace the prevous for loop they still need to be optimized a little bit (and made clearer) can probably unnest the list comprehensions as well """ """ IGNORE Legacy:: def old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list): fm_nestlist_ = [] fs_nestlist_ = [] daid_nestlist_ = [] for scores, qfxs, dfxs, daids in zip(sparse_list, qfxs_list, dfxs_list, daids_list): for rx, cx, score in zip(scores.row, scores.col, scores.data): _fm = tuple(product(qfxs[rx], dfxs[cx])) _fs = [score / len(_fm)] * len(_fm) _daid = [daids[cx]] * len(_fm) fm_nestlist_.append(_fm) fs_nestlist_.append(_fs) daid_nestlist_.append(_daid) return fm_nestlist_, fs_nestlist_, daid_nestlist_ oldtup_ = old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) fm_nestlist_, fs_nestlist_, daid_nestlist_ = oldtup_ newtup_ = build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) fm_nestlist, fs_nestlist, daid_nestlist = newtup_ assert fm_nestlist == fm_nestlist_ assert fs_nestlist == fs_nestlist_ assert daid_nestlist == daid_nestlist_ 47ms %timeit build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) 59ms %timeit old_build_correspondences(sparse_list, qfxs_list, dfxs_list, daids_list) IGNORE """ # FIXME: rewrite double comprehension as a flat comprehension # Build nested feature matches (a single match might have many members) fm_nestlist = [ tuple(product(qfxs[rx], dfxs[cx])) for scores, qfxs, dfxs in zip(sparse_list, qfxs_list, dfxs_list) for rx, cx in zip(scores.row, scores.col) ] nFm_list = [len(fm) for fm in fm_nestlist] #fs_unsplit = (score # for scores in sparse_list # for score in scores.data) #daid_unsplit = (daids[cx] # for scores, daids in zip(sparse_list, daids_list) # for cx in scores.col) # Build nested feature scores fs_unsplit = utool.iflatten((scores.data for scores in sparse_list)) # Build nested feature matches (a single match might have many members) daid_unsplit = utool.iflatten( (daids.take(scores.col) for scores, daids in zip(sparse_list, daids_list))) # Expand feature scores and daids splitting scores amongst match members fs_nestlist = [[score / nFm] * nFm for score, nFm in zip(fs_unsplit, nFm_list)] daid_nestlist = [[daid] * nFm for daid, nFm in zip(daid_unsplit, nFm_list)] if DEBUG_SMK: assert len(fm_nestlist) == len(fs_nestlist), 'inconsistent len' assert len(fm_nestlist) == len(nFm_list), 'inconsistent len' assert len(daid_nestlist) == len(fs_nestlist), 'inconsistent len' min_ = min(2, len(nFm_list)) max_ = min(15, len(nFm_list)) print('nFm_list[_min:_max] = ' + utool.list_str(nFm_list[min_:max_])) print('fm_nestlist[_min:_max] = ' + utool.list_str(fm_nestlist[min_:max_])) print('fs_nestlist[_min:_max] = ' + utool.list_str(fs_nestlist[min_:max_])) print('daid_nestlist[_min:_max] = ' + utool.list_str(daid_nestlist[min_:max_])) for fm_, fs_, daid_ in zip(fm_nestlist, fs_nestlist, daid_nestlist): assert len(fm_) == len(fs_), 'inconsistent len' assert len(fm_) == len(daid_), 'inconsistent len' print('[smk_core] checked build_chipmatch correspondence ...ok') return fm_nestlist, fs_nestlist, daid_nestlist
def run_devcmds(ibs, qaid_list, daid_list, acfg=None): """ This function runs tests passed in with the -t flag """ print('\n') #print('[dev] run_devcmds') print('==========================') print('[DEV] RUN EXPERIMENTS %s' % ibs.get_dbname()) print('==========================') input_test_list = params.args.tests[:] print('input_test_list = %s' % (ut.list_str(input_test_list), )) # fnum = 1 valid_test_list = [] # build list for printing in case of failure valid_test_helpstr_list = [] # for printing def mark_test_handled(testname): input_test_list.remove(testname) def intest(*args, **kwargs): helpstr = kwargs.get('help', '') valid_test_helpstr_list.append(' -t ' + ', '.join(args) + helpstr) for testname in args: valid_test_list.append(testname) ret = testname in input_test_list ret2 = testname in params.unknown # Let unparsed args count towards tests if ret or ret2: if ret: mark_test_handled(testname) else: ret = ret2 print('\n+===================') print(' [dev] running testname = %s' % (args, )) print('+-------------------\n') return ret return False valid_test_helpstr_list.append(' # --- Simple Tests ---') # Explicit (simple) test functions if intest('export'): export(ibs) if intest('dbinfo'): dbinfo.get_dbinfo(ibs) if intest('headers', 'schema'): ibs.db.print_schema() if intest('info'): print(ibs.get_infostr()) if intest('printcfg'): printcfg(ibs) if intest('tables'): ibs.print_tables() if intest('imgtbl'): ibs.print_image_table() valid_test_helpstr_list.append(' # --- Decor Tests ---') locals_ = locals() # Implicit (decorated) test functions for (func_aliases, func) in DEVCMD_FUNCTIONS: if intest(*func_aliases): funcname = get_funcname(func) #with utool.Indenter('[dev.' + funcname + ']'): with utool.Timer(funcname): #print('[dev] qid_list=%r' % (qaid_list,)) # FIXME: , daid_list if len(ut.get_func_argspec(func).args) == 0: ret = func() else: ret = func(ibs, qaid_list, daid_list) # Add variables returned by the function to the # "local scope" (the exec scop) if hasattr(ret, 'items'): for key, val in ret.items(): if utool.is_valid_varname(key): locals_[key] = val valid_test_helpstr_list.append(' # --- Config Tests ---') # ------ # RUNS EXPERIMENT HARNESS OVER VALID TESTNAMES SPECIFIED WITH -t # ------ # Config driven test functions # Allow any testcfg to be in tests like: vsone_1 or vsmany_3 test_cfg_name_list = [] for test_cfg_name in experiment_configs.TEST_NAMES: if intest(test_cfg_name): test_cfg_name_list.append(test_cfg_name) # Hack to allow for very customized harness tests for testname in input_test_list[:]: if testname.startswith('custom:'): test_cfg_name_list.append(testname) mark_test_handled(testname) if len(test_cfg_name_list): fnum = pt.next_fnum() # Run Experiments # backwards compatibility yo acfgstr_name_list = {'OVERRIDE_HACK': (qaid_list, daid_list)} assert False, 'This way of running tests no longer works. It may be fixed in the future' #acfg harness.test_configurations(ibs, acfgstr_name_list, test_cfg_name_list) valid_test_helpstr_list.append(' # --- Help ---') if intest('help'): print('valid tests are:') print('\n'.join(valid_test_helpstr_list)) return locals_ if len(input_test_list) > 0: print('valid tests are: \n') print('\n'.join(valid_test_list)) raise Exception('Unknown tests: %r ' % input_test_list) return locals_
def run_devcmds(ibs, qaid_list, daid_list, acfg=None): """ This function runs tests passed in with the -t flag """ print('\n') #print('[dev] run_devcmds') print('==========================') print('[DEV] RUN EXPERIMENTS %s' % ibs.get_dbname()) print('==========================') input_test_list = params.args.tests[:] print('input_test_list = %s' % (ut.list_str(input_test_list),)) # fnum = 1 valid_test_list = [] # build list for printing in case of failure valid_test_helpstr_list = [] # for printing def mark_test_handled(testname): input_test_list.remove(testname) def intest(*args, **kwargs): helpstr = kwargs.get('help', '') valid_test_helpstr_list.append(' -t ' + ', '.join(args) + helpstr) for testname in args: valid_test_list.append(testname) ret = testname in input_test_list ret2 = testname in params.unknown # Let unparsed args count towards tests if ret or ret2: if ret: mark_test_handled(testname) else: ret = ret2 print('\n+===================') print(' [dev] running testname = %s' % (args,)) print('+-------------------\n') return ret return False valid_test_helpstr_list.append(' # --- Simple Tests ---') # Explicit (simple) test functions if intest('export'): export(ibs) if intest('dbinfo'): dbinfo.get_dbinfo(ibs) if intest('headers', 'schema'): ibs.db.print_schema() if intest('info'): print(ibs.get_infostr()) if intest('printcfg'): printcfg(ibs) if intest('tables'): ibs.print_tables() if intest('imgtbl'): ibs.print_image_table() valid_test_helpstr_list.append(' # --- Decor Tests ---') locals_ = locals() # Implicit (decorated) test functions for (func_aliases, func) in DEVCMD_FUNCTIONS: if intest(*func_aliases): funcname = get_funcname(func) #with utool.Indenter('[dev.' + funcname + ']'): with utool.Timer(funcname): #print('[dev] qid_list=%r' % (qaid_list,)) # FIXME: , daid_list if len(ut.get_func_argspec(func).args) == 0: ret = func() else: ret = func(ibs, qaid_list, daid_list) # Add variables returned by the function to the # "local scope" (the exec scop) if hasattr(ret, 'items'): for key, val in ret.items(): if utool.is_valid_varname(key): locals_[key] = val valid_test_helpstr_list.append(' # --- Config Tests ---') # ------ # RUNS EXPERIMENT HARNESS OVER VALID TESTNAMES SPECIFIED WITH -t # ------ # Config driven test functions # Allow any testcfg to be in tests like: vsone_1 or vsmany_3 test_cfg_name_list = [] for test_cfg_name in experiment_configs.TEST_NAMES: if intest(test_cfg_name): test_cfg_name_list.append(test_cfg_name) # Hack to allow for very customized harness tests for testname in input_test_list[:]: if testname.startswith('custom:'): test_cfg_name_list.append(testname) mark_test_handled(testname) if len(test_cfg_name_list): fnum = pt.next_fnum() # Run Experiments # backwards compatibility yo acfgstr_name_list = {'OVERRIDE_HACK': (qaid_list, daid_list)} assert False, 'This way of running tests no longer works. It may be fixed in the future' #acfg harness.test_configurations(ibs, acfgstr_name_list, test_cfg_name_list) valid_test_helpstr_list.append(' # --- Help ---') if intest('help'): print('valid tests are:') print('\n'.join(valid_test_helpstr_list)) return locals_ if len(input_test_list) > 0: print('valid tests are: \n') print('\n'.join(valid_test_list)) raise Exception('Unknown tests: %r ' % input_test_list) return locals_
def parse_latex_comments_for_commmands(): r""" CommandLine: python -m ibeis.scripts.gen_cand_expts --exec-parse_latex_comments_for_commmands Example: >>> # SCRIPT >>> from ibeis.scripts.gen_cand_expts import * # NOQA >>> parse_latex_comments_for_commmands() """ fname = ut.get_argval('--fname', type_=str, default='figdefexpt.tex') text = ut.read_from(ut.truepath('~/latex/crall-candidacy-2015/' + fname)) #text = ut.read_from(ut.truepath('~/latex/crall-candidacy-2015/figdefindiv.tex')) lines = text.split('\n') cmd_list = [''] in_comment = True for line in lines: if line.startswith('% ---'): # Keep separators toadd = line.replace('%', '#') if not (len(cmd_list) > 1 and cmd_list[-1].startswith('# ---')): cmd_list[-1] += (toadd) else: cmd_list.append(toadd) cmd_list.append('') if line.strip().startswith(r'\begin{comment}'): in_comment = True continue if in_comment: line = line.strip() if line == '' or line.startswith('#') or line.startswith('%'): in_comment = False else: cmd_list[-1] = cmd_list[-1] + line if not line.strip().endswith('\\'): cmd_list[-1] = cmd_list[-1] + ' $@' #cmd_list.append('') #cmd_list.append('#--') cmd_list.append('') in_comment = False else: cmd_list[-1] = cmd_list[-1] + '\n' cmd_list = [ cmd.replace('--render', '').replace('--diskshow', '') for cmd in cmd_list ] # formatting cmd_list2 = [] for cmd in cmd_list: #cmd = cmd.replace(' -t ', ' \\\n -t ') #cmd = cmd.replace('--db', '\\\n --db') #cmd = cmd.replace('python -m ibeis.dev', './dev.py') cmd = cmd.replace('python -m ibeis.dev -e', 'ibeis -e') cmd_list2.append(cmd) cmd_list = cmd_list2 print('cmd_list = %s' % (ut.list_str(cmd_list), )) from os.path import splitext script_fname = 'regen_' + splitext(fname)[0] + '.sh' fname, script, line_list = write_script_lines(cmd_list, script_fname)
def parse_shark_tags(orig_fname_list): import re invalid_tag_patterns = [ re.escape('-'), re.escape('(') + '?\\d*' + re.escape(')') + '?', '\\d+-\\d+-\\d+', '\\d+,', '\\d+', 'vi*', 'i*v', 'i+', '\\d+th', '\\d+nd', '\\d+rd', 'remant', 'timnfe', 't', 'e', 'sjl', 'disc', 'dec', 'road', 'easter', 'western', 'west', 'tn', '\\d*ap', 'whaleshark\\d*', 'shark\\d*', 'whale\\d*', 'whalesharking', 'sharking', 'whalesharks', 'whales', 'picture', 'australien', 'australia', 'nick', 'tim\\d*', 'imageset', 'holiday', 'visit', 'tour', 'trip', 'pec', 'sv', 'a', 'b', 'gender', 'sex', 'img', 'image', 'pic', 'pics', 'leith', 'trips', 'kings', 'photo', 'video', 'media', 'fix', 'feeding', 'nrd', 'nd', 'gen', 'wa', 'nmp', 'bo', 'kd', 'ow', 'ne', 'dsc', 'nwd', 'mg', 'w', 'mai', 'blue', 'stumpy', 'oea', 'cbe', 'edc', 'knrt', 'tiws2', 'ando', 'adv', 'str', 'adventure', 'camera', 'tag', 'id', 'of', 'and', 'tagged', 'from', 'day', '\\d*april', '\\d*may', '\\d*july', '\\d*june', 'ningaloo', 'ningblue\\d*', 'kooling', ] valid_tag_level_set = [ ['view-left', 'left', 'lhs', 'l', 'leftside'], ['view-right', 'right', 'rhs', 'r', 'rightside'], ['view-back', 'back'], ['view-top', 'top'], ['sex-male', 'male', 'm', 'sexm'], ['sex-female', 'female', 'f'], ['sex-unknown', 'unknown', 'u'], ['part-tail', 'tail'], ['part-flank', 'side', 'flank'], ['part-head', 'head'], ['part-pectoral', 'pectoral', 'pec'], ['part-dorsal', 'dorsal', 'dorsals'], ['part-claspers', 'claspers', 'clasper'], ['part-fin', 'fin'], ['cropped', 'crop'], ['scar', 'scar2'], ['notch'], ['small'], ['bite'], ['cam-slr2', 'slr2'], #['cam-5m', '5m'] ['5m'], ['7m'], ['4m'], ['copy'], ['qual-resize'], ['qual-stretched'], ] def apply_enum_regex(pat_list): enum_endings = [ '[a-g]', '\\d*', 'i*', ] expanded_pats = ut.flatten([ [pat + end for end in enum_endings] for pat in pat_list ]) return expanded_pats def apply_regex_endings(pat_list): return [p + '$' for p in pat_list] tag_alias_map = {} for level_set in valid_tag_level_set: main_key = level_set[0] for key in level_set: tag_alias_map[key] = main_key inverse_alias_map = {} for level_set in valid_tag_level_set: inverse_alias_map[level_set[0]] = level_set regex_alias_map = { 'view-left': apply_regex_endings(apply_enum_regex(inverse_alias_map['view-left'])), 'view-right': apply_regex_endings(apply_enum_regex(inverse_alias_map['view-right'])), } valid_tags = list(inverse_alias_map.keys()) invalid_tag_patterns = apply_regex_endings(invalid_tag_patterns) def parse_all_fname_tags(fname): _tags = [splitext(fname)[0]] _tags = ut.flatten([t.split('_') for t in _tags]) _tags = ut.flatten([t.split('.') for t in _tags]) _tags = [t.lower() for t in _tags] _tags = [tag_alias_map.get(t, t) for t in _tags] for key, vals in regex_alias_map.items(): pat = ut.regex_or(vals) _tags = [key if re.match(pat, t) else t for t in _tags] pat = ut.regex_or(invalid_tag_patterns) _tags = [t for t in _tags if not re.match(pat, t)] _tags = ut.unique_ordered(_tags) return _tags all_img_tag_list = list(map(parse_all_fname_tags, orig_fname_list)) known_img_tag_list = [list(set(tags).intersection(set(valid_tags))) for tags in all_img_tag_list] if False: # Help figure out which tags are important _parsed_tags = ut.flatten(all_img_tag_list) taghist = ut.dict_hist(_parsed_tags) taghist = {key: val for key, val in taghist.items() if val > 1} unknown_taghist = sorted([ (val, key) for key, val in taghist.items() if key not in valid_tags ])[::-1] known_taghist = sorted([ (val, key) for key, val in taghist.items() if key in valid_tags ])[::-1] print('Known') print(ut.list_str(known_taghist[0:100])) print('Unknown') print(ut.list_str(unknown_taghist[0:100])) print(ut.dict_str( ut.dict_hist(ut.flatten(known_img_tag_list)), key_order_metric='val' )) return known_img_tag_list
def fix_duplicates(drive): r""" for every duplicate file passing a (eg avi) filter, remove the file that is in the smallest directory. On a tie use the smallest dpath. This will filter all duplicate files in a folder into a single folder. but... need to look at non-duplicates in that folder and decide if they should be moved as well. So, should trigger on folders that have at least 50% duplicate. Might not want to move curated folders. Example: cd ~/local/scripts >>> from register_files import * # NOQA >>> dpaths = ut.get_argval('--drives', type_=list, default=['E:/'])#'D:/', 'E:/', 'F:/']) >>> drives = [Drive(root_dpath) for root_dpath in dpaths] >>> E = drive = drives[0] >>> #D, E, F = drives """ print('Fixing Duplicates in %r' % (drive,)) list_ = drive.fpath_hashX_list multiindex_dict_ = build_multindex(list_) duplicate_hashes = [ key for key, val in six.iteritems(multiindex_dict_) if len(val) > 1 ] duplicate_idxs = ut.dict_take(multiindex_dict_, duplicate_hashes) unflat_fpaths = ut.list_unflat_take(drive.fpath_list, duplicate_idxs) # Check if any dups have been removed still_exists = ut.unflat_map(exists, unflat_fpaths) unflat_idxs2 = ut.zipcompress(duplicate_idxs, still_exists) duplicate_idxs = [idxs for idxs in unflat_idxs2 if len(idxs) > 1] # Look at duplicate files unflat_fpaths = ut.list_unflat_take(drive.fpath_list, duplicate_idxs) unflat_sizes = ut.list_unflat_take(drive.fpath_bytes_list, duplicate_idxs) # Find highly coupled directories if True: coupled_dirs = [] for fpaths in unflat_fpaths: #basedir = ut.longest_existing_path(commonprefix(fpaths)) dirs = sorted(list(map(dirname, fpaths))) _list = list(range(len(dirs))) idxs = ut.upper_diag_self_prodx(_list) coupled_dirs.extend(list(map(tuple, ut.list_unflat_take(dirs, idxs)))) hist_ = ut.dict_hist(coupled_dirs) coupled_idxs = ut.list_argsort(hist_.values())[::-1] most_coupled = ut.take(list(hist_.keys()), coupled_idxs[0:100]) print('Coupled fpaths: ' + ut.list_str(most_coupled, nl=True)) print('%d unique files are duplicated' % (len(unflat_sizes),)) #print('Duplicate sizes: ' + ut.list_str(unflat_sizes[0:10], nl=True)) #print('Duplicate fpaths: ' + ut.list_str(unflat_fpaths[0:10], nl=True)) #print('Duplicate fpaths: ' + ut.list_str(unflat_fpaths[0::5], nl=True)) print('Duplicate fpaths: ' + ut.list_str(unflat_fpaths, nl=True)) # Find duplicate directories dpath_list = list(drive.dpath_to_fidx.keys()) fidxs_list = ut.dict_take(drive.dpath_to_fidx, drive.dpath_list) #exists_list = list(map(exists, drive.fpath_list)) #unflat_exists = ut.list_unflat_take(exists_list, fidxs_list) fname_registry = [basename(fpath) for fpath in drive.fpath_list] unflat_fnames = ut.list_unflat_take(fname_registry, fidxs_list) def unsorted_list_hash(list_): return ut.hashstr27(str(sorted(list_))) unflat_fname_sets = list(map(unsorted_list_hash, ut.ProgIter(unflat_fnames, freq=10000))) fname_based_duplicate_dpaths = [] multiindex_dict2_ = build_multindex(unflat_fname_sets) fname_based_duplicate_hashes = [key for key, val in multiindex_dict2_.items() if len(val) > 1] print('#fname_based_duplicate_dpaths = %r' % (len(fname_based_duplicate_hashes),)) fname_based_duplicate_didxs = ut.dict_take(multiindex_dict2_, fname_based_duplicate_hashes) fname_based_duplicate_dpaths = ut.list_unflat_take(dpath_list, fname_based_duplicate_didxs) print(ut.repr3(fname_based_duplicate_dpaths[0:10]))
def make_individual_latex_figures(ibs, fpaths_list, flat_case_labels, cfgx2_shortlbl, case_figdir, analysis_fpath_list): # HACK MAKE LATEX CONVINENCE STUFF #print('LATEX HACK') if len(fpaths_list) == 0: print('nothing to render') return RENDER = ut.get_argflag('--render') DUMP_FIGDEF = ut.get_argflag(('--figdump', '--dump-figdef', '--figdef')) if not (DUMP_FIGDEF or RENDER): # HACK return latex_code_blocks = [] latex_block_keys = [] caption_prefix = ut.get_argval('--cappref', type_=str, default='') caption_suffix = ut.get_argval('--capsuf', type_=str, default='') cmdaug = ut.get_argval('--cmdaug', type_=str, default='custom') selected = None for case_idx, (fpaths, labels) in enumerate(zip(fpaths_list, flat_case_labels)): if labels is None: labels = [cmdaug] if len(fpaths) < 4: nCols = len(fpaths) else: nCols = 2 _cmdname = ibs.get_dbname() + ' Case ' + ' '.join(labels) + '_' + str( case_idx) #print('_cmdname = %r' % (_cmdname,)) cmdname = ut.latex_sanitize_command_name(_cmdname) label_str = cmdname if len(caption_prefix) == 0: caption_str = ut.escape_latex( 'Casetags: ' + ut.list_str(labels, nl=False, strvals=True) + ', db=' + ibs.get_dbname() + '. ') else: caption_str = '' use_sublbls = len(cfgx2_shortlbl) > 1 if use_sublbls: caption_str += ut.escape_latex( 'Each figure shows a different configuration: ') sublbls = [ '(' + chr(97 + count) + ') ' for count in range(len(cfgx2_shortlbl)) ] else: #caption_str += ut.escape_latex('This figure depicts correct and #incorrect matches from configuration: ') sublbls = [''] * len(cfgx2_shortlbl) def wrap_tt(text): return r'{\tt ' + text + '}' _shortlbls = cfgx2_shortlbl _shortlbls = list(map(ut.escape_latex, _shortlbls)) # Adjust spacing for breaks #tex_small_space = r'' tex_small_space = r'\hspace{0pt}' # Remove query specific config flags in individual results _shortlbls = [ re.sub('\\bq[^,]*,?', '', shortlbl) for shortlbl in _shortlbls ] # Let config strings be broken over newlines _shortlbls = [ re.sub('\\+', tex_small_space + '+' + tex_small_space, shortlbl) for shortlbl in _shortlbls ] _shortlbls = [ re.sub(', *', ',' + tex_small_space, shortlbl) for shortlbl in _shortlbls ] _shortlbls = list(map(wrap_tt, _shortlbls)) cfgx2_texshortlbl = [ '\n ' + lbl + shortlbl for lbl, shortlbl in zip(sublbls, _shortlbls) ] caption_str += ut.conj_phrase(cfgx2_texshortlbl, 'and') + '.\n ' caption_str = '\n ' + caption_prefix + caption_str + caption_suffix caption_str = caption_str.rstrip() figure_str = ut.get_latex_figure_str(fpaths, nCols=nCols, label_str=label_str, caption_str=caption_str, use_sublbls=None, use_frame=True) latex_block = ut.latex_newcommand(cmdname, figure_str) latex_block = '\n%----------\n' + latex_block latex_code_blocks.append(latex_block) latex_block_keys.append(cmdname) # HACK remove_fpath = ut.truepath('~/latex/crall-candidacy-2015') + '/' latex_fpath = join(case_figdir, 'latex_cases.tex') if selected is not None: selected_keys = selected else: selected_keys = latex_block_keys selected_blocks = ut.dict_take( dict(zip(latex_block_keys, latex_code_blocks)), selected_keys) figdef_block = '\n'.join(selected_blocks) figcmd_block = '\n'.join(['\\' + key for key in latex_block_keys]) selected_block = figdef_block + '\n\n' + figcmd_block # HACK: need full paths to render selected_block_renderable = selected_block selected_block = selected_block.replace(remove_fpath, '') if RENDER: ut.render_latex_text(selected_block_renderable) if DUMP_FIGDEF: ut.writeto(latex_fpath, selected_block) #if NOT DUMP AND NOT RENDER: # print('STANDARD LATEX RESULTS') # cmdname = ibs.get_dbname() + 'Results' # latex_block = ut.get_latex_figure_str2(analysis_fpath_list, cmdname, nCols=1) # ut.print_code(latex_block, 'latex') if DUMP_FIGDEF or RENDER: ut.print_code(selected_block, 'latex')
def fix_empty_dirs(drive): """ # --- FIND EMPTY DIRECTORIES --- """ print('Fixing Empty Dirs in %r' % (drive,)) fidxs_list = ut.dict_take(drive.dpath_to_fidx, drive.dpath_list) isempty_flags = [len(fidxs) == 0 for fidxs in fidxs_list] empty_dpaths = ut.compress(drive.dpath_list, isempty_flags) def is_cplat_link(path_): try: if islink(path_): return True os.listdir(d) return False except SystemErrors: return True valid_flags = [not is_cplat_link(d) for d in empty_dpaths] if not all(valid_flags): print('Filtered windows links %r / %r' % ( len(empty_dpaths) - sum(valid_flags), len(empty_dpaths))) #print(ut.list_str(empty_dpaths[0:10])) empty_dpaths = ut.compress(empty_dpaths, valid_flags) print('Found %r / %r empty_dpaths' % (len(empty_dpaths), len(drive.dpath_list))) print(ut.list_str(empty_dpaths[0:10])) # Ensure actually still empty current_contents = [ut.glob(d, with_dirs=False) for d in ut.ProgIter(empty_dpaths, 'checking empty status')] current_lens = list(map(len, current_contents)) assert not any(current_lens), 'some dirs are not empty' # n ** 2 check to get only the base directories isbase_dir = [ not any([d.startswith(dpath_) and d != dpath_ for dpath_ in empty_dpaths]) for d in ut.ProgIter(empty_dpaths, 'finding base dirs') ] base_empty_dirs = ut.compress(empty_dpaths, isbase_dir) def list_only_files(dpath): # glob is too slow for root, dirs, fpaths in os.walk(dpath): for fpath in fpaths: yield fpath base_current_contents = [ list(list_only_files(d)) for d in ut.ProgIter(base_empty_dirs, 'checking emptyness', freq=10)] is_actually_empty = [len(fs) == 0 for fs in base_current_contents] not_really_empty = ut.compress(base_empty_dirs, ut.not_list(is_actually_empty)) print('%d dirs are not actually empty' % (len(not_really_empty),)) print('not_really_empty = %s' % (ut.list_str(not_really_empty[0:10]),)) truly_empty_dirs = ut.compress(base_empty_dirs, is_actually_empty) def list_all(dpath): # glob is too slow for root, dirs, fpaths in os.walk(dpath): for dir_ in dirs: yield dir_ for fpath in fpaths: yield fpath exclude_base_dirs = [join(drive.root_dpath, 'AppData')] exclude_end_dirs = ['__pycache__'] truly_empty_dirs1 = truly_empty_dirs for ed in exclude_base_dirs: truly_empty_dirs1 = [ d for d in truly_empty_dirs1 if ( not any(d.startswith(ed) for ed in exclude_base_dirs) and not any(d.endswith(ed) for ed in exclude_end_dirs) ) ] # Ensure actually still empty (with recursive checks for hidden files) print('truly_empty_dirs1[::5] = %s' % ( ut.list_str(truly_empty_dirs1[0::5], strvals=True),)) #print('truly_empty_dirs1 = %s' % (ut.list_str(truly_empty_dirs1, strvals=True),)) if not dryrun: # FIX PART #from os.path import normpath #for d in ut.ProgIter(truly_empty_dirs): # break # if ut.WIN32: # # http://www.sevenforums.com/system-security/53095-file-folder-read-only-attribute-wont-disable.html # ut.cmd('attrib', '-r', '-s', normpath(d), verbose=False) #x = ut.remove_fpaths(truly_empty_dirs, strict=False) print('Deleting %d truly_empty_dirs1' % (len(truly_empty_dirs1),)) for d in ut.ProgIter(truly_empty_dirs1, 'DELETE empty dirs', freq=1000): # NOQA ut.delete(d, quiet=True) if ut.WIN32 and False: # remove file that failed removing flags = list(map(exists, truly_empty_dirs1)) truly_empty_dirs1 = ut.compress(truly_empty_dirs1, flags) for d in ut.ProgIter(truly_empty_dirs1, 'rming', freq=1000): ut.cmd('rmdir', d)