def get_match_thumbtup(ibs, qaid2_cm, qaids, daids, index, qreq_=None, thumbsize=(128, 128), match_thumbtup_cache={}): daid = daids[index] qaid = qaids[index] cm = qaid2_cm[qaid] assert cm.qaid == qaid, 'aids do not aggree' OLD = False if OLD: fpath = ensure_match_img(ibs, cm, daid, qreq_=qreq_, match_thumbtup_cache=match_thumbtup_cache) if isinstance(thumbsize, int): thumbsize = (thumbsize, thumbsize) thumbtup = (ut.augpath(fpath, 'thumb_%d,%d' % thumbsize), fpath, thumbsize, [], []) return thumbtup else: # Hacky new way of drawing fpath, func, func2 = make_ensure_match_img_nosql_func(qreq_, cm, daid) #match_thumbdir = ibs.get_match_thumbdir() #match_thumb_fname = get_match_thumb_fname(cm, daid, qreq_) #fpath = ut.unixjoin(match_thumbdir, match_thumb_fname) thumbdat = { 'fpath': fpath, 'thread_func': func, 'main_func': func2, #'args': (ibs, cm, daid), #'kwargs': dict(qreq_=qreq_, # match_thumbtup_cache=match_thumbtup_cache) } return thumbdat
def outline(): """ ./texfix.py --fpaths chapter4-application.tex --outline --asmarkdown --numlines=999 -w --ignoreinputstartswith=def,Crall,header,colordef,figdef """ fpaths = testdata_fpaths() print('fpaths = %r' % (fpaths, )) for fpath in fpaths: text = ut.readfrom(fpath) root = latex_parser.LatexDocPart.parse_text(text, debug=None) # HACK new_text = '\n'.join(root.reformat_blocks(debug=None)) # remove trailing spaces new_text = re.sub(' *$', '', new_text, flags=re.MULTILINE) # remove double newlines new_text = re.sub('(\n *)+\n+', '\n\n', new_text, flags=re.MULTILINE) document = root.find_descendant_type('document') #document = root.find_descendant_type('section', pat='Identification') print('document = %r' % (document, )) if document is not None: root = document sectionpat = ut.get_argval('--section', default=None) if sectionpat is not None: root = root.find_descendant_type('section', pat=sectionpat) print('root = %r' % (root, )) if root is None: # import utool # utool.embed() raise Exception('section %r does not exist' % (sectionpat)) #print(root.get_debug_tree_text()) #ut.colorprint(root.summary_str(outline=True), 'yellow') print('---outline---') outline = True # outline = False outline_text = root.summary_str(outline=outline, highlight=False) summary = root.summary_str(outline=outline, highlight=True) if not ut.get_argflag('-w'): print(summary) print('---/outline---') if root._config['asmarkdown']: codetype = 'markdown' newext = '.md' else: codetype = 'latex' newext = None ut.dump_autogen_code(ut.augpath(fpath, augpref='outline_', newext=newext), outline_text, codetype=codetype, fullprint=False)
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath, )) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath, )) gs_exe = find_ghostscript_exe() cmd_list = (gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath) ut.cmd(*cmd_list) return output_pdf_fpath
def resize_imagelist_to_sqrtarea(gpath_list, new_gpath_list=None, sqrt_area=800, output_dir=None, checkexists=True, **kwargs): """ Resizes images and yeilds results asynchronously """ import vtool as vt target_area = sqrt_area ** 2 # Read image sizes gsize_list = [vt.open_image_size(gpath) for gpath in gpath_list] # Compute new sizes which preserve aspect ratio newsize_list = [vt.ScaleStrat.area(target_area, wh) for wh in gsize_list] if new_gpath_list is None: # Compute names for the new images if not given if output_dir is None: # Create an output directory if not specified output_dir = 'resized_sqrtarea%r' % sqrt_area ut.ensuredir(output_dir) size_suffixs = ['_' + repr(newsize).replace(' ', '') for newsize in newsize_list] from os.path import basename old_gnames = [basename(p) for p in gpath_list] new_gname_list = [ut.augpath(p, suffix=s) for p, s in zip(old_gnames, size_suffixs)] new_gpath_list = [join(output_dir, gname) for gname in new_gname_list] new_gpath_list = list(map(ut.unixpath, new_gpath_list)) assert len(new_gpath_list) == len(gpath_list), 'unequal len' assert len(newsize_list) == len(gpath_list), 'unequal len' # Evaluate generator if checkexists: exists_list = list(map(exists, new_gpath_list)) gpath_list_ = ut.filterfalse_items(gpath_list, exists_list) new_gpath_list_ = ut.filterfalse_items(new_gpath_list, exists_list) newsize_list_ = ut.filterfalse_items(newsize_list, exists_list) else: gpath_list_ = gpath_list new_gpath_list_ = new_gpath_list newsize_list_ = newsize_list generator = resize_imagelist_generator(gpath_list_, new_gpath_list_, newsize_list_, **kwargs) for res in generator: pass #return [res for res in generator] return new_gpath_list
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath,)) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath,)) gs_exe = find_ghostscript_exe() cmd_list = ( gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath ) ut.cmd(*cmd_list) return output_pdf_fpath
def save_pretrained_weights_slice(pretrained_weights, weights_path, slice_=slice(None)): """ Used to save a slice of pretrained weights. The doctest will publish a new set of weights CommandLine: python -m ibeis_cnn.utils --test-save_pretrained_weights_slice --net='vggnet_full' --slice='slice(0,6)' python -m ibeis_cnn.utils --test-save_pretrained_weights_slice --net='vggnet_full' --slice='slice(0,30)' python -m ibeis_cnn.utils --test-save_pretrained_weights_slice --net='caffenet_full' --slice='slice(0,6)' python -m ibeis_cnn.utils --test-save_pretrained_weights_slice --net='caffenet_full' --slice='slice(0,?)' Example: >>> # DISABLE_DOCTEST >>> # Build a new subset of an existing model >>> from ibeis_cnn.models import * # NOQA >>> from ibeis_cnn._plugin_grabmodels import ensure_model >>> # Get base model weights >>> modelname = ut.get_argval('--net', type_=str, default='vggnet_full') >>> weights_path = ensure_model(modelname) >>> pretrained_weights = ut.load_cPkl(weights_path) >>> # Get the slice you want >>> slice_str = ut.get_argval('--slice', type_=str, default='slice(0, 6)') >>> slice_ = eval(slice_str, globals(), locals()) >>> # execute function >>> sliced_weights_path = save_pretrained_weights_slice(pretrained_weights, weights_path, slice_) >>> # PUT YOUR PUBLISH PATH HERE >>> publish_fpath = ut.truepath('~/Dropbox/IBEIS') >>> ut.copy(sliced_weights_path, publish_fpath) """ # slice and save suffix = '.slice_%r_%r_%r' % (slice_.start, slice_.stop, slice_.step) sliced_weights_path = ut.augpath(weights_path, suffix) sliced_pretrained_weights = pretrained_weights[slice_] ut.save_cPkl(sliced_weights_path, sliced_pretrained_weights) # print info net_strs.print_pretrained_weights(pretrained_weights, weights_path) net_strs.print_pretrained_weights(sliced_pretrained_weights, sliced_weights_path) return sliced_weights_path
def get_extramargin_detectchip_info(ibs, aid_list, config2_=None, species=None, FACTOR=4): r""" Computes a detection chip with a bit of spatial context so the detection algorithm doesn't clip boundaries CommandLine: python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show --qaid 27 python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show --qaid 2 Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.preproc.preproc_probchip import * # NOQA >>> import ibeis >>> from ibeis.init import main_helpers >>> ibs = ibeis.opendb('PZ_MTEST') >>> aid_list = main_helpers.get_test_qaids(ibs) >>> arg_list, newsize_list, halfoffset_cs_list = get_extramargin_detectchip_info(ibs, aid_list) >>> ut.quit_if_noshow() >>> testshow_extramargin_info(ibs, aid_list, arg_list, newsize_list, halfoffset_cs_list) """ from vtool import chip as ctool target_width = 128 * FACTOR gfpath_list = ibs.get_annot_image_paths(aid_list) bbox_list = ibs.get_annot_bboxes(aid_list) theta_list = ibs.get_annot_thetas(aid_list) bbox_size_list = ut.get_list_column(bbox_list, [2, 3]) newsize_list = list(map( lambda size: ctool.get_scaled_size_with_width(target_width, *size), bbox_size_list)) invalid_aids = [aid for aid, (w, h) in zip(aid_list, bbox_size_list) if w == 0 or h == 0] if len(invalid_aids) > 0: msg = ("REMOVE INVALID (BAD WIDTH AND/OR HEIGHT) AIDS TO COMPUTE AND WRITE CHIPS") msg += ("INVALID AIDS: %r" % (invalid_aids, )) print(msg) raise Exception(msg) # There are two spaces we are working in here # probchipspace _pcs (the space of the margined chip computed for probchip) and # imagespace _gs (the space using in bbox specification) # Compute the offset we would like in chip space for margin expansion halfoffset_cs_list = [ # TODO: Find correct offsets (16 * FACTOR, 16 * FACTOR) # (w / 16, h / 16) for (w, h) in newsize_list ] # Compute expanded newsize list to include the extra margin offset expanded_newsize_list = [ (w_pcs + (2 * xo_pcs), h_pcs + (2 * yo_pcs)) for (w_pcs, h_pcs), (xo_pcs, yo_pcs) in zip(newsize_list, halfoffset_cs_list) ] # Get the conversion from chip to image space to_imgspace_scale_factors = [ (w_gs / w_pcs, h_gs / h_pcs) for ((w_pcs, h_pcs), (w_gs, h_gs)) in zip(newsize_list, bbox_size_list) ] # Convert the chip offsets to image space halfoffset_gs_list = [ ((sx * xo), (sy * yo)) for (sx, sy), (xo, yo) in zip(to_imgspace_scale_factors, halfoffset_cs_list) ] # Find the size of the expanded margin bbox in image space expanded_bbox_gs_list = [ (x_gs - xo_gs, y_gs - yo_gs, w_gs + (2 * xo_gs), h_gs + (2 * yo_gs)) for (x_gs, y_gs, w_gs, h_gs), (xo_gs, yo_gs) in zip(bbox_list, halfoffset_gs_list) ] # TODO: make this work probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aid_list, config2_=config2_, species=species) #probchip_extramargin_fpath_list = [ut.augpath(fpath, '_extramargin') for #fpath in probchip_fpath_list] extramargin_fpath_list = [ut.augpath(fpath, '_extramargin').replace('probchip', 'detectchip') for fpath in probchip_fpath_list] # # filter by species and add a suffix for the probchip_input # # also compute a probchip fpath with an expanded suffix for the detector #probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aids, config2_=None, species=species) # Then crop the output and write that as the real probchip filtlist_iter = ([] for _ in range(len(aid_list))) arg_iter = zip(extramargin_fpath_list, gfpath_list, expanded_bbox_gs_list, theta_list, expanded_newsize_list, filtlist_iter) arg_list = list(arg_iter) return arg_list, newsize_list, halfoffset_cs_list
def update_bindings(): r""" Returns: dict: matchtups CommandLine: python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-update_bindings utprof.py ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-update_bindings Example: >>> # DISABLE_DOCTEST >>> from autogen_bindings import * # NOQA >>> import sys >>> import utool as ut >>> sys.path.append(ut.truepath('~/local/build_scripts/flannscripts')) >>> matchtups = update_bindings() >>> result = ('matchtups = %s' % (ut.repr2(matchtups),)) >>> print(result) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ from os.path import basename import difflib import numpy as np import re binding_names = [ 'build_index', 'used_memory', 'add_points', 'remove_point', 'compute_cluster_centers', 'load_index', 'save_index', 'find_nearest_neighbors', 'radius_search', 'remove_points', 'free_index', 'find_nearest_neighbors_index', # 'size', # 'veclen', # 'get_point', # 'flann_get_distance_order', # 'flann_get_distance_type', # 'flann_log_verbosity', # 'clean_removed_points', ] _places = [ '~/code/flann/src/cpp/flann/flann.cpp', '~/code/flann/src/cpp/flann/flann.h', '~/code/flann/src/python/pyflann/flann_ctypes.py', '~/code/flann/src/python/pyflann/index.py', ] eof_sentinals = { # 'flann_ctypes.py': '# END DEFINE BINDINGS', 'flann_ctypes.py': 'def ensure_2d_array(arr', # 'flann.h': '// END DEFINE BINDINGS', 'flann.h': '#ifdef __cplusplus', 'flann.cpp': None, 'index.py': None, } block_sentinals = { 'flann.h': re.escape('/**'), 'flann.cpp': 'template *<typename Distance>', # 'flann_ctypes.py': '\n', 'flann_ctypes.py': 'flann\.[a-z_.]* =', # 'index.py': ' def .*', 'index.py': ' [^ ].*', } places = { basename(fpath): fpath for fpath in ut.lmap(ut.truepath, _places) } text_dict = ut.map_dict_vals(ut.readfrom, places) lines_dict = {key: val.split('\n') for key, val in text_dict.items()} orig_texts = text_dict.copy() # NOQA binding_defs = {} named_blocks = {} print('binding_names = %r' % (binding_names, )) for binding_name in binding_names: blocks, defs = autogen_parts(binding_name) binding_defs[binding_name] = defs named_blocks[binding_name] = blocks for binding_name in ut.ProgIter(binding_names): ut.colorprint('+--- GENERATE BINDING %s -----' % (binding_name, ), 'yellow') blocks_dict = named_blocks[binding_name] for key in places.keys(): ut.colorprint( '---- generating %s for %s -----' % ( binding_name, key, ), 'yellow') # key = 'flann_ctypes.py' # print(text_dict[key]) old_text = text_dict[key] line_list = lines_dict[key] #text = old_text block = blocks_dict[key] debug = ut.get_argflag('--debug') # debug = True # if debug: # print(ut.highlight_code(block, splitext(key)[1])) # Find a place in the code that already exists searchblock = block if key.endswith('.cpp') or key.endswith('.h'): searchblock = re.sub(ut.REGEX_C_COMMENT, '', searchblock, flags=re.MULTILINE | re.DOTALL) searchblock = '\n'.join(searchblock.splitlines()[0:3]) # @ut.cached_func(verbose=False) def cached_match(old_text, searchblock): def isjunk(x): return False return x in ' \t,*()' def isjunk2(x): return x in ' \t,*()' # Not sure why the first one just doesnt find it # isjunk = None sm = difflib.SequenceMatcher(isjunk, old_text, searchblock, autojunk=False) sm0 = difflib.SequenceMatcher(isjunk, old_text, searchblock, autojunk=True) sm1 = difflib.SequenceMatcher(isjunk2, old_text, searchblock, autojunk=False) sm2 = difflib.SequenceMatcher(isjunk2, old_text, searchblock, autojunk=True) matchtups = (sm.get_matching_blocks() + sm0.get_matching_blocks() + sm1.get_matching_blocks() + sm2.get_matching_blocks()) return matchtups matchtups = cached_match(old_text, searchblock) # Find a reasonable match in matchtups found = False if debug: # print('searchblock =\n%s' % (searchblock,)) print('searchblock = %r' % (searchblock, )) for (a, b, size) in matchtups: matchtext = old_text[a:a + size] pybind = binding_defs[binding_name]['py_binding_name'] if re.search(binding_name + '\\b', matchtext) or re.search( pybind + '\\b', matchtext): found = True pos = a + size if debug: print('MATCHING TEXT') print(matchtext) break else: if debug and 0: print('Not matching') print('matchtext = %r' % (matchtext, )) matchtext2 = old_text[a - 10:a + size + 20] print('matchtext2 = %r' % (matchtext2, )) if found: linelens = np.array(ut.lmap(len, line_list)) + 1 sumlen = np.cumsum(linelens) row = np.where(sumlen < pos)[0][-1] + 1 #print(line_list[row]) # Search for extents of the block to overwrite block_sentinal = block_sentinals[key] row1 = ut.find_block_end(row, line_list, block_sentinal, -1) - 1 row2 = ut.find_block_end(row + 1, line_list, block_sentinal, +1) eof_sentinal = eof_sentinals[key] if eof_sentinal is not None: print('eof_sentinal = %r' % (eof_sentinal, )) row2 = min([ count for count, line in enumerate(line_list) if line.startswith(eof_sentinal) ][-1], row2) nr = len((block + '\n\n').splitlines()) new_line_list = ut.insert_block_between_lines( block + '\n', row1, row2, line_list) rtext1 = '\n'.join(line_list[row1:row2]) rtext2 = '\n'.join(new_line_list[row1:row1 + nr]) if debug: print('-----') ut.colorprint('FOUND AND REPLACING %s' % (binding_name, ), 'yellow') print(ut.highlight_code(rtext1)) if debug: print('-----') ut.colorprint( 'FOUND AND REPLACED WITH %s' % (binding_name, ), 'yellow') print(ut.highlight_code(rtext2)) if not ut.get_argflag('--diff') and not debug: print( ut.color_diff_text( ut.difftext(rtext1, rtext2, num_context_lines=7, ignore_whitespace=True))) else: # Append to end of the file eof_sentinal = eof_sentinals[key] if eof_sentinal is None: row2 = len(line_list) - 1 else: row2_choice = [ count for count, line in enumerate(line_list) if line.startswith(eof_sentinal) ] if len(row2_choice) == 0: row2 = len(line_list) - 1 assert False else: row2 = row2_choice[-1] - 1 # row1 = row2 - 1 # row2 = row2 - 1 row1 = row2 new_line_list = ut.insert_block_between_lines( block + '\n', row1, row2, line_list) # block + '\n\n\n', row1, row2, line_list) rtext1 = '\n'.join(line_list[row1:row2]) nr = len((block + '\n\n').splitlines()) rtext2 = '\n'.join(new_line_list[row1:row1 + nr]) if debug: print('-----') ut.colorprint( 'NOT FOUND AND REPLACING %s' % (binding_name, ), 'yellow') print(ut.highlight_code(rtext1)) if debug: print('-----') ut.colorprint( 'NOT FOUND AND REPLACED WITH %s' % (binding_name, ), 'yellow') print(ut.highlight_code(rtext2)) if not ut.get_argflag('--diff') and not debug: print( ut.color_diff_text( ut.difftext(rtext1, rtext2, num_context_lines=7, ignore_whitespace=True))) text_dict[key] = '\n'.join(new_line_list) lines_dict[key] = new_line_list ut.colorprint('L___ GENERATED BINDING %s ___' % (binding_name, ), 'yellow') for key in places: new_text = '\n'.join(lines_dict[key]) #ut.writeto(ut.augpath(places[key], '.new'), new_text) ut.writeto(ut.augpath(places[key]), new_text) for key in places: if ut.get_argflag('--diff'): difftext = ut.get_textdiff(orig_texts[key], new_text, num_context_lines=7, ignore_whitespace=True) difftext = ut.color_diff_text(difftext) print(difftext)
def on_save(nnindexer, depc, fpath): #print('NNINDEX ON SAVE') # Save FLANN as well flann_fpath = ut.augpath(fpath, '_flann', newext='.flann') nnindexer.save(fpath=flann_fpath)
def get_extramargin_detectchip_info(ibs, aid_list, config2_=None, species=None, FACTOR=4): r""" Computes a detection chip with a bit of spatial context so the detection algorithm doesn't clip boundaries CommandLine: python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show --qaid 27 python -m ibeis.algo.preproc.preproc_probchip --test-get_extramargin_detectchip_info --show --qaid 2 Example: >>> # ENABLE_DOCTEST >>> from ibeis.algo.preproc.preproc_probchip import * # NOQA >>> import ibeis >>> from ibeis.init import main_helpers >>> ibs = ibeis.opendb('PZ_MTEST') >>> aid_list = ibeis.testdata_aids(ibs=ibs, a='default') >>> arg_list, newsize_list, halfoffset_cs_list = get_extramargin_detectchip_info(ibs, aid_list) >>> ut.quit_if_noshow() >>> testshow_extramargin_info(ibs, aid_list, arg_list, newsize_list, halfoffset_cs_list) """ from vtool import chip as ctool target_width = 128 * FACTOR gfpath_list = ibs.get_annot_image_paths(aid_list) bbox_list = ibs.get_annot_bboxes(aid_list) theta_list = ibs.get_annot_thetas(aid_list) bbox_size_list = ut.get_list_column(bbox_list, [2, 3]) newsize_list = list( map(lambda size: ctool.get_scaled_size_with_width(target_width, *size), bbox_size_list)) invalid_aids = [ aid for aid, (w, h) in zip(aid_list, bbox_size_list) if w == 0 or h == 0 ] if len(invalid_aids) > 0: msg = ( "REMOVE INVALID (BAD WIDTH AND/OR HEIGHT) AIDS TO COMPUTE AND WRITE CHIPS" ) msg += ("INVALID AIDS: %r" % (invalid_aids, )) print(msg) raise Exception(msg) # There are two spaces we are working in here # probchipspace _pcs (the space of the margined chip computed for probchip) and # imagespace _gs (the space using in bbox specification) # Compute the offset we would like in chip space for margin expansion halfoffset_cs_list = [ # TODO: Find correct offsets (16 * FACTOR, 16 * FACTOR) # (w / 16, h / 16) for (w, h) in newsize_list ] # Compute expanded newsize list to include the extra margin offset expanded_newsize_list = [ (w_pcs + (2 * xo_pcs), h_pcs + (2 * yo_pcs)) for (w_pcs, h_pcs), (xo_pcs, yo_pcs) in zip(newsize_list, halfoffset_cs_list) ] # Get the conversion from chip to image space to_imgspace_scale_factors = [ (w_gs / w_pcs, h_gs / h_pcs) for ((w_pcs, h_pcs), (w_gs, h_gs)) in zip(newsize_list, bbox_size_list) ] # Convert the chip offsets to image space halfoffset_gs_list = [ ((sx * xo), (sy * yo)) for (sx, sy), (xo, yo) in zip(to_imgspace_scale_factors, halfoffset_cs_list) ] # Find the size of the expanded margin bbox in image space expanded_bbox_gs_list = [ (x_gs - xo_gs, y_gs - yo_gs, w_gs + (2 * xo_gs), h_gs + (2 * yo_gs)) for (x_gs, y_gs, w_gs, h_gs), (xo_gs, yo_gs) in zip(bbox_list, halfoffset_gs_list) ] # TODO: make this work probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aid_list, config2_=config2_, species=species) #probchip_extramargin_fpath_list = [ut.augpath(fpath, '_extramargin') for #fpath in probchip_fpath_list] extramargin_fpath_list = [ ut.augpath(fpath, '_extramargin').replace('probchip', 'detectchip') for fpath in probchip_fpath_list ] # # filter by species and add a suffix for the probchip_input # # also compute a probchip fpath with an expanded suffix for the detector #probchip_fpath_list = get_annot_probchip_fpath_list(ibs, aids, config2_=None, species=species) # Then crop the output and write that as the real probchip filtlist_iter = ([] for _ in range(len(aid_list))) arg_iter = zip(extramargin_fpath_list, gfpath_list, expanded_bbox_gs_list, theta_list, expanded_newsize_list, filtlist_iter) arg_list = list(arg_iter) return arg_list, newsize_list, halfoffset_cs_list
def update_bindings(): r""" Returns: dict: matchtups CommandLine: python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-update_bindings utprof.py ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-update_bindings Example: >>> # DISABLE_DOCTEST >>> from autogen_bindings import * # NOQA >>> import sys >>> import utool as ut >>> sys.path.append(ut.truepath('~/local/build_scripts/flannscripts')) >>> matchtups = update_bindings() >>> result = ('matchtups = %s' % (ut.repr2(matchtups),)) >>> print(result) >>> ut.quit_if_noshow() >>> import plottool as pt >>> ut.show_if_requested() """ from os.path import basename import difflib import numpy as np import re binding_names = [ 'build_index', 'used_memory', 'add_points', 'remove_point', 'compute_cluster_centers', 'load_index', 'save_index', 'find_nearest_neighbors', 'radius_search', 'remove_points', 'free_index', 'find_nearest_neighbors_index', # 'size', # 'veclen', # 'get_point', # 'flann_get_distance_order', # 'flann_get_distance_type', # 'flann_log_verbosity', # 'clean_removed_points', ] _places = [ '~/code/flann/src/cpp/flann/flann.cpp', '~/code/flann/src/cpp/flann/flann.h', '~/code/flann/src/python/pyflann/flann_ctypes.py', '~/code/flann/src/python/pyflann/index.py', ] eof_sentinals = { # 'flann_ctypes.py': '# END DEFINE BINDINGS', 'flann_ctypes.py': 'def ensure_2d_array(arr', # 'flann.h': '// END DEFINE BINDINGS', 'flann.h': '#ifdef __cplusplus', 'flann.cpp': None, 'index.py': None, } block_sentinals = { 'flann.h': re.escape('/**'), 'flann.cpp': 'template *<typename Distance>', # 'flann_ctypes.py': '\n', 'flann_ctypes.py': 'flann\.[a-z_.]* =', # 'index.py': ' def .*', 'index.py': ' [^ ].*', } places = {basename(fpath): fpath for fpath in ut.lmap(ut.truepath, _places)} text_dict = ut.map_dict_vals(ut.readfrom, places) lines_dict = {key: val.split('\n') for key, val in text_dict.items()} orig_texts = text_dict.copy() # NOQA binding_defs = {} named_blocks = {} print('binding_names = %r' % (binding_names,)) for binding_name in binding_names: blocks, defs = autogen_parts(binding_name) binding_defs[binding_name] = defs named_blocks[binding_name] = blocks for binding_name in ut.ProgIter(binding_names): ut.colorprint('+--- GENERATE BINDING %s -----' % (binding_name,), 'yellow') blocks_dict = named_blocks[binding_name] for key in places.keys(): ut.colorprint('---- generating %s for %s -----' % (binding_name, key,), 'yellow') # key = 'flann_ctypes.py' # print(text_dict[key]) old_text = text_dict[key] line_list = lines_dict[key] #text = old_text block = blocks_dict[key] debug = ut.get_argflag('--debug') # debug = True # if debug: # print(ut.highlight_code(block, splitext(key)[1])) # Find a place in the code that already exists searchblock = block if key.endswith('.cpp') or key.endswith('.h'): searchblock = re.sub(ut.REGEX_C_COMMENT, '', searchblock, flags=re.MULTILINE | re.DOTALL) searchblock = '\n'.join(searchblock.splitlines()[0:3]) # @ut.cached_func(verbose=False) def cached_match(old_text, searchblock): def isjunk(x): return False return x in ' \t,*()' def isjunk2(x): return x in ' \t,*()' # Not sure why the first one just doesnt find it # isjunk = None sm = difflib.SequenceMatcher(isjunk, old_text, searchblock, autojunk=False) sm0 = difflib.SequenceMatcher(isjunk, old_text, searchblock, autojunk=True) sm1 = difflib.SequenceMatcher(isjunk2, old_text, searchblock, autojunk=False) sm2 = difflib.SequenceMatcher(isjunk2, old_text, searchblock, autojunk=True) matchtups = (sm.get_matching_blocks() + sm0.get_matching_blocks() + sm1.get_matching_blocks() + sm2.get_matching_blocks()) return matchtups matchtups = cached_match(old_text, searchblock) # Find a reasonable match in matchtups found = False if debug: # print('searchblock =\n%s' % (searchblock,)) print('searchblock = %r' % (searchblock,)) for (a, b, size) in matchtups: matchtext = old_text[a: a + size] pybind = binding_defs[binding_name]['py_binding_name'] if re.search(binding_name + '\\b', matchtext) or re.search(pybind + '\\b', matchtext): found = True pos = a + size if debug: print('MATCHING TEXT') print(matchtext) break else: if debug and 0: print('Not matching') print('matchtext = %r' % (matchtext,)) matchtext2 = old_text[a - 10: a + size + 20] print('matchtext2 = %r' % (matchtext2,)) if found: linelens = np.array(ut.lmap(len, line_list)) + 1 sumlen = np.cumsum(linelens) row = np.where(sumlen < pos)[0][-1] + 1 #print(line_list[row]) # Search for extents of the block to overwrite block_sentinal = block_sentinals[key] row1 = ut.find_block_end(row, line_list, block_sentinal, -1) - 1 row2 = ut.find_block_end(row + 1, line_list, block_sentinal, +1) eof_sentinal = eof_sentinals[key] if eof_sentinal is not None: print('eof_sentinal = %r' % (eof_sentinal,)) row2 = min([count for count, line in enumerate(line_list) if line.startswith(eof_sentinal)][-1], row2) nr = len((block + '\n\n').splitlines()) new_line_list = ut.insert_block_between_lines( block + '\n', row1, row2, line_list) rtext1 = '\n'.join(line_list[row1:row2]) rtext2 = '\n'.join(new_line_list[row1:row1 + nr]) if debug: print('-----') ut.colorprint('FOUND AND REPLACING %s' % (binding_name,), 'yellow') print(ut.highlight_code(rtext1)) if debug: print('-----') ut.colorprint('FOUND AND REPLACED WITH %s' % (binding_name,), 'yellow') print(ut.highlight_code(rtext2)) if not ut.get_argflag('--diff') and not debug: print(ut.color_diff_text(ut.difftext(rtext1, rtext2, num_context_lines=7, ignore_whitespace=True))) else: # Append to end of the file eof_sentinal = eof_sentinals[key] if eof_sentinal is None: row2 = len(line_list) - 1 else: row2_choice = [count for count, line in enumerate(line_list) if line.startswith(eof_sentinal)] if len(row2_choice) == 0: row2 = len(line_list) - 1 assert False else: row2 = row2_choice[-1] - 1 # row1 = row2 - 1 # row2 = row2 - 1 row1 = row2 new_line_list = ut.insert_block_between_lines( block + '\n', row1, row2, line_list) # block + '\n\n\n', row1, row2, line_list) rtext1 = '\n'.join(line_list[row1:row2]) nr = len((block + '\n\n').splitlines()) rtext2 = '\n'.join(new_line_list[row1:row1 + nr]) if debug: print('-----') ut.colorprint('NOT FOUND AND REPLACING %s' % (binding_name,), 'yellow') print(ut.highlight_code(rtext1)) if debug: print('-----') ut.colorprint('NOT FOUND AND REPLACED WITH %s' % (binding_name,), 'yellow') print(ut.highlight_code(rtext2)) if not ut.get_argflag('--diff') and not debug: print(ut.color_diff_text(ut.difftext(rtext1, rtext2, num_context_lines=7, ignore_whitespace=True))) text_dict[key] = '\n'.join(new_line_list) lines_dict[key] = new_line_list ut.colorprint('L___ GENERATED BINDING %s ___' % (binding_name,), 'yellow') for key in places: new_text = '\n'.join(lines_dict[key]) #ut.writeto(ut.augpath(places[key], '.new'), new_text) ut.writeto(ut.augpath(places[key]), new_text) for key in places: if ut.get_argflag('--diff'): difftext = ut.get_textdiff(orig_texts[key], new_text, num_context_lines=7, ignore_whitespace=True) difftext = ut.color_diff_text(difftext) print(difftext)
def sort_module_functions(): from os.path import dirname, join import utool as ut import ibeis.control import re #import re #regex = r'[^@]*\ndef' modfpath = dirname(ibeis.control.__file__) fpath = join(modfpath, 'manual_annot_funcs.py') #fpath = join(modfpath, 'manual_dependant_funcs.py') #fpath = join(modfpath, 'manual_lblannot_funcs.py') #fpath = join(modfpath, 'manual_name_species_funcs.py') text = ut.read_from(fpath, verbose=False) lines = text.splitlines() indent_list = [ut.get_indentation(line) for line in lines] isfunc_list = [line.startswith('def ') for line in lines] isblank_list = [len(line.strip(' ')) == 0 for line in lines] isdec_list = [line.startswith('@') for line in lines] tmp = [ 'def' if isfunc else indent for isfunc, indent in zip(isfunc_list, indent_list) ] tmp = ['b' if isblank else t for isblank, t in zip(isblank_list, tmp)] tmp = ['@' if isdec else t for isdec, t in zip(isdec_list, tmp)] #print('\n'.join([str((t, count + 1)) for (count, t) in enumerate(tmp)])) block_list = re.split('\n\n\n', text, flags=re.MULTILINE) #for block in block_list: # print('#====') # print(block) isfunc_list = [ re.search('^def ', block, re.MULTILINE) is not None for block in block_list ] whole_varname = ut.whole_word(ut.REGEX_VARNAME) funcname_regex = r'def\s+' + ut.named_field('funcname', whole_varname) def findfuncname(block): match = re.search(funcname_regex, block) return match.group('funcname') funcnameblock_list = [ findfuncname(block) if isfunc else None for isfunc, block in zip(isfunc_list, block_list) ] funcblock_list = ut.filter_items(block_list, isfunc_list) funcname_list = ut.filter_items(funcnameblock_list, isfunc_list) nonfunc_list = ut.filterfalse_items(block_list, isfunc_list) nonfunc_list = ut.filterfalse_items(block_list, isfunc_list) ismain_list = [ re.search('^if __name__ == ["\']__main__["\']', nonfunc) is not None for nonfunc in nonfunc_list ] mainblock_list = ut.filter_items(nonfunc_list, ismain_list) nonfunc_list = ut.filterfalse_items(nonfunc_list, ismain_list) newtext_list = [] for nonfunc in nonfunc_list: newtext_list.append(nonfunc) newtext_list.append('\n') #funcname_list for funcblock in ut.sortedby(funcblock_list, funcname_list): newtext_list.append(funcblock) newtext_list.append('\n') for mainblock in mainblock_list: newtext_list.append(mainblock) newtext = '\n'.join(newtext_list) print('newtext = %s' % (newtext, )) print('len(newtext) = %r' % (len(newtext), )) print('len(text) = %r' % (len(text), )) backup_fpath = ut.augpath(fpath, augext='.bak', augdir='_backup', ensure=True) ut.write_to(backup_fpath, text) ut.write_to(fpath, newtext)
def sort_module_functions(): from os.path import dirname, join import utool as ut import ibeis.control import re #import re #regex = r'[^@]*\ndef' modfpath = dirname(ibeis.control.__file__) fpath = join(modfpath, 'manual_annot_funcs.py') #fpath = join(modfpath, 'manual_dependant_funcs.py') #fpath = join(modfpath, 'manual_lblannot_funcs.py') #fpath = join(modfpath, 'manual_name_species_funcs.py') text = ut.read_from(fpath, verbose=False) lines = text.splitlines() indent_list = [ut.get_indentation(line) for line in lines] isfunc_list = [line.startswith('def ') for line in lines] isblank_list = [len(line.strip(' ')) == 0 for line in lines] isdec_list = [line.startswith('@') for line in lines] tmp = ['def' if isfunc else indent for isfunc, indent in zip(isfunc_list, indent_list)] tmp = ['b' if isblank else t for isblank, t in zip(isblank_list, tmp)] tmp = ['@' if isdec else t for isdec, t in zip(isdec_list, tmp)] #print('\n'.join([str((t, count + 1)) for (count, t) in enumerate(tmp)])) block_list = re.split('\n\n\n', text, flags=re.MULTILINE) #for block in block_list: # print('#====') # print(block) isfunc_list = [re.search('^def ', block, re.MULTILINE) is not None for block in block_list] whole_varname = ut.whole_word(ut.REGEX_VARNAME) funcname_regex = r'def\s+' + ut.named_field('funcname', whole_varname) def findfuncname(block): match = re.search(funcname_regex, block) return match.group('funcname') funcnameblock_list = [findfuncname(block) if isfunc else None for isfunc, block in zip(isfunc_list, block_list)] funcblock_list = ut.filter_items(block_list, isfunc_list) funcname_list = ut.filter_items(funcnameblock_list, isfunc_list) nonfunc_list = ut.filterfalse_items(block_list, isfunc_list) nonfunc_list = ut.filterfalse_items(block_list, isfunc_list) ismain_list = [re.search('^if __name__ == ["\']__main__["\']', nonfunc) is not None for nonfunc in nonfunc_list] mainblock_list = ut.filter_items(nonfunc_list, ismain_list) nonfunc_list = ut.filterfalse_items(nonfunc_list, ismain_list) newtext_list = [] for nonfunc in nonfunc_list: newtext_list.append(nonfunc) newtext_list.append('\n') #funcname_list for funcblock in ut.sortedby(funcblock_list, funcname_list): newtext_list.append(funcblock) newtext_list.append('\n') for mainblock in mainblock_list: newtext_list.append(mainblock) newtext = '\n'.join(newtext_list) print('newtext = %s' % (newtext,)) print('len(newtext) = %r' % (len(newtext),)) print('len(text) = %r' % (len(text),)) backup_fpath = ut.augpath(fpath, augext='.bak', augdir='_backup', ensure=True) ut.write_to(backup_fpath, text) ut.write_to(fpath, newtext)