def get_inspect_str(qres): assert_qres(qres) nFeatMatch_list = get_num_feats_in_matches(qres) nFeatMatch_stats = utool.mystats(nFeatMatch_list) top_lbl = utool.unindent(''' top aids scores ranks''').strip() top_aids = qres.get_top_aids(num=5) top_scores = qres.get_aid_scores(top_aids) top_ranks = qres.get_aid_ranks(top_aids) top_stack = np.vstack((top_aids, top_scores, top_ranks)) top_stack = np.array(top_stack, dtype=np.int32) top_str = str(top_stack) inspect_str = '\n'.join([ 'QueryResult', 'qaid=%r ' % qres.qaid, utool.horiz_string(top_lbl, ' ', top_str), 'num Feat Matches stats:', utool.indent(utool.dict_str(nFeatMatch_stats)), ]) inspect_str = utool.indent(inspect_str, '[INSPECT] ') return inspect_str
def check_importable(rman): import utool as ut label = " %s" % rman.label if rman.label else rman.label missing = [] print("Checking if%s modules are importable" % (label,)) msg_list = [] recommended_fixes = [] for repo in rman.repos: flag, msg, errors = repo.check_importable() if not flag: msg_list.append(" * !!!%s REPO %s HAS IMPORT ISSUES" % (label.upper(), repo)) if any([str(ex).find("undefined symbol") > -1 for ex in errors]): recommended_fixes.append("rebuild") else: recommended_fixes.append(None) if ut.VERBOSE: msg_list.append(ut.indent(msg, " ")) missing.append(repo) else: if ut.VERBOSE: msg_list.append(ut.indent(msg, " ")) print("\n".join(msg_list)) problems = list(zip(missing, recommended_fixes)) return problems
def _build_typed_params_kwargs_docstr_block(typed_params): r""" Args: typed_params (dict): CommandLine: python -m pyhesaff build_typed_params_docstr Example: >>> # DISABLE_DOCTEST >>> from pyhesaff._pyhesaff import * # NOQA >>> typed_params = HESAFF_TYPED_PARAMS >>> result = build_typed_params_docstr(typed_params) >>> print(result) """ kwargs_lines = [] for tup in typed_params: type_, name, default = tup typestr = str(type_).replace('<class \'ctypes.c_', '').replace('\'>', '') line_fmtstr = '{name} ({typestr}): default={default}' line = line_fmtstr.format(name=name, typestr=typestr, default=default) kwargs_lines.append(line) kwargs_docstr_block = ('Kwargs:\n' + ut.indent('\n'.join(kwargs_lines), ' ')) return ut.indent(kwargs_docstr_block, ' ')
def print_tree_struct(*args, **kwargs): tree_str = (ut.indent(ut.repr3(get_tree_info(*args, **kwargs), nl=1))) print(tree_str) #bytes_str = ut.byte_str2(drive.get_total_nbytes(dpath_to_unique_fidx[path])) #print('path = %r, %s' % (path, bytes_str)) #print(ut.repr3(key_list)) return tree_str
def markdown_cell(markdown): r""" Args: markdown (str): Returns: str: json formatted ipython notebook markdown cell CommandLine: python -m ibeis.templates.generate_notebook --exec-markdown_cell Example: >>> # DISABLE_DOCTEST >>> from ibeis.templates.generate_notebook import * # NOQA >>> markdown = '# Title' >>> result = markdown_cell(markdown) >>> print(result) """ markdown_header = ut.codeblock( """ { "cell_type": "markdown", "metadata": {}, "source": [ """ ) markdown_footer = ut.codeblock( """ ] } """ ) return (markdown_header + '\n' + ut.indent(repr_single(markdown), ' ' * 2) + '\n' + markdown_footer)
def dictinfo(dict_): if not isinstance(dict_, dict): return 'expected dict got %r' % type(dict_) keys = list(dict_.keys()) vals = list(dict_.values()) num_keys = len(keys) key_types = list(set(map(type, keys))) val_types = list(set(map(type, vals))) fmtstr_ = '\n' + ut.unindent(''' * num_keys = {num_keys} * key_types = {key_types} * val_types = {val_types} '''.strip('\n')) if len(val_types) == 1: if val_types[0] == np.ndarray: # each key holds an ndarray val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0) val_shape_stats_str = ut.dict_str(val_shape_stats, strvals=True, newlines=False) val_dtypes = list(set([val.dtype for val in vals])) fmtstr_ += ut.unindent(''' * val_shape_stats = {val_shape_stats_str} * val_dtypes = {val_dtypes} '''.strip('\n')) elif val_types[0] == list: # each key holds a list val_len_stats = ut.get_stats(set(map(len, vals))) val_len_stats_str = ut.dict_str(val_len_stats, strvals=True, newlines=False) depth = ut.list_depth(vals) deep_val_types = list(set(ut.list_deep_types(vals))) fmtstr_ += ut.unindent(''' * list_depth = {depth} * val_len_stats = {val_len_stats_str} * deep_types = {deep_val_types} '''.strip('\n')) if len(deep_val_types) == 1: if deep_val_types[0] == np.ndarray: deep_val_dtypes = list(set([val.dtype for val in vals])) fmtstr_ += ut.unindent(''' * deep_val_dtypes = {deep_val_dtypes} ''').strip('\n') elif val_types[0] in [ np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32, np.float64 ]: # each key holds a scalar val_stats = ut.get_stats(vals) fmtstr_ += ut.unindent(''' * val_stats = {val_stats} ''').strip('\n') fmtstr = fmtstr_.format(**locals()) return ut.indent(fmtstr)
def parse_benchmarks(funcname, docstring, py_modname): test_tuples_, setup_script_ = make_benchmarks(funcname, docstring, py_modname) if len(test_tuples_) == 0: test_tuples = '[]' else: test_tuples = '[\n' + (' ' * 8) + '\n '.join( list(map(str, test_tuples_))) + '\n ]' # NOQA setup_script = utool.indent(setup_script_).strip() # NOQA benchmark_name = utool.quasiquote('run_benchmark_{funcname}') #test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''') # http://en.wikipedia.org/wiki/Relative_change_and_difference bench_code_fmt_ = r''' def {benchmark_name}(iterations): test_tuples = {test_tuples} setup_script = textwrap.dedent(""" {setup_script} """) time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations) time_pair = lambda (x, y): (time_line(x), time_line(y)) def print_timing_info(tup): from math import log test_lines = [] def test_print(str): if not utool.QUIET: print(str) test_lines.append(str) test_print('\n---------------') test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations)) test_print('[bench] tests:') test_print(' ' + str(tup)) (pyth_time, cyth_time) = time_pair(tup) test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time)) test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time)) time_delta = cyth_time - pyth_time #pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100 #pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100 pyth_per_cyth = (pyth_time / cyth_time) * 100 inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100 nepers = log(cyth_time / pyth_time) if time_delta < 0: test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,)) #test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,)) test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,)) test_print('[bench.result] cython was faster by %f seconds' % -time_delta) else: test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,)) #test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,)) test_print('[bench.result] cython was %.1f nepers slower' % (nepers,)) test_print('[bench.result] python was faster by %f seconds' % time_delta) pyth_call, cyth_call = tup run_doctest(pyth_call, cyth_call, setup_script) return (pyth_time, cyth_time, test_lines) test_results = list(map(print_timing_info, test_tuples)) # results are lists of (time1, time2, strlist) return test_results''' bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n') bench_code = utool.quasiquote(bench_code_fmt) return (benchmark_name, bench_code)
def parse_benchmarks(funcname, docstring, py_modname): test_tuples_, setup_script_ = make_benchmarks(funcname, docstring, py_modname) if len(test_tuples_) == 0: test_tuples = '[]' else: test_tuples = '[\n' + (' ' * 8) + '\n '.join(list(map(str, test_tuples_))) + '\n ]' # NOQA setup_script = utool.indent(setup_script_).strip() # NOQA benchmark_name = utool.quasiquote('run_benchmark_{funcname}') #test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''') # http://en.wikipedia.org/wiki/Relative_change_and_difference bench_code_fmt_ = r''' def {benchmark_name}(iterations): test_tuples = {test_tuples} setup_script = textwrap.dedent(""" {setup_script} """) time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations) time_pair = lambda (x, y): (time_line(x), time_line(y)) def print_timing_info(tup): from math import log test_lines = [] def test_print(str): if not utool.QUIET: print(str) test_lines.append(str) test_print('\n---------------') test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations)) test_print('[bench] tests:') test_print(' ' + str(tup)) (pyth_time, cyth_time) = time_pair(tup) test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time)) test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time)) time_delta = cyth_time - pyth_time #pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100 #pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100 pyth_per_cyth = (pyth_time / cyth_time) * 100 inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100 nepers = log(cyth_time / pyth_time) if time_delta < 0: test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,)) #test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,)) test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,)) test_print('[bench.result] cython was faster by %f seconds' % -time_delta) else: test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,)) #test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,)) test_print('[bench.result] cython was %.1f nepers slower' % (nepers,)) test_print('[bench.result] python was faster by %f seconds' % time_delta) pyth_call, cyth_call = tup run_doctest(pyth_call, cyth_call, setup_script) return (pyth_time, cyth_time, test_lines) test_results = list(map(print_timing_info, test_tuples)) # results are lists of (time1, time2, strlist) return test_results''' bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n') bench_code = utool.quasiquote(bench_code_fmt) return (benchmark_name, bench_code)
def convert_tests_from_ibeis_to_nose(module_list): # PARSE OUT TESTABLE DOCTESTTUPS #import utool as ut testtup_list = [] seen_ = set() topimport_list = [] for module in module_list: mod_doctest_tup = ut.get_module_doctest_tup(module=module, verbose=False, allexamples=True) enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup flags = [tup.src not in seen_ for tup in enabled_testtup_list] enabled_testtup_list = ut.compress(enabled_testtup_list, flags) testtup_list.extend(enabled_testtup_list) if len(enabled_testtup_list) > 0: topimport_list.append('from %s import * # NOQA' % (module.__name__,)) print('Found %d test tups' % (len(testtup_list))) autogen_test_src_funcs = [] #import redbaron for testtup in testtup_list: name = testtup.name num = testtup.num src = testtup.src want = testtup.want import re src = re.sub('# ENABLE_DOCTEST\n', '', src) src = re.sub('from [^*]* import \* *# NOQA\n', '', src) src = re.sub(r'from [^*]* import \*\n', '', src) src = ut.str_between(src, None, 'ut.quit_if_noshow').rstrip('\n') src = ut.str_between(src, None, 'ut.show_if_requested').rstrip('\n') # import utool # utool.embed() """ """ #flag = testtup.flag if want.endswith('\n'): want = want[:-1] if want: #src_node = redbaron.RedBaron(src) #if len(src_node.find_all('name', 'result')) > 0: # src_node.append('assert result == %r' % (want,)) if '\nresult = ' in src: src += '\nassert str(result) == %r' % (want,) func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src) autogen_test_src_funcs.append(func_src) autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n' from ibeis import tests from os.path import join moddir = ut.get_module_dir(tests) ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
def make_bench_text(benchmark_codes, benchmark_names, py_modname): # TODO: let each function individually specify number codes = '\n\n\n'.join(benchmark_codes) # NOQA list_ = [utool.quasiquote('results.extend({benchfunc}(iterations))') for benchfunc in benchmark_names] all_benchmarks = utool.indent('\n'.join(list_), ' ' * 8).strip() # NOQA timestamp = utool.get_timestamp() # NOQA bench_text_fmt = get_bench_text_fmt() bench_text = utool.quasiquote(bench_text_fmt) return bench_text
def expand_closure_source(funcname, func): source = ut.get_func_sourcecode(func) closure_vars = [(k, v.cell_contents) for k, v in zip(func.func_code.co_freevars, func.func_closure)] source = ut.unindent(source) import re for k, v in closure_vars: source = re.sub('\\b' + k + '\\b', ut.repr2(v), source) source = re.sub(r'def .*\(self', 'def ' + funcname + '(self', source) source = ut.indent(source.strip(), ' ') + '\n' return source
def make_bench_text(benchmark_codes, benchmark_names, py_modname): # TODO: let each function individually specify number codes = '\n\n\n'.join(benchmark_codes) # NOQA list_ = [ utool.quasiquote('results.extend({benchfunc}(iterations))') for benchfunc in benchmark_names ] all_benchmarks = utool.indent('\n'.join(list_), ' ' * 8).strip() # NOQA timestamp = utool.get_timestamp() # NOQA bench_text_fmt = get_bench_text_fmt() bench_text = utool.quasiquote(bench_text_fmt) return bench_text
def dictinfo(dict_): if not isinstance(dict_, dict): return 'expected dict got %r' % type(dict_) keys = list(dict_.keys()) vals = list(dict_.values()) num_keys = len(keys) key_types = list(set(map(type, keys))) val_types = list(set(map(type, vals))) fmtstr_ = '\n' + ut.unindent(''' * num_keys = {num_keys} * key_types = {key_types} * val_types = {val_types} '''.strip('\n')) if len(val_types) == 1: if val_types[0] == np.ndarray: # each key holds an ndarray val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0) val_shape_stats_str = ut.dict_str(val_shape_stats, strvals=True, newlines=False) val_dtypes = list(set([val.dtype for val in vals])) fmtstr_ += ut.unindent(''' * val_shape_stats = {val_shape_stats_str} * val_dtypes = {val_dtypes} '''.strip('\n')) elif val_types[0] == list: # each key holds a list val_len_stats = ut.get_stats(set(map(len, vals))) val_len_stats_str = ut.dict_str(val_len_stats, strvals=True, newlines=False) depth = ut.list_depth(vals) deep_val_types = list(set(ut.list_deep_types(vals))) fmtstr_ += ut.unindent(''' * list_depth = {depth} * val_len_stats = {val_len_stats_str} * deep_types = {deep_val_types} '''.strip('\n')) if len(deep_val_types) == 1: if deep_val_types[0] == np.ndarray: deep_val_dtypes = list(set([val.dtype for val in vals])) fmtstr_ += ut.unindent(''' * deep_val_dtypes = {deep_val_dtypes} ''').strip('\n') elif val_types[0] in [np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32, np.float64]: # each key holds a scalar val_stats = ut.get_stats(vals) fmtstr_ += ut.unindent(''' * val_stats = {val_stats} ''').strip('\n') fmtstr = fmtstr_.format(**locals()) return ut.indent(fmtstr)
def check_cpp_build(rman): import utool as ut label = ' %s' % rman.label if rman.label else rman.label missing = [] print('Checking if%s modules are built' % (label,)) for repo in rman.repos: flag, msg = repo.check_cpp_build() if not flag: print(' * !!!%s REPO %s NEEDS TO BE BUILT' % (label.upper(), repo,)) if ut.VERBOSE: print(ut.indent(msg, ' ')) missing.append(repo) return missing
def get_inspect_str(qres, ibs=None, name_scoring=False): qres.assert_self() #ut.embed() top_lbls = [' top aids', ' scores', ' rawscores', ' ranks'] top_aids = np.array(qres.get_top_aids(num=6, name_scoring=name_scoring, ibs=ibs), dtype=np.int32) top_scores = np.array(qres.get_aid_scores(top_aids), dtype=np.float64) top_rawscores = np.array(qres.get_aid_scores(top_aids, rawscore=True), dtype=np.float64) top_ranks = np.array(qres.get_aid_ranks(top_aids), dtype=np.int32) top_list = [top_aids, top_scores, top_rawscores, top_ranks] if ibs is not None: top_lbls += [' isgt'] istrue = qres.get_aid_truth(ibs, top_aids) top_list.append(np.array(istrue, dtype=np.int32)) if name_scoring: top_lbls = ['top nid'] + top_lbls top_list = [ibs.get_annot_name_rowids(top_aids)] + top_list top_stack = np.vstack(top_list) #top_stack = np.array(top_stack, dtype=object) top_stack = np.array(top_stack, dtype=np.float32) #np.int32) top_str = np.array_str(top_stack, precision=3, suppress_small=True, max_line_width=200) top_lbl = '\n'.join(top_lbls) inspect_list = ['QueryResult', qres.cfgstr, ] if ibs is not None: gt_ranks = qres.get_gt_ranks(ibs=ibs) gt_scores = qres.get_gt_scores(ibs=ibs) inspect_list.append('gt_ranks = %r' % gt_ranks) inspect_list.append('gt_scores = %r' % gt_scores) nFeatMatch_list = get_num_feats_in_matches(qres) nFeatMatch_stats_str = ut.get_stats_str(nFeatMatch_list, newlines=True, exclude_keys=('nMin', 'nMax')) inspect_list.extend([ 'qaid=%r ' % qres.qaid, ut.hz_str(top_lbl, ' ', top_str), 'num feat matches per annotation stats:', #ut.indent(ut.dict_str(nFeatMatch_stats)), ut.indent(nFeatMatch_stats_str), ]) inspect_str = '\n'.join(inspect_list) #inspect_str = ut.indent(inspect_str, '[INSPECT] ') return inspect_str
def make_notebook(cell_list): """ References: # Change cell width http://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-ipython-notebook-in-my-browser/24207353#24207353 """ import utool as ut header = ut.codeblock( ''' { "cells": [ ''' ) footer = ut.codeblock( ''' ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 0 } ''') cell_body = ut.indent(',\n'.join(cell_list), ' ') notebook_str = header + '\n' + cell_body + '\n' + footer try: import json json.loads(notebook_str) except ValueError as ex: ut.printex(ex, 'Invalid notebook JSON') raise return notebook_str
def convert_tests_from_utool_to_nose(module_list): # PARSE OUT TESTABLE DOCTESTTUPS #import utool as ut testtup_list = [] seen_ = set() topimport_list = [] for module in module_list: mod_doctest_tup = ut.get_module_doctest_tup(module=module, verbose=False, allexamples=True) enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup flags = [tup.src not in seen_ for tup in enabled_testtup_list] enabled_testtup_list = ut.compress(enabled_testtup_list, flags) testtup_list.extend(enabled_testtup_list) if len(enabled_testtup_list) > 0: topimport_list.append('from %s import * # NOQA' % (module.__name__,)) print('Found %d test tups' % (len(testtup_list))) autogen_test_src_funcs = [] #import redbaron for testtup in testtup_list: name = testtup.name num = testtup.num src = testtup.src want = testtup.want import re src = re.sub('# ENABLE_DOCTEST\n', '', src) src = re.sub('from [^*]* import \* *# NOQA\n', '', src) src = re.sub('from [^*]* import \*\n', '', src) #flag = testtup.flag if want.endswith('\n'): want = want[:-1] if want: #src_node = redbaron.RedBaron(src) #if len(src_node.find_all('name', 'result')) > 0: # src_node.append('assert result == %r' % (want,)) if '\nresult = ' in src: src += '\nassert str(result) == %r' % (want,) func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src) autogen_test_src_funcs.append(func_src) autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n' from utool import tests from os.path import join moddir = ut.get_module_dir(tests) ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
def as_table(self, caption=None): if caption is None: caption = self.caption tabular = self.as_tabular() table = ut.codeblock(r""" \begin{{table}}[h] \centering \caption{{{caption}}} """).format(caption=caption) if tabular: table += '\n' + ut.indent(tabular) table += ('\n' + ut.codeblock(""" \\end{{table}} """).format()) return table
def check_importable(rman): import utool as ut label = ' %s' % rman.label if rman.label else rman.label missing = [] print('Checking if%s modules are importable' % (label,)) msg_list = [] recommended_fixes = [] for repo in rman.repos: flag, msg, errors = repo.check_importable() if not flag: msg_list.append(' * !!!%s REPO %s HAS IMPORT ISSUES' % (label.upper(), repo,)) if any([str(ex).find('undefined symbol') > -1 for ex in errors]): recommended_fixes.append('rebuild') else: recommended_fixes.append(None) if ut.VERBOSE: msg_list.append(ut.indent(msg, ' ')) missing.append(repo) else: if ut.VERBOSE: msg_list.append(ut.indent(msg, ' ')) print('\n'.join(msg_list)) problems = list(zip(missing, recommended_fixes)) return problems
def code_cell(sourcecode): r""" Args: sourcecode (str): Returns: str: json formatted ipython notebook code cell CommandLine: python -m ibeis.templates.generate_notebook --exec-code_cell Example: >>> # DISABLE_DOCTEST >>> from ibeis.templates.generate_notebook import * # NOQA >>> sourcecode = notebook_cells.timestamp_distribution[1] >>> sourcecode = notebook_cells.initialize[1] >>> result = code_cell(sourcecode) >>> print(result) """ import utool as ut sourcecode = ut.remove_codeblock_syntax_sentinals(sourcecode) cell_header = ut.codeblock( ''' { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": ''') cell_footer = ut.codeblock( ''' } ''') if sourcecode is None: source_line_repr = ' []\n' else: lines = sourcecode.split('\n') line_list = [line + '\n' if count < len(lines) else line for count, line in enumerate(lines, start=1)] #repr_line_list = [repr_single_for_md(line) for line in line_list] repr_line_list = [repr_single_for_md(line) for line in line_list] source_line_repr = ut.indent(',\n'.join(repr_line_list), ' ' * 2) source_line_repr = ' [\n' + source_line_repr + '\n ]\n' return (cell_header + source_line_repr + cell_footer)
def check_installed(rman): import utool as ut label = ' %s' % rman.label if rman.label else rman.label missing = [] msg_list = [] print('Checking if%s modules are installed' % (label,)) for repo in rman.repos: flag, msg = repo.check_installed() if not flag: msg_list.append(' * !!!%s REPO %s NEEDS TO BE INSTALLED' % (label.upper(), repo,)) if ut.VERBOSE: msg_list.append(ut.indent(msg, ' ')) missing.append(repo) # else: # print(' * found%s module = %s' % (label, repo,)) print('\n'.join(msg_list)) return missing
def test_body(count, logmode, backspace): ut.colorprint('\n---- count = %r -----' % (count,), 'yellow') ut.colorprint('backspace = %r' % (backspace,), 'yellow') ut.colorprint('logmode = %r' % (logmode,), 'yellow') if logmode: ut.delete('test.log') ut.start_logging('test.log') print('Start main loop') import time for count in ut.ProgressIter(range(20), freq=3, backspace=backspace): time.sleep(.01) print('Done with main loop work') print('Exiting main body') if logmode: ut.stop_logging() #print('-----DONE LOGGING----') testlog_text = ut.readfrom('test.log') print(ut.indent(testlog_text.replace('\r', '\n'), ' '))
def test_body(count, logmode, backspace): ut.colorprint('\n---- count = %r -----' % (count, ), 'yellow') ut.colorprint('backspace = %r' % (backspace, ), 'yellow') ut.colorprint('logmode = %r' % (logmode, ), 'yellow') if logmode: ut.delete('test.log') ut.start_logging('test.log') print('Start main loop') import time for count in ut.ProgressIter(range(20), freq=3, backspace=backspace): time.sleep(.01) print('Done with main loop work') print('Exiting main body') if logmode: ut.stop_logging() #print('-----DONE LOGGING----') testlog_text = ut.readfrom('test.log') print(ut.indent(testlog_text.replace('\r', '\n'), ' '))
def __str__(allres): #print = tores.append ibs = allres.ibs toret = ('+======================\n') scalar_summary = str(allres.scalar_summary).strip() toret += ('| All Results: %s \n' % ibs.get_db_name()) toret += ('| title_suffix=%s\n' % str(allres.title_suffix)) toret += ('| scalar_summary=\n%s\n' % utool.indent(scalar_summary, '| ')) toret += ('| ' + str(allres.scalar_mAP_str)) toret += ('|---\n') toret += ('| greater5_%s \n' % (ibs.cidstr(allres.greater5_rids),)) toret += ('|---\n') toret += ('| greater1_%s \n' % (ibs.cidstr(allres.greater1_rids),)) toret += ('|---\n') toret += ('+======================.\n') #toret+=('| problem_false_pairs=\n%r' % allres.problem_false_pairs) #toret+=('| problem_true_pairs=\n%r' % allres.problem_true_pairs) return toret
def infer_monitor_specs(res_w, res_h, inches_diag): """ monitors = [ dict(name='work1', inches_diag=23, res_w=1920, res_h=1080), dict(name='work2', inches_diag=24, res_w=1920, res_h=1200), dict(name='hp-129', inches_diag=25, res_w=1920, res_h=1080), dict(name='?-26', inches_diag=26, res_w=1920, res_h=1080), dict(name='?-27', inches_diag=27, res_w=1920, res_h=1080), ] for info in monitors: name = info['name'] inches_diag = info['inches_diag'] res_h = info['res_h'] res_w = info['res_w'] print('---') print(name) inches_w = inches_diag * res_w / np.sqrt(res_h**2 + res_w**2) inches_h = inches_diag * res_h / np.sqrt(res_h**2 + res_w**2) print('inches diag = %.2f' % (inches_diag)) print('inches WxH = %.2f x %.2f' % (inches_w, inches_h)) #inches_w = inches_diag * res_w/sqrt(res_h**2 + res_w**2) """ import sympy # Build a system of equations and solve it inches_w, inches_h = sympy.symbols( 'inches_w inches_h'.split(), real=True, positive=True ) res_w, res_h = sympy.symbols('res_w res_h'.split(), real=True, positive=True) (inches_diag,) = sympy.symbols('inches_diag'.split(), real=True, positive=True) equations = [ sympy.Eq(inches_diag, (inches_w ** 2 + inches_h ** 2) ** 0.5), sympy.Eq(res_w / res_h, inches_w / inches_h), ] print('Possible solutions:') query_vars = [inches_w, inches_h] for solution in sympy.solve(equations, query_vars): print('Solution:') reprstr = ut.repr3( ut.odict(zip(query_vars, solution)), explicit=True, nobr=1, with_comma=False ) print(ut.indent(ut.autopep8_format(reprstr)))
def make_notebook(cell_list): """ References: # Change cell width http://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-ipython-notebook-in-my-browser/24207353#24207353 """ header = ut.codeblock( """ { "cells": [ """ ) footer = ut.codeblock( """ ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 0 } """) cell_body = ut.indent(',\n'.join(cell_list), ' ') notebook_str = header + '\n' + cell_body + '\n' + footer return notebook_str
def __str__(allres): #print = tores.append ibs = allres.ibs toret = ('+======================\n') scalar_summary = str(allres.scalar_summary).strip() toret += ('| All Results: %s \n' % ibs.get_db_name()) toret += ('| title_suffix=%s\n' % str(allres.title_suffix)) toret += ('| scalar_summary=\n%s\n' % utool.indent(scalar_summary, '| ')) toret += ('| ' + str(allres.scalar_mAP_str)) toret += ('|---\n') toret += ('| greater5_%s \n' % (ibs.cidstr(allres.greater5_rids), )) toret += ('|---\n') toret += ('| greater1_%s \n' % (ibs.cidstr(allres.greater1_rids), )) toret += ('|---\n') toret += ('+======================.\n') #toret+=('| problem_false_pairs=\n%r' % allres.problem_false_pairs) #toret+=('| problem_true_pairs=\n%r' % allres.problem_true_pairs) return toret
def handle_endmacro(matcher): """ this is quite a bit hacky, but this is the most straightforward way to implement them until the parse_cyth_preproc_markup/visit_FunctionDef 'coroutine' blob is refactored """ (macro_name,) = suspended_macro_context_ptr[0] lines = macro_input_buffer_ptr[0] #print('macro invokation of "%s" on lines %r' % (macro_name, lines)) expander = MACRO_EXPANDERS_DICT.get(macro_name, None) if expander: expanded_lines = ['\n'] + expander(self.gensym, lines) + ['\n'] else: errmsg = 'No macro named %r has been registered via the cyth.macro decorator' raise NotImplementedError(errmsg % macro_name) indented_expanded_lines = [utool.indent(x, indent=self.indent_with * (self.indentation + 1)) for x in expanded_lines] #print('output is %r' % expanded_lines) source_lines.extend(indented_expanded_lines) collect_macro_input_ptr[0] = False macro_input_buffer_ptr[0] = []
def handle_endmacro(matcher): """ this is quite a bit hacky, but this is the most straightforward way to implement them until the parse_cyth_preproc_markup/visit_FunctionDef 'coroutine' blob is refactored """ (macro_name, ) = suspended_macro_context_ptr[0] lines = macro_input_buffer_ptr[0] #print('macro invokation of "%s" on lines %r' % (macro_name, lines)) expander = MACRO_EXPANDERS_DICT.get(macro_name, None) if expander: expanded_lines = ['\n'] + expander(self.gensym, lines) + ['\n'] else: errmsg = 'No macro named %r has been registered via the cyth.macro decorator' raise NotImplementedError(errmsg % macro_name) indented_expanded_lines = [ utool.indent(x, indent=self.indent_with * (self.indentation + 1)) for x in expanded_lines ] #print('output is %r' % expanded_lines) source_lines.extend(indented_expanded_lines) collect_macro_input_ptr[0] = False macro_input_buffer_ptr[0] = []
def markdown_cell(markdown): r""" Args: markdown (str): Returns: str: json formatted ipython notebook markdown cell CommandLine: python -m ibeis.templates.generate_notebook --exec-markdown_cell Example: >>> # DISABLE_DOCTEST >>> from ibeis.templates.generate_notebook import * # NOQA >>> markdown = '# Title' >>> result = markdown_cell(markdown) >>> print(result) """ import utool as ut markdown_header = ut.codeblock( ''' { "cell_type": "markdown", "metadata": {}, "source": [ ''' ) markdown_footer = ut.codeblock( ''' ] } ''' ) return (markdown_header + '\n' + ut.indent(repr_single_for_md(markdown), ' ' * 2) + '\n' + markdown_footer)
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None, repodir=None, exclude_list=[]): """ Autogeneration function TODO move to util_autogen or just depricate Examples: >>> from utool.util_tests import * # NOQA >>> import utool # NOQA >>> testdirs = ['~/code/ibeis/test_ibs*.py'] """ import utool as ut from os.path import relpath, join, dirname # NOQA exclude_list += ['__init__.py'] # General format of the testing script script_fmtstr = ut.codeblock( r''' #!/bin/bash # Runs all tests # Win32 path hacks export CWD=$(pwd) export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")" # <CORRECT_PYTHON> # GET CORRECT PYTHON ON ALL PLATFORMS export SYSNAME="$(expr substr $(uname -s) 1 10)" if [ "$SYSNAME" = "MINGW32_NT" ]; then export PYEXE=python else if [ "$PYMAJOR" = "3" ]; then # virtual env? export PYEXE=python else export PYEXE=python2.7 fi fi # </CORRECT_PYTHON> PRINT_DELIMETER() {{ printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n" }} export TEST_ARGV="{test_argvs} $@" {dirdef_block} # Default tests to run set_test_flags() {{ export DEFAULT=$1 {testdefault_block} }} set_test_flags OFF {testdefaulton_block} # Parse for bash commandline args for i in "$@" do case $i in --testall) set_test_flags ON ;; esac {testcmdline_block} done BEGIN_TESTS() {{ cat <<EOF {runtests_bubbletext} EOF echo "BEGIN: TEST_ARGV=$TEST_ARGV" PRINT_DELIMETER num_passed=0 num_ran=0 export FAILED_TESTS='' }} RUN_TEST() {{ echo "RUN_TEST: $@" export TEST="$PYEXE $@ $TEST_ARGV" $TEST export RETURN_CODE=$? echo "RETURN_CODE=$RETURN_CODE" PRINT_DELIMETER num_ran=$(($num_ran + 1)) if [ "$RETURN_CODE" == "0" ] ; then num_passed=$(($num_passed + 1)) fi if [ "$RETURN_CODE" != "0" ] ; then export FAILED_TESTS="$FAILED_TESTS\n$TEST" fi }} END_TESTS() {{ echo "RUN_TESTS: DONE" if [ "$FAILED_TESTS" != "" ] ; then echo "-----" printf "Failed Tests:" printf "$FAILED_TESTS\n" printf "$FAILED_TESTS\n" >> failed_shelltests.txt echo "-----" fi echo "$num_passed / $num_ran tests passed" }} #--------------------------------------------- # START TESTS BEGIN_TESTS {quicktest_block} {test_block} #--------------------------------------------- # END TESTING END_TESTS ''') testcmdline_fmtstr = ut.codeblock( r''' case $i in --notest{header_lower}) export {testflag}=OFF ;; esac case $i in --test{header_lower}) export {testflag}=ON ;; esac ''') header_test_block_fmstr = ut.codeblock( r''' #--------------------------------------------- #{header_text} if [ "${testflag}" = "ON" ] ; then cat <<EOF {header_bubble_text} EOF {testlines_block} fi ''') #specialargv = '--noshow' specialargv = '' testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}' testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}' def format_testline(fpath, dirvar): if dirvar is None: return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv) else: return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv) default_flag_line_list = [] defaulton_flag_line_list = [] testcmdline_list = [] dirdef_list = [] header_test_block_list = [] known_tests = ut.ddict(list) # Tests to always run if quick_tests is not None: quicktest_block = '\n'.join( ['# Quick Tests (always run)'] + ['RUN_TEST ' + testline for testline in quick_tests]) else: quicktest_block = '# No quick tests' # Loop over different test types for testdef_tup in test_headers: header, default, modname, dpath, pats, testcmds = testdef_tup # Build individual test type information header_upper = header.upper() header_lower = header.lower() testflag = header_upper + '_TEST' if modname is not None: dirvar = header_upper + '_DIR' dirdef = ''.join([ 'export {dirvar}=$($PYEXE -c "', 'import os, {modname};', 'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))', '")']).format(dirvar=dirvar, modname=modname) dirdef_list.append(dirdef) else: dirvar = None # Build test dir #dirvar = header_upper + '_DIR' #dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname) #dirdef_list.append(dirdef) # Build command line flags default_flag_line = 'export {testflag}=$DEFAULT'.format(testflag=testflag) if default: defaulton_flag_line = 'export {testflag}=ON'.format(testflag=testflag) defaulton_flag_line_list.append(defaulton_flag_line) testcmdline_fmtdict = dict(header_lower=header_lower, testflag=testflag,) testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict) #ut.ls(dpath) # VERY HACK BIT OF CODE # Get list of tests from patterns if testcmds is None: if modname is not None: module = __import__(modname) repo_path = dirname(dirname(module.__file__)) else: repo_path = repodir dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath)) if header_upper == 'OTHER': # Hacky way to grab any other tests not explicitly seen in this directory _testfpath_list = list(set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_])) #_testfpath_list = ut.glob(dpath_, '*.py') #set(known_tests[dpath_]) else: _testfpath_list = ut.flatten([ut.glob(dpath_, pat) for pat in pats]) def not_excluded(x): return not any([x.find(exclude) > -1 for exclude in exclude_list]) _testfpath_list = list(filter(not_excluded, _testfpath_list)) known_tests[dpath_].extend(_testfpath_list) #print(_testfpath_list) testfpath_list = [util_path.unixjoin(dpath, relpath(fpath, dpath_)) for fpath in _testfpath_list] testline_list = [format_testline(fpath, dirvar) for fpath in testfpath_list] else: testline_list = testcmds testlines_block = ut.indentjoin(testline_list).strip('\n') # Construct test block for this type header_text = header_upper + ' TESTS' headerfont = 'cybermedium' header_bubble_text = ut.indent(ut.bubbletext(header_text, headerfont).strip()) header_test_block_dict = dict( testflag=testflag, header_text=header_text, testlines_block=testlines_block, header_bubble_text=header_bubble_text,) header_test_block = header_test_block_fmstr.format(**header_test_block_dict) # Append to script lists header_test_block_list.append(header_test_block) default_flag_line_list.append(default_flag_line) testcmdline_list.append(testcmdline) runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge') test_block = '\n'.join(header_test_block_list) dirdef_block = '\n'.join(dirdef_list) testdefault_block = ut.indent('\n'.join(default_flag_line_list)) testdefaulton_block = '\n'.join(defaulton_flag_line_list) testcmdline_block = '\n'.join(testcmdline_list) script_fmtdict = dict( quicktest_block=quicktest_block, runtests_bubbletext=runtests_bubbletext, test_argvs=test_argvs, dirdef_block=dirdef_block, testdefault_block=testdefault_block, testdefaulton_block=testdefaulton_block, testcmdline_block=testcmdline_block, test_block=test_block,) script_text = script_fmtstr.format(**script_fmtdict) return script_text
def make_default_module_maintest(modname, modpath=None): """ make_default_module_maintest TODO: use path relative to home dir if the file is a script Args: modname (str): module name Returns: str: text source code CommandLine: python -m utool.util_autogen --test-make_default_module_maintest References: http://legacy.python.org/dev/peps/pep-0338/ Example: >>> # ENABLE_DOCTEST >>> from utool.util_autogen import * # NOQA >>> modname = 'utool.util_autogen' >>> text = make_default_module_maintest(modname) >>> result = str(text) >>> print(result) """ import utool as ut # Need to use python -m to run a module # otherwise their could be odd platform specific errors. #python -c "import utool, {modname}; # ut.doctest_funcs({modname}, allexamples=True)" #in_pythonpath, module_type, path = find_modname_in_pythonpath(modname) # only use the -m if it is part of a package directory if modpath is not None: moddir = dirname(modpath) pkginit_fpath = join(moddir, '__init__.py') use_modrun = exists(pkginit_fpath) else: use_modrun = True if use_modrun: pyargs = '-m ' + modname else: if ut.WIN32: modpath = normpath(modpath).replace(expanduser('~'), '%HOME%') pyargs = '-B ' + ut.ensure_unixslash(modpath) else: modpath = normpath(modpath).replace(expanduser('~'), '~') pyargs = modpath cmdline = ut.codeblock( ''' python {pyargs} python {pyargs} --allexamples # REM python {pyargs} --allexamples --noface --nosrc ''') if not use_modrun: if ut.WIN32: augpath = 'set PYTHONPATH=%PYTHONPATH%' + os.pathsep + moddir else: augpath = 'export PYTHONPATH=$PYTHONPATH' + os.pathsep + moddir cmdline = augpath + '\n' + cmdline cmdline = ut.indent(cmdline, ' ' * 8).lstrip(' ').format(pyargs=pyargs) text = ut.codeblock( r''' # STARTBLOCK if __name__ == '__main__': r""" CommandLine: {cmdline} """ import multiprocessing multiprocessing.freeze_support() # for win32 import utool as ut # NOQA ut.doctest_funcs() # ENDBLOCK ''' ).format(cmdline=cmdline) text = remove_codeblock_syntax_sentinals(text) return text
def autogen_ibeis_runtest(): """ special case to generate tests script for IBEIS Example: >>> from autogen_test_script import * # NOQA >>> test_script = autogen_ibeis_runtest() >>> print(test_script) CommandLine: python -c "import utool; utool.autogen_ibeis_runtest()" python -c "import utool; print(utool.autogen_ibeis_runtest())" python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh chmod +x run_tests.sh """ quick_tests = ['ibeis/tests/assert_modules.py'] #test_repos = [ # '~/code/ibeis' # '~/code/vtool' # '~/code/hesaff' # '~/code/guitool' #] #test_pattern = [ # '~/code/ibeis/test_ibs*.py' #] test_argvs = '--quiet --noshow' misc_pats = [ 'test_utool_parallel.py', 'test_pil_hash.py', ] repodir = '~/code/ibeis' testdir = 'ibeis/tests' exclude_list = [] # Hacky, but not too bad way of getting in doctests # Test to see if doctest_funcs appears after main # Do not doctest these modules #exclude_doctests_fnames = set(['template_definitions.py', # 'autogen_test_script.py']) #exclude_dirs = [ # '_broken', # 'old', # 'tests', # 'timeits', # '_scripts', # '_timeits', # '_doc', # 'notebook', #] #dpath_list = ['ibeis'] #doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) # Verbosity to show which modules at least have some tests #untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs) #print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames)) #print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list)) # The implict list is exactly the code we will use to make the implicit list #module_list = None #doctest_modname_list = None #'export_subset.py', #'_autogen_ibeiscontrol_funcs.py', #'randomforest.py', implicit_build_modlist_str = ut.codeblock(''' import sys exclude_doctests_fnames = set([ 'template_definitions.py', 'autogen_test_script.py', ]) exclude_dirs = [ '_broken', 'old', 'tests', 'timeits', '_scripts', '_timeits', '_doc', 'notebook', ] dpath_list = ['ibeis'] doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) for modname in doctest_modname_list: exec('import ' + modname, globals(), locals()) module_list = [sys.modules[name] for name in doctest_modname_list] ''') globals_ = globals() locals_ = locals() exec(implicit_build_modlist_str, globals_, locals_) module_list = locals_['module_list'] doctest_modname_list = locals_['doctest_modname_list'] #module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list] #for modname in doctest_modname_list: # exec('import ' + modname, globals(), locals()) #module_list = [sys.modules[name] for name in doctest_modname_list] #print('\n'.join(testcmds)) #print('\n'.join(['python -m ' + modname for modname in doctest_modname_list])) import_str = '\n'.join( ['import ' + modname for modname in doctest_modname_list]) modlist_str = ( 'module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list])) explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str)) build_modlist_str = implicit_build_modlist_str #build_modlist_str = explicit_build_modlist_str pyscript_fmtstr = ut.codeblock(r''' #!/usr/bin/env python from __future__ import absolute_import, division, print_function import utool as ut def run_tests(): # Build module list and run tests {build_modlist_str} ut.doctest_module_list(module_list) if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() run_tests() ''') pyscript_text = pyscript_fmtstr.format( build_modlist_str=ut.indent(build_modlist_str).strip()) pyscript_text = ut.autofix_codeblock(pyscript_text) # BUILD OLD SHELL RUN TESTS HARNESS testcmds_ = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST', testslow=True) testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_] test_headers = [ # title, default, module, testpattern ut.def_test('VTOOL', dpath='vtool/tests', pat=['test*.py'], modname='vtool'), ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']), ut.def_test('IBEIS', dpath=testdir, pat=['test_ibs*.py', 'test_delete*.py'], default=False), ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']), ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']), ut.def_test('MISC', dpath=testdir, pat=misc_pats), ut.def_test('OTHER', dpath=testdir, pat='OTHER'), ut.def_test('HESAFF', dpath='pyhesaff/tests', pat=['test_*.py'], modname='pyhesaff'), ut.def_test('DOC', testcmds=testcmds, default=True) ] # Referencs: https://docs.python.org/2/library/runpy.html shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list) #print(pyscript_text) return shscript_text, pyscript_text
def print_layer_info(output_layer): str_ = get_layer_info_str(output_layer) str_ = ut.indent('[info] ' + str_) print('\n' + str_)
def autogen_parts(binding_name=None): r""" CommandLine: python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=set_dataset python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=add_points python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_point --py python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=used_memory --py python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_points --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=veclen --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=size --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=clean_removed_points --py --c Ignore: # Logic goes here ~/code/flann/src/cpp/flann/algorithms/kdtree_index.h ~/code/flann/src/cpp/flann/util/serialization.h ~/code/flann/src/cpp/flann/util/dynamic_bitset.h # Bindings go here ~/code/flann/src/cpp/flann/flann.cpp ~/code/flann/src/cpp/flann/flann.h # Contains stuff for the flann namespace like flann::log_level # Also has Index with # Matrix<ElementType> features; SEEMS USEFUL ~/code/flann/src/cpp/flann/flann.hpp # Wrappers go here ~/code/flann/src/python/pyflann/flann_ctypes.py ~/code/flann/src/python/pyflann/index.py ~/local/build_scripts/flannscripts/autogen_bindings.py Example: >>> # ENABLE_DOCTEST >>> from autogen_bindings import * # NOQA >>> result = autogen_parts() >>> print(result) """ #ut.get_dynlib_exports(pyflann.flannlib._name) #flannlib if binding_name is None: binding_name = ut.get_argval('--bindingname', type_=str, default='add_points') # Variable names used in flann cpp source simple_c_to_ctypes = { 'void' : None, 'char*' : 'c_char_p', 'unsigned int' : 'c_uint', 'int' : 'c_int', 'float' : 'c_float', 'float*' : 'POINTER(c_float)', 'flann_index_t' : 'FLANN_INDEX', 'FLANNParameters*' : 'POINTER(FLANNParameters)', 'Distance::ResultType*' : "ndpointer(%(restype)s, flags='aligned, c_contiguous, writeable')", 'Distance::ElementType*' : "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')", 'typename Distance::ElementType*' : "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')", 'typename Distance::ResultType*' : "ndpointer(%(restype), ndim=2, flags='aligned, c_contiguous, writeable')", } templated_ctype_map = { 'filename' : 'char*', 'level' : 'int', 'rows' : 'int', 'cols' : 'int', 'point_id' : 'unsigned int', 'num' : 'int', 'max_nn' : 'int', 'tcount' : 'int', 'nn' : 'int', 'radius' : 'float', 'clusters' : 'int', 'rebuild_threshold' : 'float', 'index_ptr' : 'flann_index_t', 'flann_params' : 'FLANNParameters*', 'speedup' : 'float*', 'id_list' : 'int*', #'indices' : 'int*', 'dataset' : 'typename Distance::ElementType*', 'points' : 'typename Distance::ElementType*', 'query' : 'typename Distance::ElementType*', 'query1d' : 'typename Distance::ElementType*', 'testset' : 'typename Distance::ElementType*', 'dists' : 'typename Distance::ResultType*', 'dists1d' : 'typename Distance::ResultType*', 'result_centers' : 'typename Distance::ResultType*', 'result_ids' : 'int*', } # Python ctype bindings python_ctype_map = { 'flann_params' : 'POINTER(FLANNParameters)', 'id_list' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous')", #'indices' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous, writeable')", 'query1d' : "ndpointer(%(numpy)s, ndim=1, flags='aligned, c_contiguous')", 'dists1d' : "ndpointer(%(restype), ndim=1, flags='aligned, c_contiguous, writeable')", 'result_ids' : "ndpointer(int32, ndim=2, flags='aligned, c_contiguous, writeable')", #'query' : "ndpointer(float64, ndim=1, flags='aligned, c_contiguous')", } for key, val in templated_ctype_map.items(): if key not in python_ctype_map: python_ctype_map[key] = simple_c_to_ctypes[val] binding_def = define_flann_bindings(binding_name) docstr_cpp = binding_def['docstr_cpp'] docstr_py = binding_def['docstr_py'] cpp_binding_name = binding_def['cpp_binding_name'] return_type = binding_def['return_type'] binding_argnames = binding_def['binding_argnames'] c_source = binding_def['c_source'] py_source = binding_def['py_source'] optional_args = binding_def['optional_args'] py_alias = binding_def['py_alias'] py_args = binding_def['py_args'] binding_args = [python_ctype_map[name] + ', # ' + name for name in binding_argnames] binding_args_str = ' ' + '\n '.join(binding_args) callargs = ', '.join(binding_argnames) pycallargs = ', '.join([name for name in ['self'] + binding_argnames if name != 'index_ptr']) if py_args is None: py_args = binding_argnames pyinputargs = pycallargs # FIXME else: pyinputargs = ', '.join(['self'] + py_args) pyrestype = simple_c_to_ctypes[return_type] if py_alias is None: py_binding_name = binding_name else: py_binding_name = py_alias binding_def['py_binding_name'] = py_binding_name #### flann_ctypes.py flann_ctypes_codeblock = ut.codeblock( ''' flann.{binding_name} = {{}} define_functions(r""" flannlib.flann_{binding_name}_%(C)s.restype = {pyrestype} flannlib.flann_{binding_name}_%(C)s.argtypes = [ {binding_args_str} ] flann.{binding_name}[%(numpy)s] = flannlib.flann_{binding_name}_%(C)s """) ''' ).format(binding_name=binding_name, binding_args_str=binding_args_str, pyrestype=pyrestype) #### index.py default_py_source_parts = [] if 'pts' in py_args: default_py_source_parts.append(ut.codeblock( ''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) ''')) default_py_source_parts.append(ut.codeblock( ''' rows = pts.shape[0] raise NotImplementedError('requires custom implementation') flann.{binding_name}[self.__curindex_type](self.__curindex, {pycallargs}) ''' )) if py_source is None: py_source = '\n'.join(default_py_source_parts) flann_index_codeblock = ut.codeblock( r''' def {py_binding_name}({pyinputargs}): """''' + ut.indent('\n' + docstr_py, ' ') + ''' """''' + '\n' + ut.indent(py_source, ' ') + ''' ''' ).format(binding_name=binding_name, pycallargs=pycallargs, py_binding_name=py_binding_name, pyinputargs=pyinputargs, py_source=py_source) #### flann.cpp #// {binding_name} BEGIN CPP BINDING #template <typename Distance> #{return_type} __flann_{binding_name}({templated_args}) flann_cpp_code_fmtstr_ = ut.codeblock( r''' {{''' + '\n' + ut.indent(c_source, ' ' * (4 * 3)) + r''' }} ''' ) #implicit_type_bindings_fmtstr = ut.codeblock( # NOQA # ''' # DISTANCE_TYPE_BINDINGS({return_type}, {binding_name}, # SINGLE_ARG({T_typed_sigargs_cpp}), # SINGLE_ARG({callargs})) # ''' #) #type_bindings_fmtstr = implicit_type_bindings_fmtstr explicit_type_bindings_part3_fmtstr = ut.codeblock( r''' {{ if (flann_distance_type==FLANN_DIST_EUCLIDEAN) {{ return __flann_{binding_name}<L2<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_MANHATTAN) {{ return __flann_{binding_name}<L1<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_MINKOWSKI) {{ return __flann_{binding_name}<MinkowskiDistance<T> >({callargs}{minkowski_option}); }} else if (flann_distance_type==FLANN_DIST_HIST_INTERSECT) {{ return __flann_{binding_name}<HistIntersectionDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_HELLINGER) {{ return __flann_{binding_name}<HellingerDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_CHI_SQUARE) {{ return __flann_{binding_name}<ChiSquareDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_KULLBACK_LEIBLER) {{ return __flann_{binding_name}<KL_Divergence<T> >({callargs}); }} else {{ Logger::error( "Distance type unsupported in the C bindings, use the C++ bindings instead\n"); {errorhandle} }} }} ''' ) # c binding body c_bodyblock_fmtstr = ut.codeblock( ''' {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2}) {{ {ifreturns}_flann_{binding_name}{iftemplate}({callargs}); }} ''' ) #### flann.h # c binding header #c_headersig_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2});' c_headersig_part1_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}(' #### format cpp parts binding_argtypes = ut.dict_take(templated_ctype_map, binding_argnames) _fix_T = { 'typename Distance::ElementType*': 'T*', 'typename Distance::ResultType*': 'R*', } templated_ctype_map_cpp = {name: _fix_T.get(type_, type_) for name, type_ in templated_ctype_map.items()} binding_argtypes_cpp = ut.dict_take(templated_ctype_map_cpp, binding_argnames) binding_sigargs_cpp = [type_ + ' ' + name for type_, name in zip(binding_argtypes_cpp, binding_argnames)] templated_bindings = [templated_ctype_map[name] + ' ' + name for name in binding_argnames] if optional_args is not None: templated_bindings += optional_args minkowski_option = ', MinkowskiDistance<T>(flann_distance_order)' else: minkowski_option = '' templated_args = ', '.join(templated_bindings) if binding_name == 'remove_point': # HACK templated_args += '_uint' cpp_sig_part1 = '{return_type} __flann_{binding_name}('.format(return_type=return_type, binding_name=binding_name) maxlen = 100 cpp_sig_ = ut.packstr(cpp_sig_part1 + templated_args, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(cpp_sig_part1)) cpp_sig = cpp_sig_[:-2] + ')' flann_cpp_code_fmtstr = 'template <typename Distance>\n' + cpp_sig + '\n' + flann_cpp_code_fmtstr_ #print(cpp_sig) T_typed_sigargs_cpp = ', '.join(binding_sigargs_cpp) used_template_list = [] used_template_list.append('typename T') if 'typename Distance::ResultType*' in binding_argtypes: used_template_list.append('typename R') used_templates = ', '.join(used_template_list) type_binding_part1 = 'template <{used_templates}>'.format(used_templates=used_templates) type_binding_part2_ = '{return_type} _flann_{binding_name}('.format(return_type=return_type, binding_name=binding_name) maxlen = 100 cpp_type_sig_ = ut.packstr(type_binding_part2_ + T_typed_sigargs_cpp, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(type_binding_part2_)) cpp_type_sig = cpp_type_sig_[:-2] + ')' type_binding_part12 = type_binding_part1 + '\n' + cpp_type_sig explicit_type_bindings_fmtstr = type_binding_part12 + '\n' + explicit_type_bindings_part3_fmtstr flann_cpp_codeblock_fmtstr = flann_cpp_code_fmtstr + '\n\n\n' + explicit_type_bindings_fmtstr + '\n' if return_type == 'int': errorhandle = 'return -1;' elif return_type == 'flann_index_t': errorhandle = 'return NULL;' else: errorhandle = 'throw 0;' # print('------') # print('flann_cpp_codeblock_fmtstr.format = %s' % (flann_cpp_codeblock_fmtstr,)) try: flann_cpp_codeblock = flann_cpp_codeblock_fmtstr.format( cpp_binding_name=cpp_binding_name, minkowski_option=minkowski_option, binding_name=binding_name, #templated_args=templated_args, callargs=callargs, #T_typed_sigargs_cpp=T_typed_sigargs_cpp, errorhandle=errorhandle, used_templates=used_templates, return_type=return_type) except KeyError as ex: ut.printex(ex, keys=['binding_name']) raise dataset_types = [ '', 'float', 'double', 'byte', 'int', ] #### format c parts c_header_sigs = [] c_body_blocks = [] templated_ctype_map_c = templated_ctype_map.copy() #templated_ctype_map_c['dataset'] = 'float' _fix_explicit_ctype = { '' : 'float', 'byte' : 'unsigned char', } # For each explicit c type for dataset_type in dataset_types: T_type = _fix_explicit_ctype.get(dataset_type, dataset_type) if dataset_type != '': signame_type = '_' + dataset_type else: signame_type = dataset_type R_type = 'float' if T_type != 'double' else 'double' dstype = T_type + '*' rstype = R_type + '*' # Overwrite template types with explicit c types needstemplate = True for type_, name in zip(binding_argtypes, binding_argnames): if type_ == 'typename Distance::ElementType*': templated_ctype_map_c[name] = dstype needstemplate = False if type_ == 'typename Distance::ResultType*': templated_ctype_map_c[name] = rstype needstemplate = False if type_ == 'Distance::ResultType*': templated_ctype_map_c[name] = rstype needstemplate = False #if type_ == 'struct FLANNParameters*': # # hack # templated_ctype_map_c[name] = 'FLANNParameters*' # HACK if binding_name == 'load_index' or binding_name == 'add_points': needstemplate = True if binding_name == 'build_index': needstemplate = True binding_argtypes2 = ut.dict_take(templated_ctype_map_c, binding_argnames) binding_sigargs2 = [type_ + ' ' + name for type_, name in zip(binding_argtypes2, binding_argnames)] T_typed_sigargs2 = ', '.join(binding_sigargs2) T_typed_sigargs2_nl = ',\n'.join(binding_sigargs2) if needstemplate: iftemplate = '<{T_type}>'.format(T_type=T_type) else: iftemplate = '' if return_type != 'void': ifreturns = 'return ' else: ifreturns = '' bodyblock = c_bodyblock_fmtstr.format(signame_type=signame_type, T_typed_sigargs2=T_typed_sigargs2, binding_name=binding_name, callargs=callargs, iftemplate=iftemplate, ifreturns=ifreturns, return_type=return_type) header_line_part1 = c_headersig_part1_fmtstr.format(signame_type=signame_type, binding_name=binding_name, return_type=return_type) header_line = header_line_part1 + ut.indent(T_typed_sigargs2_nl, ' ' * len(header_line_part1)).lstrip(' ') + ');' # Hack for header header_line = header_line.replace('FLANNParameters* flann_params', 'struct FLANNParameters* flann_params') #header_line = c_headersig_fmtstr.format(signame_type=signame_type, # T_typed_sigargs2=T_typed_sigargs2, # binding_name=binding_name, # return_type=return_type) c_header_sigs.append(header_line) c_body_blocks.append(bodyblock) flann_cpp_codeblock += '\n' + '\n'.join(c_body_blocks) #flann_cpp_codeblock += '\n' + '// {binding_name} END'.format(binding_name=binding_name) #BEGIN {binding_name} flann_h_codeblock = ut.codeblock( r''' /** {docstr_cpp} */ ''' ).format(docstr_cpp=docstr_cpp, binding_name=binding_name) flann_h_codeblock += '\n\n' + '\n\n'.join(c_header_sigs) blocks_dict = {} import re flann_index_codeblock = ut.indent(flann_index_codeblock, ' ') blocks_dict['flann_ctypes.py'] = flann_ctypes_codeblock blocks_dict['index.py'] = flann_index_codeblock blocks_dict['flann.h'] = flann_h_codeblock blocks_dict['flann.cpp'] = flann_cpp_codeblock for key in blocks_dict.keys(): blocks_dict[key] = re.sub('\n\s+\n', '\n\n', blocks_dict[key]) # , flags=re.MULTILINE) blocks_dict[key] = re.sub('\s\s+\n', '\n', blocks_dict[key]) pass if ut.get_argflag('--py'): print('\n\n# ---------------\n\n') print('GOES IN flann_ctypes.py') print('\n\n# ---------------\n\n') print(flann_ctypes_codeblock) print('\n\n# ---------------\n\n') print('GOES IN index.py') print('\n\n# ---------------\n\n') print(flann_index_codeblock) if ut.get_argflag('--c'): print('\n\n# ---------------\n\n') print('GOES IN flann.h') print('\n\n# ---------------\n\n') print(flann_h_codeblock) print('\n\n# ---------------\n\n') print('GOES IN flann.cpp') print('\n\n# ---------------\n\n') print(flann_cpp_codeblock) return blocks_dict, binding_def
def get_dbinfo(ibs, verbose=True, with_imgsize=False, with_bytes=False, with_contrib=False, with_agesex=False, with_header=True, short=False, tag='dbinfo', aid_list=None): """ Returns dictionary of digestable database information Infostr is a string summary of all the stats. Prints infostr in addition to returning locals Args: ibs (IBEISController): verbose (bool): with_imgsize (bool): with_bytes (bool): Returns: dict: CommandLine: python -m ibeis.other.dbinfo --exec-get_dbinfo:0 python -m ibeis.other.dbinfo --test-get_dbinfo:1 python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3 python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1 python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1 python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a ctrl python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0 python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0 Example1: >>> # SCRIPT >>> from ibeis.other.dbinfo import * # NOQA >>> import ibeis >>> defaultdb = 'testdb1' >>> ibs, aid_list = ibeis.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1') >>> kwargs = ut.get_kwdefaults(get_dbinfo) >>> kwargs['verbose'] = False >>> kwargs['aid_list'] = aid_list >>> kwargs = ut.parse_dict_from_argv(kwargs) >>> output = get_dbinfo(ibs, **kwargs) >>> result = (output['info_str']) >>> print(result) >>> #ibs = ibeis.opendb(defaultdb='testdb1') >>> # <HACK FOR FILTERING> >>> #from ibeis.expt import cfghelpers >>> #from ibeis.expt import annotation_configs >>> #from ibeis.init import filter_annots >>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__, >>> # annotation_configs.TEST_NAMES) >>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES, >>> # ut.get_list_column(named_defaults_dict, 'qcfg'))) >>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0] >>> #aid_list = ibs.get_valid_aids() >>> # </HACK FOR FILTERING> Example1: >>> # ENABLE_DOCTEST >>> from ibeis.other.dbinfo import * # NOQA >>> import ibeis >>> verbose = True >>> short = True >>> #ibs = ibeis.opendb(db='GZ_ALL') >>> #ibs = ibeis.opendb(db='PZ_Master0') >>> ibs = ibeis.opendb('testdb1') >>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS' >>> ibs.delete_contributors(ibs.get_valid_contrib_rowids()) >>> ibs.delete_empty_nids() >>> #ibs = ibeis.opendb(db='PZ_MTEST') >>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True) >>> result = (output['info_str']) >>> print(result) +============================ DB Info: testdb1 DB Notes: None DB NumContrib: 0 ---------- # Names = 7 # Names (unassociated) = 0 # Names (singleton) = 5 # Names (multiton) = 2 ---------- # Annots = 13 # Annots (unknown) = 4 # Annots (singleton) = 5 # Annots (multiton) = 4 ---------- # Img = 13 L============================ """ # TODO Database size in bytes # TODO: occurrence, contributors, etc... # Basic variables request_annot_subset = False _input_aid_list = aid_list # NOQA if aid_list is None: valid_aids = ibs.get_valid_aids() valid_nids = ibs.get_valid_nids() valid_gids = ibs.get_valid_gids() else: if isinstance(aid_list, str): # Hack to get experiment stats on aids acfg_name_list = [aid_list] print('Specified custom aids via acfgname %s' % (acfg_name_list,)) from ibeis.expt import experiment_helpers acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list( ibs, acfg_name_list) aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list))))) #aid_list = if verbose: print('Specified %d custom aids' % (len(aid_list,))) request_annot_subset = True valid_aids = aid_list valid_nids = list( set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False)) - {const.UNKNOWN_NAME_ROWID} ) valid_gids = list(set(ibs.get_annot_gids(aid_list))) #associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation FILTER_HACK = True if FILTER_HACK: # HUGE HACK - get only images and names with filtered aids valid_aids_ = ibs.filter_aids_custom(valid_aids) valid_nids_ = ibs.filter_nids_custom(valid_nids) valid_gids_ = ibs.filter_gids_custom(valid_gids) if verbose: print('Filtered %d names' % (len(valid_nids) - len(valid_nids_))) print('Filtered %d images' % (len(valid_gids) - len(valid_gids_))) print('Filtered %d annots' % (len(valid_aids) - len(valid_aids_))) valid_gids = valid_gids_ valid_nids = valid_nids_ valid_aids = valid_aids_ #associated_nids = ut.compress(associated_nids, map(any, #ibs.unflat_map(ibs.get_annot_custom_filterflags, # ibs.get_name_aids(associated_nids)))) # Image info if verbose: print('Checking Image Info') gx2_aids = ibs.get_image_aids(valid_gids) if FILTER_HACK: gx2_aids = [ibs.filter_aids_custom(aids) for aids in gx2_aids] # HACK FOR FILTER if request_annot_subset: # remove annots not in this subset valid_aids_set = set(valid_aids) gx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in gx2_aids] gx2_nAnnots = np.array(list(map(len, gx2_aids))) image_without_annots = len(np.where(gx2_nAnnots == 0)[0]) gx2_nAnnots_stats = ut.get_stats_str(gx2_nAnnots, newlines=True, use_median=True) image_reviewed_list = ibs.get_image_reviewed(valid_gids) # Name stats if verbose: print('Checking Name Info') nx2_aids = ibs.get_name_aids(valid_nids) if FILTER_HACK: nx2_aids = [ibs.filter_aids_custom(aids) for aids in nx2_aids] # HACK FOR FILTER if request_annot_subset: # remove annots not in this subset valid_aids_set = set(valid_aids) nx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in nx2_aids] associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids))) ibs.check_name_mapping_consistency(nx2_aids) # Occurrence Info def compute_annot_occurrence_ids(ibs, aid_list): from ibeis.algo.preproc import preproc_occurrence gid_list = ibs.get_annot_gids(aid_list) gid2_aids = ut.group_items(aid_list, gid_list) flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False) occurid2_gids = ut.group_items(flat_gids, flat_imgsetids) occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()} return occurid2_aids import utool with utool.embed_on_exception_context: occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids) occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values()) occur_unique_nids = [ut.unique(nids) for nids in occur_nids] nid2_occurxs = ut.ddict(list) for occurx, nids in enumerate(occur_unique_nids): for nid in nids: nid2_occurxs[nid].append(occurx) nid2_occurx_single = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1} nid2_occurx_resight = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1} singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys()) singlesight_annot_stats = ut.get_stats(list(map(len, singlesight_encounters)), use_median=True, use_sum=True) resight_name_stats = ut.get_stats(list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True) try: aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0) undirected_tags = ibs.get_aidpair_tags(aid_pairs.T[0], aid_pairs.T[1], directed=False) tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags)) tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags) pair_tag_info = ut.map_dict_vals(len, tag_dict) num_reviewed_pairs = sum(ibs.get_annot_pair_is_reviewed(aid_pairs.T[0], aid_pairs.T[1])) pair_tag_info['num_reviewed'] = num_reviewed_pairs except Exception: pair_tag_info = {} #print(ut.dict_str(pair_tag_info)) # Annot Stats # TODO: number of images where chips cover entire image # TODO: total image coverage of annotation # TODO: total annotation overlap """ ax2_unknown = ibs.is_aid_unknown(valid_aids) ax2_nid = ibs.get_annot_name_rowids(valid_aids) assert all([nid < 0 if unknown else nid > 0 for nid, unknown in zip(ax2_nid, ax2_unknown)]), 'bad annot nid' """ # if verbose: print('Checking Annot Species') unknown_aids = ut.compress(valid_aids, ibs.is_aid_unknown(valid_aids)) species_list = ibs.get_annot_species_texts(valid_aids) species2_aids = ut.group_items(valid_aids, species_list) species2_nAids = {key: len(val) for key, val in species2_aids.items()} if verbose: print('Checking Multiton/Singleton Species') nx2_nAnnots = np.array(list(map(len, nx2_aids))) # Seperate singleton / multitons multiton_nxs = np.where(nx2_nAnnots > 1)[0] singleton_nxs = np.where(nx2_nAnnots == 1)[0] unassociated_nxs = np.where(nx2_nAnnots == 0)[0] assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names' valid_nxs = np.hstack([multiton_nxs, singleton_nxs]) num_names_with_gt = len(multiton_nxs) # Annot Info if verbose: print('Checking Annot Info') multiton_aids_list = ut.take(nx2_aids, multiton_nxs) assert len(set(multiton_nxs)) == len(multiton_nxs) if len(multiton_aids_list) == 0: multiton_aids = np.array([], dtype=np.int) else: multiton_aids = np.hstack(multiton_aids_list) assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot' singleton_aids = ut.take(nx2_aids, singleton_nxs) multiton_nid2_nannots = list(map(len, multiton_aids_list)) # Image size stats if with_imgsize: if verbose: print('Checking ImageSize Info') gpath_list = ibs.get_image_paths(valid_gids) def wh_print_stats(wh_list): if len(wh_list) == 0: return '{empty}' wh_list = np.asarray(wh_list) stat_dict = OrderedDict( [( 'max', wh_list.max(0)), ( 'min', wh_list.min(0)), ('mean', wh_list.mean(0)), ( 'std', wh_list.std(0))]) def arr2str(var): return ('[' + ( ', '.join(list(map(lambda x: '%.1f' % x, var))) ) + ']') ret = (',\n '.join([ '%s:%s' % (key, arr2str(val)) for key, val in stat_dict.items() ])) return '{\n ' + ret + '\n}' print('reading image sizes') # Image size stats img_size_list = ibs.get_image_sizes(valid_gids) img_size_stats = wh_print_stats(img_size_list) # Chip size stats annotation_bbox_list = ibs.get_annot_bboxes(valid_aids) annotation_bbox_arr = np.array(annotation_bbox_list) if len(annotation_bbox_arr) == 0: annotation_size_list = [] else: annotation_size_list = annotation_bbox_arr[:, 2:4] chip_size_stats = wh_print_stats(annotation_size_list) imgsize_stat_lines = [ (' # Img in dir = %d' % len(gpath_list)), (' Image Size Stats = %s' % (img_size_stats,)), (' * Chip Size Stats = %s' % (chip_size_stats,)), ] else: imgsize_stat_lines = [] if verbose: print('Building Stats String') multiton_stats = ut.get_stats_str(multiton_nid2_nannots, newlines=True, use_median=True) # Time stats unixtime_list = ibs.get_image_unixtime(valid_gids) unixtime_list = ut.list_replace(unixtime_list, -1, float('nan')) #valid_unixtime_list = [time for time in unixtime_list if time != -1] #unixtime_statstr = ibs.get_image_time_statstr(valid_gids) if ut.get_argflag('--hackshow-unixtime'): show_time_distributions(ibs, unixtime_list) ut.show_if_requested() unixtime_statstr = ut.get_timestats_str(unixtime_list, newlines=True, full=True) # GPS stats gps_list_ = ibs.get_image_gps(valid_gids) gpsvalid_list = [gps != (-1, -1) for gps in gps_list_] gps_list = ut.compress(gps_list_, gpsvalid_list) def get_annot_age_stats(aid_list): annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list) annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list) age_dict = ut.ddict((lambda : 0)) for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max): if (min_age is None or min_age < 12) and max_age < 12: age_dict['Infant'] += 1 elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36: age_dict['Juvenile'] += 1 elif 36 <= min_age and (36 <= max_age or max_age is None): age_dict['Adult'] += 1 else: print('Found UNKNOWN Age: %r, %r' % (min_age, max_age, )) age_dict['UNKNOWN'] += 1 return age_dict def get_annot_sex_stats(aid_list): annot_sextext_list = ibs.get_annot_sex_texts(aid_list) sextext2_aids = ut.group_items(aid_list, annot_sextext_list) sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys()) assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(set(annot_sextext_list) - set(sex_keys)) sextext2_nAnnots = ut.odict([(key, len(sextext2_aids.get(key, []))) for key in sex_keys]) # Filter 0's sextext2_nAnnots = {key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0} return sextext2_nAnnots if verbose: print('Checking Other Annot Stats') qualtext2_nAnnots = ibs.get_annot_qual_stats(valid_aids) yawtext2_nAnnots = ibs.get_annot_yaw_stats(valid_aids) agetext2_nAnnots = get_annot_age_stats(valid_aids) sextext2_nAnnots = get_annot_sex_stats(valid_aids) if verbose: print('Checking Contrib Stats') # Contributor Statistics # hack remove colon for image alignment def fix_tag_list(tag_list): return [None if tag is None else tag.replace(':', ';') for tag in tag_list] image_contrib_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids)) annot_contrib_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids)) contrib_tag_to_gids = ut.group_items(valid_gids, image_contrib_tags) contrib_tag_to_aids = ut.group_items(valid_aids, annot_contrib_tags) contrib_tag_to_qualstats = {key: ibs.get_annot_qual_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)} contrib_tag_to_viewstats = {key: ibs.get_annot_yaw_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)} contrib_tag_to_nImages = {key: len(val) for key, val in six.iteritems(contrib_tag_to_gids)} contrib_tag_to_nAnnots = {key: len(val) for key, val in six.iteritems(contrib_tag_to_aids)} if verbose: print('Summarizing') # Summarize stats num_names = len(valid_nids) num_names_unassociated = len(valid_nids) - len(associated_nids) num_names_singleton = len(singleton_nxs) num_names_multiton = len(multiton_nxs) num_singleton_annots = len(singleton_aids) num_multiton_annots = len(multiton_aids) num_unknown_annots = len(unknown_aids) num_annots = len(valid_aids) if with_bytes: if verbose: print('Checking Disk Space') ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir())) dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir())) imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir())) cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir())) if True: if verbose: print('Check asserts') try: bad_aids = np.intersect1d(multiton_aids, unknown_aids) _num_names_total_check = num_names_singleton + num_names_unassociated + num_names_multiton _num_annots_total_check = num_unknown_annots + num_singleton_annots + num_multiton_annots assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids' assert _num_names_total_check == num_names, 'inconsistent num names' #if not request_annot_subset: # dont check this if you have an annot subset assert _num_annots_total_check == num_annots, 'inconsistent num annots' except Exception as ex: ut.printex(ex, keys=[ '_num_names_total_check', 'num_names', '_num_annots_total_check', 'num_annots', 'num_names_singleton', 'num_names_multiton', 'num_unknown_annots', 'num_multiton_annots', 'num_singleton_annots', ]) raise # Get contributor statistics contrib_rowids = ibs.get_valid_contrib_rowids() num_contributors = len(contrib_rowids) # print num_tabs = 5 def align2(str_): return ut.align(str_, ':', ' :') def align_dict2(dict_): str_ = ut.dict_str(dict_) return align2(str_) header_block_lines = ( [('+============================'), ] + ( [ ('+ singleton := single sighting'), ('+ multiton := multiple sightings'), ('--' * num_tabs), ] if not short and with_header else [] ) ) source_block_lines = [ ('DB Info: ' + ibs.get_dbname()), ('DB Notes: ' + ibs.get_dbnotes()), ('DB NumContrib: %d' % num_contributors), ] bytes_block_lines = [ ('--' * num_tabs), ('DB Bytes: '), (' +- dbdir nBytes: ' + dbdir_space), (' | +- _ibsdb nBytes: ' + ibsdir_space), (' | | +-imgdir nBytes: ' + imgdir_space), (' | | +-cachedir nBytes: ' + cachedir_space), ] if with_bytes else [] name_block_lines = [ ('--' * num_tabs), ('# Names = %d' % num_names), ('# Names (unassociated) = %d' % num_names_unassociated), ('# Names (singleton) = %d' % num_names_singleton), ('# Names (multiton) = %d' % num_names_multiton), ] subset_str = ' ' if not request_annot_subset else '(SUBSET)' annot_block_lines = [ ('--' * num_tabs), ('# Annots %s = %d' % (subset_str, num_annots,)), ('# Annots (unknown) = %d' % num_unknown_annots), ('# Annots (singleton) = %d' % num_singleton_annots), ('# Annots (multiton) = %d' % num_multiton_annots), ] annot_per_basic_block_lines = [ ('--' * num_tabs), ('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)), ('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)), ('# Annots per Species = %s' % (align_dict2(species2_nAids),)), ] if not short else [] occurrence_block_lines = [ ('--' * num_tabs), ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)), ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)), ('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)), ] if not short else [] annot_per_qualview_block_lines = [ None if short else '# Annots per Viewpoint = %s' % align_dict2(yawtext2_nAnnots), None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots), ] annot_per_agesex_block_lines = [ '# Annots per Age = %s' % align_dict2(agetext2_nAnnots), '# Annots per Sex = %s' % align_dict2(sextext2_nAnnots), ] if not short and with_agesex else [] contrib_block_lines = [ '# Images per contributor = ' + align_dict2(contrib_tag_to_nImages), '# Annots per contributor = ' + align_dict2(contrib_tag_to_nAnnots), '# Quality per contributor = ' + ut.dict_str(contrib_tag_to_qualstats, sorted_=True), '# Viewpoint per contributor = ' + ut.dict_str(contrib_tag_to_viewstats, sorted_=True), ] if with_contrib else [] img_block_lines = [ ('--' * num_tabs), ('# Img = %d' % len(valid_gids)), None if short else ('# Img reviewed = %d' % sum(image_reviewed_list)), None if short else ('# Img with gps = %d' % len(gps_list)), #('# Img with timestamp = %d' % len(valid_unixtime_list)), None if short else ('Img Time Stats = %s' % (align2(unixtime_statstr),)), ] info_str_lines = ( header_block_lines + bytes_block_lines + source_block_lines + name_block_lines + annot_block_lines + annot_per_basic_block_lines + occurrence_block_lines + annot_per_qualview_block_lines + annot_per_agesex_block_lines + img_block_lines + contrib_block_lines + imgsize_stat_lines + [('L============================'), ] ) info_str = '\n'.join(ut.filter_Nones(info_str_lines)) info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag)) if verbose: print(info_str2) locals_ = locals() return locals_
def outline(self, indent): import utool as ut yield indent + self.__nice__() indent = indent + ' ' * 4 yield ut.indent(ut.format_single_paragraph_sentences(self.resolve()), indent)
def autogen_ibeis_runtest(): """ special case to generate tests script for IBEIS Example: >>> from autogen_test_script import * # NOQA >>> test_script = autogen_ibeis_runtest() >>> print(test_script) CommandLine: python -c "import utool; utool.autogen_ibeis_runtest()" python -c "import utool; print(utool.autogen_ibeis_runtest())" python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh chmod +x run_tests.sh """ quick_tests = ['ibeis/tests/assert_modules.py'] #test_repos = [ # '~/code/ibeis' # '~/code/vtool' # '~/code/hesaff' # '~/code/guitool' #] #test_pattern = [ # '~/code/ibeis/test_ibs*.py' #] test_argvs = '--quiet --noshow' misc_pats = [ 'test_utool_parallel.py', 'test_pil_hash.py', ] repodir = '~/code/utool' exclude_list = [] # Verbosity to show which modules at least have some tests #untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs) #print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames)) #print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list)) implicit_build_modlist_str = ut.codeblock(''' import sys exclude_doctests_fnames = set(['__init__.py']) exclude_dirs = [ '_broken', 'old', 'tests', 'timeits', '_scripts', '_timeits', '_doc', 'notebook', ] dpath_list = ['utool'] doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) for modname in doctest_modname_list: exec('import ' + modname, globals(), locals()) module_list = [sys.modules[name] for name in doctest_modname_list] ''') globals_ = globals() locals_ = locals() exec(implicit_build_modlist_str, globals_, locals_) module_list = locals_['module_list'] doctest_modname_list = locals_['doctest_modname_list'] import_str = '\n'.join( ['import ' + modname for modname in doctest_modname_list]) modlist_str = ( 'module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list])) explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str)) build_modlist_str = implicit_build_modlist_str #build_modlist_str = explicit_build_modlist_str pyscript_fmtstr = ut.codeblock(r''' #!/usr/bin/env python from __future__ import absolute_import, division, print_function import utool as ut def run_tests(): # Build module list and run tests {build_modlist_str} ut.doctest_module_list(module_list) if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() run_tests() ''') pyscript_text = pyscript_fmtstr.format( build_modlist_str=ut.indent(build_modlist_str).strip()) pyscript_text = ut.autofix_codeblock(pyscript_text) def def_test(header, pat=None, dpath=None, modname=None, default=False, testcmds=None): """ interface to make test tuple """ return (header, default, modname, dpath, pat, testcmds) # BUILD OLD SHELL RUN TESTS HARNESS testcmds = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST') test_headers = [ # title, default, module, testpattern def_test('DOC', testcmds=testcmds, default=True) ] shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list) return shscript_text, pyscript_text
def define_flann_bindings(binding_name): """ Define the binding names for flann """ # default c source c_source = None optional_args = None c_source_part = None py_source = None py_alias = None py_args = None pydoc = None cpp_param_doc = { 'cols': 'number of columns in the dataset (feature dimensionality)', 'dataset': 'pointer to a data set stored in row major order', 'dists': ut.packtext( '''pointer to matrix for the distances of the nearest neighbors of the testset features in the dataset'''), 'flann_params': 'generic flann parameters', 'index_ptr': 'the index (constructed previously using flann_build_index)', 'nn': 'how many nearest neighbors to return', 'rebuild_threshold': ut.packtext( '''reallocs index when it grows by factor of `rebuild_threshold`. A smaller value results is more space efficient but less computationally efficient. Must be greater than 1.'''), 'result_ids': ut.packtext( '''pointer to matrix for the indices of the nearest neighbors of the testset features in the dataset (must have tcount number of rows and nn number of columns)'''), 'rows': 'number of rows (features) in the dataset', 'tcount': ut.packtext( '''number of rows (features) in the query dataset (same dimensionality as features in the dataset)'''), 'testset': 'pointer to a query set stored in row major order', 'level': 'verbosity level' } standard_csource = ut.codeblock( r''' try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; return index->{cpp_binding_name}(); }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); throw; }} ''' ) return_doc = None cpp_binding_name = binding_name zero_success = 'zero or a number <0 for error' if binding_name == 'clean_removed_points': cpp_binding_name = ut.to_camel_case(binding_name) return_type = 'void' docstr = 'Deletes removed points in index?' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'veclen': return_type = 'int' docstr = 'Returns number of features in this index' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'size': return_type = 'int' docstr = 'returns The dimensionality of the features in this index.' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'getType': return_type = 'flann_algorithm_t' docstr = 'returns The index type (kdtree, kmeans,...)' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'used_memory': docstr = ut.codeblock( ''' Returns the amount of memory (in bytes) used by the index index_ptr = pointer to pre-built index. Returns: int ''' ) c_source = ut.codeblock( r''' try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; return index->usedMemory(); }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''' ) py_source = ut.codeblock( ''' if self.__curindex is None: return 0 return flann.used_memory[self.__curindex_type](self.__curindex) ''') binding_argnames = ['index_ptr'] return_type = 'int' elif binding_name == 'add_points': c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<ElementType> points = Matrix<ElementType>(points, rows, index->veclen()); index->addPoints(points, rebuild_threshold); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''' ) py_source = ut.codeblock( ''' if new_pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % new_pts.dtype) if new_pts.dtype != self.__curindex_type: raise FLANNException('New points must have the same type') new_pts = ensure_2d_array(new_pts, default_flags) rows = new_pts.shape[0] flann.add_points[self.__curindex_type](self.__curindex, new_pts, rows, rebuild_threshold) return self.__added_data.append(new_pts) ''') #return_type = 'void' return_type = 'int' docstr = 'Adds points to pre-built index.' if False: binding_argnames = [ 'index_ptr', 'points', 'rows', 'cols', # TODO: can remove 'rebuild_threshold', ] else: binding_argnames = [ 'index_ptr', 'points', 'rows', 'rebuild_threshold', ] return_doc = '0 if success otherwise -1' py_args = ['new_pts', 'rebuild_threshold=2.'] cpp_param_doc['points'] = 'pointer to array of points' elif binding_name == 'remove_point': c_source = ut.codeblock( r''' size_t point_id(point_id_uint); try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; index->removePoint(point_id); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''' ) py_source = ut.codeblock( ''' flann.remove_point[self.__curindex_type](self.__curindex, point_id) self.__removed_ids.append(point_id) ''') #return_type = 'void' return_type = 'int' docstr = 'Removes a point from the index' return_doc = zero_success cpp_param_doc['point_id'] = 'point id to be removed' cpp_param_doc['index_ptr'] = 'The index that should be modified' binding_argnames = ['index_ptr', 'point_id'] elif binding_name == 'remove_points': c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; try {{ if (index_ptr==NULL) {{ thow FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; index->removePoints(id_list, num); return; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return; }} ''' ) py_source = ut.codeblock( ''' id_list = np.array(id_list, dtype=np.int32) num = len(id_list) flann.remove_points[self.__curindex_type](self.__curindex, id_list, num) self.__removed_ids.extend(id_list) ''') cpp_param_doc['index_ptr'] = 'The index that should be modified' cpp_param_doc['id_list'] = 'list of point ids to be removed' cpp_param_doc['num'] = 'number of points in id_list' docstr = 'Removes multiple points from the index' return_doc = 'void' py_args = ['id_list'] return_type = 'void' binding_argnames = ['index_ptr', 'id_list', 'num'] elif binding_name == 'compute_cluster_centers': docstr = ut.textblock( r''' Clusters the features in the dataset using a hierarchical kmeans clustering approach. This is significantly faster than using a flat kmeans clustering for a large number of clusters. ''') c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try { init_flann_parameters(flann_params); Matrix<ElementType> inputData(dataset,rows,cols); KMeansIndexParams params(flann_params->branching, flann_params->iterations, flann_params->centers_init, flann_params->cb_index); Matrix<DistanceType> centers(result_centers, clusters,cols); int clusterNum = hierarchicalClustering<Distance>(inputData, centers, params, d); return clusterNum; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return -1; } '''.replace('{', '{{').replace('}', '}}') ) py_source = ut.codeblock( ''' # First verify the paremeters are sensible. if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if int(branch_size) != branch_size or branch_size < 2: raise FLANNException('branch_size must be an integer >= 2.') branch_size = int(branch_size) if int(num_branches) != num_branches or num_branches < 1: raise FLANNException('num_branches must be an integer >= 1.') num_branches = int(num_branches) if max_iterations is None: max_iterations = -1 else: max_iterations = int(max_iterations) # init the arrays and starting values pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape num_clusters = (branch_size - 1) * num_branches + 1 if pts.dtype.type == np.float64: result = np.empty((num_clusters, dim), dtype=np.float64) else: result = np.empty((num_clusters, dim), dtype=np.float32) # set all the parameters appropriately self.__ensureRandomSeed(kwargs) params = {'iterations': max_iterations, 'algorithm': 'kmeans', 'branching': branch_size, 'random_seed': kwargs['random_seed']} self.__flann_parameters.update(params) numclusters = flann.compute_cluster_centers[pts.dtype.type]( pts, npts, dim, num_clusters, result, pointer(self.__flann_parameters)) if numclusters <= 0: raise FLANNException('Error occured during clustering procedure.') if dtype is None: return result else: return dtype(result) ''').replace('}', '}}').replace('{', '{{') return_doc = ut.packtext( '''number of clusters computed or a number <0 for error. This number can be different than the number of clusters requested, due to the way hierarchical clusters are computed. The number of clusters returned will be the highest number of the form (branch_size-1)*K+1 smaller than the number of clusters requested.''') cpp_param_doc['clusters'] = 'number of cluster to compute' cpp_param_doc['result_centers'] = 'memory buffer where the output cluster centers are stored' cpp_param_doc['flann_params'] = 'generic flann parameters and index_params used to specify the kmeans tree parameters (branching factor, max number of iterations to use)' return_type = 'int' binding_argnames = ['dataset', 'rows', 'cols', 'clusters', 'result_centers', 'flann_params'] optional_args = ['Distance d = Distance()'] py_alias = 'hierarchical_kmeans' py_args = 'pts, branch_size, num_branches, max_iterations=None, dtype=None, **kwargs'.split(', ') elif binding_name == 'radius_search': docstr = ut.codeblock( r''' Performs an radius search using an already constructed index. In case of radius search, instead of always returning a predetermined number of nearest neighbours (for example the 10 nearest neighbours), the search will return all the neighbours found within a search radius of the query point. The check parameter in the FLANNParameters below sets the level of approximation for the search by only visiting "checks" number of features in the index (the same way as for the KNN search). A lower value for checks will give a higher search speedup at the cost of potentially not returning all the neighbours in the specified radius. ''') c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try {{ init_flann_parameters(flann_params); if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_result_ids(result_ids, 1, max_nn); Matrix<DistanceType> m_dists(dists1d, 1, max_nn); SearchParams search_params = create_search_params(flann_params); int count = index->radiusSearch(Matrix<ElementType>(query1d, 1, index->veclen()), m_result_ids, m_dists, radius, search_params ); return count; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''') py_source = ut.codeblock( ''' if self.__curindex is None: raise FLANNException( 'build_index(...) method not called first or current index deleted.') if query.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % query.dtype) if self.__curindex_type != query.dtype.type: raise FLANNException('Index and query must have the same type') npts, dim = self.get_indexed_shape() assert(query.shape[0] == dim), 'data and query must have the same dims' result = np.empty(npts, dtype=index_type) if self.__curindex_type == np.float64: dists = np.empty(npts, dtype=np.float64) else: dists = np.empty(npts, dtype=np.float32) self.__flann_parameters.update(kwargs) nn = flann.radius_search[ self.__curindex_type]( self.__curindex, query, result, dists, npts, radius, pointer(self.__flann_parameters)) return (result[0:nn], dists[0:nn]) ''') cpp_param_doc['index_ptr'] = 'the index' cpp_param_doc['query1d'] = 'query point' cpp_param_doc['dists1d'] = 'similar, but for storing distances' cpp_param_doc['result_ids'] = 'array for storing the indices found (will be modified)' cpp_param_doc['max_nn'] = 'size of arrays result_ids and dists1d' cpp_param_doc['radius'] = 'search radius (squared radius for euclidian metric)' return_doc = 'number of neighbors found or <0 for an error' return_type = 'int' binding_argnames = ['index_ptr', 'query1d', 'result_ids', 'dists1d', 'max_nn', 'radius', 'flann_params', ] py_alias = 'nn_radius' py_args = 'query, radius, **kwargs'.split(', ') elif binding_name == 'find_nearest_neighbors_index': c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try { init_flann_parameters(flann_params); if (index_ptr==NULL) { throw FLANNException("Invalid index"); } Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_indices(result_ids,tcount, nn); Matrix<DistanceType> m_dists(dists, tcount, nn); SearchParams search_params = create_search_params(flann_params); index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()), m_indices, m_dists, nn, search_params ); return 0; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return -1; } return -1; ''' ).replace('{', '{{').replace('}', '}}') py_source = ut.codeblock( ''' if self.__curindex is None: raise FLANNException( 'build_index(...) method not called first or current index deleted.') if qpts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % qpts.dtype) if self.__curindex_type != qpts.dtype.type: raise FLANNException('Index and query must have the same type') qpts = ensure_2d_array(qpts, default_flags) npts, dim = self.get_indexed_shape() if qpts.size == dim: qpts.reshape(1, dim) nqpts = qpts.shape[0] assert qpts.shape[1] == dim, 'data and query must have the same dims' assert npts >= num_neighbors, 'more neighbors than there are points' result = np.empty((nqpts, num_neighbors), dtype=index_type) if self.__curindex_type == np.float64: dists = np.empty((nqpts, num_neighbors), dtype=np.float64) else: dists = np.empty((nqpts, num_neighbors), dtype=np.float32) self.__flann_parameters.update(kwargs) flann.find_nearest_neighbors_index[ self.__curindex_type]( self.__curindex, qpts, nqpts, result, dists, num_neighbors, pointer(self.__flann_parameters)) if num_neighbors == 1: return (result.reshape(nqpts), dists.reshape(nqpts)) else: return (result, dists) ''' ) docstr = 'Searches for nearest neighbors using the index provided' return_doc = zero_success return_type = 'int' # optional_args = ['Distance d = Distance()'] binding_argnames = ['index_ptr', 'testset', 'tcount', 'result_ids', 'dists', 'nn', 'flann_params', ] py_alias = 'nn_index' py_args = ['qpts', 'num_neighbors=1', '**kwargs'] elif binding_name == 'find_nearest_neighbors': c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try {{ init_flann_parameters(flann_params); if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_indices(result_ids,tcount, nn); Matrix<DistanceType> m_dists(dists, tcount, nn); SearchParams search_params = create_search_params(flann_params); index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()), m_indices, m_dists, nn, search_params ); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} return -1; ''' ) py_source = ut.codeblock( ''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if qpts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if pts.dtype != qpts.dtype: raise FLANNException('Data and query must have the same type') pts = ensure_2d_array(pts, default_flags) qpts = ensure_2d_array(qpts, default_flags) npts, dim = pts.shape nqpts = qpts.shape[0] assert qpts.shape[1] == dim, 'data and query must have the same dims' assert npts >= num_neighbors, 'more neighbors than there are points' result = np.empty((nqpts, num_neighbors), dtype=index_type) if pts.dtype == np.float64: dists = np.empty((nqpts, num_neighbors), dtype=np.float64) else: dists = np.empty((nqpts, num_neighbors), dtype=np.float32) self.__flann_parameters.update(kwargs) flann.find_nearest_neighbors[ pts.dtype.type]( pts, npts, dim, qpts, nqpts, result, dists, num_neighbors, pointer(self.__flann_parameters)) if num_neighbors == 1: return (result.reshape(nqpts), dists.reshape(nqpts)) else: return (result, dists) ''') docstr = 'Builds an index and uses it to find nearest neighbors.' return_doc = zero_success py_alias = 'nn' py_args = ['pts', 'qpts', 'num_neighbors=1', '**kwargs'] return_type = 'int' binding_argnames = ['dataset', 'rows', 'cols', 'testset', 'tcount', 'result_ids', 'dists', 'nn', 'flann_params'] optional_args = ['Distance d = Distance()'] elif binding_name == 'load_index': c_source_part = ut.codeblock( r''' Index<Distance>* index = new Index<Distance>(Matrix<typename Distance::ElementType>(dataset,rows,cols), SavedIndexParams(filename), d); return index; ''' ) py_source = ut.codeblock( ''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None self.__curindex_data = None self.__added_data = [] self.__curindex_type = None self.__curindex = flann.load_index[pts.dtype.type]( c_char_p(to_bytes(filename)), pts, npts, dim) if self.__curindex is None: raise FLANNException( ('Error loading the FLANN index with filename=%r.' ' C++ may have thrown more detailed errors') % (filename,)) self.__curindex_data = pts self.__added_data = [] self.__removed_ids = [] self.__curindex_type = pts.dtype.type ''') docstr = 'Loads a previously saved index from a file.' return_doc = 'index_ptr' cpp_param_doc['dataset'] = 'The dataset corresponding to the index' cpp_param_doc['filename'] = 'File to load the index from' py_args = ['filename', 'pts'] return_type = 'flann_index_t' binding_argnames = ['filename', 'dataset', 'rows', 'cols'] optional_args = ['Distance d = Distance()'] elif binding_name == 'save_index': docstr = 'Saves the index to a file. Only the index is saved into the file, the dataset corresponding to the index is not saved.' cpp_param_doc['index_ptr'] = 'The index that should be saved' cpp_param_doc['filename'] = 'The filename the index should be saved to' return_doc = 'Returns 0 on success, negative value on error' c_source_part = ut.codeblock( r''' Index<Distance>* index = (Index<Distance>*)index_ptr; index->save(filename); return 0; ''') py_source = ut.codeblock( ''' if self.__curindex is not None: flann.save_index[self.__curindex_type]( self.__curindex, c_char_p(to_bytes(filename))) ''') return_type = 'int' binding_argnames = ['index_ptr', 'filename'] py_alias = None py_args = None elif binding_name == 'build_index': docstr = ut.codeblock( ''' Builds and returns an index. It uses autotuning if the target_precision field of index_params is between 0 and 1, or the parameters specified if it's -1. ''') pydoc = ut.codeblock( ''' This builds and internally stores an index to be used for future nearest neighbor matchings. It erases any previously stored indexes, so use multiple instances of this class to work with multiple stored indices. Use nn_index(...) to find the nearest neighbors in this index. pts is a 2d numpy array or matrix. All the computation is done in np.float32 type, but pts may be any type that is convertable to np.float32. ''') c_source = ut.codeblock( r''' typedef typename Distance::ElementType ElementType; try { init_flann_parameters(flann_params); if (flann_params == NULL) { throw FLANNException("The flann_params argument must be non-null"); } IndexParams params = create_parameters(flann_params); Index<Distance>* index = new Index<Distance>(Matrix<ElementType>(dataset,rows,cols), params, d); index->buildIndex(); if (flann_params->algorithm==FLANN_INDEX_AUTOTUNED) { IndexParams params = index->getParameters(); update_flann_parameters(params,flann_params); SearchParams search_params = get_param<SearchParams>(params,"search_params"); *speedup = get_param<float>(params,"speedup"); flann_params->checks = search_params.checks; flann_params->eps = search_params.eps; flann_params->cb_index = get_param<float>(params,"cb_index",0.0); } return index; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return NULL; } ''').replace('{', '{{').replace('}', '}}') py_source = ut.codeblock( ''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape self.__ensureRandomSeed(kwargs) self.__flann_parameters.update(kwargs) if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None speedup = c_float(0) self.__curindex = flann.build_index[pts.dtype.type]( pts, npts, dim, byref(speedup), pointer(self.__flann_parameters)) self.__curindex_data = pts self.__curindex_type = pts.dtype.type params = dict(self.__flann_parameters) params['speedup'] = speedup.value return params ''') # binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params'] return_doc = 'the newly created index or a number <0 for error' cpp_param_doc['speedup'] = 'speedup over linear search, estimated if using autotuning, output parameter' optional_args = ['Distance d = Distance()'] return_type = 'flann_index_t' py_args = ['pts', '**kwargs'] binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params'] elif binding_name == 'free_index': docstr = 'Deletes an index and releases the memory used by it.' pydoc = ut.codeblock( ''' Deletes the current index freeing all the momory it uses. The memory used by the dataset that was indexed is not freed unless there are no other references to those numpy arrays. ''') c_source_part = ut.codeblock( r''' Index<Distance>* index = (Index<Distance>*)index_ptr; delete index; return 0; ''') py_source = ut.codeblock( ''' self.__flann_parameters.update(kwargs) if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None self.__curindex_data = None self.__added_data = [] self.__removed_ids = [] ''') return_doc = zero_success return_type = 'int' binding_argnames = ['index_ptr', 'flann_params'] cpp_param_doc['flann_params'] = ut.textblock( '''generic flann params (only used to specify verbosity)''') py_alias = 'delete_index' py_args = ['**kwargs'] elif binding_name == 'get_point': docstr = 'Gets a point from a given index position.' return_doc = 'pointer to datapoint or NULL on miss' binding_argnames = ['index_ptr', 'point_id'] cpp_param_doc['point_id'] = 'index of datapoint to get.' return_type = 'Distance::ElementType*' elif binding_name == 'flann_get_distance_order': docstr = ut.textblock( '''Gets the distance order in use throughout FLANN (only applicable if minkowski distance is in use).''') binding_argnames = [] return_type = 'int' else: dictdef = { '_template_new': { 'docstr': '', 'binding_argnames': [], 'return_type': 'int', }, 'flann_get_distance_type': { 'docstr': '', 'binding_argnames': [], 'return_type': 'int', }, 'flann_log_verbosity': { 'docstr': ut.codeblock( ''' Sets the log level used for all flann functions (unless specified in FLANNParameters for each call ''' ), 'binding_argnames': ['level'], 'return_type': 'void', }, } if binding_name in dictdef: docstr = dictdef[binding_name].get('docstr', '') binding_argnames = dictdef[binding_name]['binding_argnames'] return_type = dictdef[binding_name]['return_type'] else: raise NotImplementedError('Unknown binding name %r' % (binding_name,)) if c_source is None: if c_source_part is not None: try_ = ut.codeblock( ''' try {{ ''' ) throw_ = '\n' + ut.indent(ut.codeblock( ''' if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} ''' ), ' ' * 4) if 'index_ptr' not in binding_argnames: throw_ = '' if 'flann_params' in binding_argnames: part1 = try_ + '\n' + ' init_flann_parameters(flann_params);' + throw_ else: part1 = try_ + throw_ if return_type == 'int': default_return = '-1' else: default_return = 'NULL' part2 = ut.codeblock( r''' }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return ''' + default_return + '''; }} ''' ) c_source = part1 + '\n' + ut.indent(c_source_part, ' ' * 4) + '\n' + part2 else: c_source = ut.codeblock( ''' TODO: IMPLEMENT THIS FUNCTION WRAPPER ''' ) try: docstr_cpp = docstr[:] if return_doc is not None: param_docs = ut.dict_take(cpp_param_doc, binding_argnames) cpp_param_docblock = '\n'.join( ['%s = %s' % (name, doc) for name, doc in zip(binding_argnames, param_docs)]) docstr_cpp += '\n\n' + 'Params:\n' + ut.indent(cpp_param_docblock, ' ') docstr_cpp += '\n\n' + 'Returns: ' + return_doc if pydoc is None: docstr_py = docstr[:] else: docstr_py = pydoc[:] if py_args: py_param_doc = cpp_param_doc.copy() py_param_doc['pts'] = py_param_doc['dataset'].replace('pointer to ', '') py_param_doc['qpts'] = (py_param_doc['testset'].replace( 'pointer to ', '') + ' (may be a single point)') py_param_doc['num_neighbors'] = py_param_doc['nn'] py_param_doc['**kwargs'] = py_param_doc['flann_params'] py_args_ = [a.split('=')[0] for a in py_args] param_docs = ut.dict_take(py_param_doc, py_args_, '') # py_types = py_param_docblock = '\n'.join(['%s: %s' % (name, doc) for name, doc in zip(py_args_, param_docs)]) docstr_py += '\n\n' + 'Params:\n' + ut.indent(py_param_docblock, ' ') except Exception as ex: ut.printex(ex, keys=['binding_name']) raise pass binding_def = { 'cpp_binding_name': cpp_binding_name, 'docstr_cpp': docstr_cpp, 'docstr_py': docstr_py, 'return_type': return_type, 'binding_argnames': binding_argnames, 'c_source': c_source, 'optional_args': optional_args, 'py_source': py_source, 'py_args': py_args, 'py_alias': py_alias, } return binding_def
def define_flann_bindings(binding_name): """ Define the binding names for flann """ # default c source c_source = None optional_args = None c_source_part = None py_source = None py_alias = None py_args = None pydoc = None cpp_param_doc = { 'cols': 'number of columns in the dataset (feature dimensionality)', 'dataset': 'pointer to a data set stored in row major order', 'dists': ut.packtext( '''pointer to matrix for the distances of the nearest neighbors of the testset features in the dataset'''), 'flann_params': 'generic flann parameters', 'index_ptr': 'the index (constructed previously using flann_build_index)', 'nn': 'how many nearest neighbors to return', 'rebuild_threshold': ut.packtext( '''reallocs index when it grows by factor of `rebuild_threshold`. A smaller value results is more space efficient but less computationally efficient. Must be greater than 1.'''), 'result_ids': ut.packtext( '''pointer to matrix for the indices of the nearest neighbors of the testset features in the dataset (must have tcount number of rows and nn number of columns)'''), 'rows': 'number of rows (features) in the dataset', 'tcount': ut.packtext('''number of rows (features) in the query dataset (same dimensionality as features in the dataset)'''), 'testset': 'pointer to a query set stored in row major order', 'level': 'verbosity level' } standard_csource = ut.codeblock(r''' try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; return index->{cpp_binding_name}(); }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); throw; }} ''') return_doc = None cpp_binding_name = binding_name zero_success = 'zero or a number <0 for error' if binding_name == 'clean_removed_points': cpp_binding_name = ut.to_camel_case(binding_name) return_type = 'void' docstr = 'Deletes removed points in index?' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'veclen': return_type = 'int' docstr = 'Returns number of features in this index' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'size': return_type = 'int' docstr = 'returns The dimensionality of the features in this index.' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'getType': return_type = 'flann_algorithm_t' docstr = 'returns The index type (kdtree, kmeans,...)' binding_argnames = ['index_ptr'] c_source = standard_csource elif binding_name == 'used_memory': docstr = ut.codeblock(''' Returns the amount of memory (in bytes) used by the index index_ptr = pointer to pre-built index. Returns: int ''') c_source = ut.codeblock(r''' try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; return index->usedMemory(); }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''') py_source = ut.codeblock(''' if self.__curindex is None: return 0 return flann.used_memory[self.__curindex_type](self.__curindex) ''') binding_argnames = ['index_ptr'] return_type = 'int' elif binding_name == 'add_points': c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<ElementType> points = Matrix<ElementType>(points, rows, index->veclen()); index->addPoints(points, rebuild_threshold); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''') py_source = ut.codeblock(''' if new_pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % new_pts.dtype) if new_pts.dtype != self.__curindex_type: raise FLANNException('New points must have the same type') new_pts = ensure_2d_array(new_pts, default_flags) rows = new_pts.shape[0] flann.add_points[self.__curindex_type](self.__curindex, new_pts, rows, rebuild_threshold) return self.__added_data.append(new_pts) ''') #return_type = 'void' return_type = 'int' docstr = 'Adds points to pre-built index.' if False: binding_argnames = [ 'index_ptr', 'points', 'rows', 'cols', # TODO: can remove 'rebuild_threshold', ] else: binding_argnames = [ 'index_ptr', 'points', 'rows', 'rebuild_threshold', ] return_doc = '0 if success otherwise -1' py_args = ['new_pts', 'rebuild_threshold=2.'] cpp_param_doc['points'] = 'pointer to array of points' elif binding_name == 'remove_point': c_source = ut.codeblock(r''' size_t point_id(point_id_uint); try {{ if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; index->removePoint(point_id); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''') py_source = ut.codeblock(''' flann.remove_point[self.__curindex_type](self.__curindex, point_id) self.__removed_ids.append(point_id) ''') #return_type = 'void' return_type = 'int' docstr = 'Removes a point from the index' return_doc = zero_success cpp_param_doc['point_id'] = 'point id to be removed' cpp_param_doc['index_ptr'] = 'The index that should be modified' binding_argnames = ['index_ptr', 'point_id'] elif binding_name == 'remove_points': c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; try {{ if (index_ptr==NULL) {{ thow FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; index->removePoints(id_list, num); return; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return; }} ''') py_source = ut.codeblock(''' id_list = np.array(id_list, dtype=np.int32) num = len(id_list) flann.remove_points[self.__curindex_type](self.__curindex, id_list, num) self.__removed_ids.extend(id_list) ''') cpp_param_doc['index_ptr'] = 'The index that should be modified' cpp_param_doc['id_list'] = 'list of point ids to be removed' cpp_param_doc['num'] = 'number of points in id_list' docstr = 'Removes multiple points from the index' return_doc = 'void' py_args = ['id_list'] return_type = 'void' binding_argnames = ['index_ptr', 'id_list', 'num'] elif binding_name == 'compute_cluster_centers': docstr = ut.textblock(r''' Clusters the features in the dataset using a hierarchical kmeans clustering approach. This is significantly faster than using a flat kmeans clustering for a large number of clusters. ''') c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try { init_flann_parameters(flann_params); Matrix<ElementType> inputData(dataset,rows,cols); KMeansIndexParams params(flann_params->branching, flann_params->iterations, flann_params->centers_init, flann_params->cb_index); Matrix<DistanceType> centers(result_centers, clusters,cols); int clusterNum = hierarchicalClustering<Distance>(inputData, centers, params, d); return clusterNum; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return -1; } '''.replace('{', '{{').replace('}', '}}')) py_source = ut.codeblock(''' # First verify the paremeters are sensible. if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if int(branch_size) != branch_size or branch_size < 2: raise FLANNException('branch_size must be an integer >= 2.') branch_size = int(branch_size) if int(num_branches) != num_branches or num_branches < 1: raise FLANNException('num_branches must be an integer >= 1.') num_branches = int(num_branches) if max_iterations is None: max_iterations = -1 else: max_iterations = int(max_iterations) # init the arrays and starting values pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape num_clusters = (branch_size - 1) * num_branches + 1 if pts.dtype.type == np.float64: result = np.empty((num_clusters, dim), dtype=np.float64) else: result = np.empty((num_clusters, dim), dtype=np.float32) # set all the parameters appropriately self.__ensureRandomSeed(kwargs) params = {'iterations': max_iterations, 'algorithm': 'kmeans', 'branching': branch_size, 'random_seed': kwargs['random_seed']} self.__flann_parameters.update(params) numclusters = flann.compute_cluster_centers[pts.dtype.type]( pts, npts, dim, num_clusters, result, pointer(self.__flann_parameters)) if numclusters <= 0: raise FLANNException('Error occured during clustering procedure.') if dtype is None: return result else: return dtype(result) ''').replace('}', '}}').replace('{', '{{') return_doc = ut.packtext( '''number of clusters computed or a number <0 for error. This number can be different than the number of clusters requested, due to the way hierarchical clusters are computed. The number of clusters returned will be the highest number of the form (branch_size-1)*K+1 smaller than the number of clusters requested.''') cpp_param_doc['clusters'] = 'number of cluster to compute' cpp_param_doc[ 'result_centers'] = 'memory buffer where the output cluster centers are stored' cpp_param_doc[ 'flann_params'] = 'generic flann parameters and index_params used to specify the kmeans tree parameters (branching factor, max number of iterations to use)' return_type = 'int' binding_argnames = [ 'dataset', 'rows', 'cols', 'clusters', 'result_centers', 'flann_params' ] optional_args = ['Distance d = Distance()'] py_alias = 'hierarchical_kmeans' py_args = 'pts, branch_size, num_branches, max_iterations=None, dtype=None, **kwargs'.split( ', ') elif binding_name == 'radius_search': docstr = ut.codeblock(r''' Performs an radius search using an already constructed index. In case of radius search, instead of always returning a predetermined number of nearest neighbours (for example the 10 nearest neighbours), the search will return all the neighbours found within a search radius of the query point. The check parameter in the FLANNParameters below sets the level of approximation for the search by only visiting "checks" number of features in the index (the same way as for the KNN search). A lower value for checks will give a higher search speedup at the cost of potentially not returning all the neighbours in the specified radius. ''') c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try {{ init_flann_parameters(flann_params); if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_result_ids(result_ids, 1, max_nn); Matrix<DistanceType> m_dists(dists1d, 1, max_nn); SearchParams search_params = create_search_params(flann_params); int count = index->radiusSearch(Matrix<ElementType>(query1d, 1, index->veclen()), m_result_ids, m_dists, radius, search_params ); return count; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} ''') py_source = ut.codeblock(''' if self.__curindex is None: raise FLANNException( 'build_index(...) method not called first or current index deleted.') if query.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % query.dtype) if self.__curindex_type != query.dtype.type: raise FLANNException('Index and query must have the same type') npts, dim = self.get_indexed_shape() assert(query.shape[0] == dim), 'data and query must have the same dims' result = np.empty(npts, dtype=index_type) if self.__curindex_type == np.float64: dists = np.empty(npts, dtype=np.float64) else: dists = np.empty(npts, dtype=np.float32) self.__flann_parameters.update(kwargs) nn = flann.radius_search[ self.__curindex_type]( self.__curindex, query, result, dists, npts, radius, pointer(self.__flann_parameters)) return (result[0:nn], dists[0:nn]) ''') cpp_param_doc['index_ptr'] = 'the index' cpp_param_doc['query1d'] = 'query point' cpp_param_doc['dists1d'] = 'similar, but for storing distances' cpp_param_doc[ 'result_ids'] = 'array for storing the indices found (will be modified)' cpp_param_doc['max_nn'] = 'size of arrays result_ids and dists1d' cpp_param_doc[ 'radius'] = 'search radius (squared radius for euclidian metric)' return_doc = 'number of neighbors found or <0 for an error' return_type = 'int' binding_argnames = [ 'index_ptr', 'query1d', 'result_ids', 'dists1d', 'max_nn', 'radius', 'flann_params', ] py_alias = 'nn_radius' py_args = 'query, radius, **kwargs'.split(', ') elif binding_name == 'find_nearest_neighbors_index': c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try { init_flann_parameters(flann_params); if (index_ptr==NULL) { throw FLANNException("Invalid index"); } Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_indices(result_ids,tcount, nn); Matrix<DistanceType> m_dists(dists, tcount, nn); SearchParams search_params = create_search_params(flann_params); index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()), m_indices, m_dists, nn, search_params ); return 0; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return -1; } return -1; ''').replace('{', '{{').replace('}', '}}') py_source = ut.codeblock(''' if self.__curindex is None: raise FLANNException( 'build_index(...) method not called first or current index deleted.') if qpts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % qpts.dtype) if self.__curindex_type != qpts.dtype.type: raise FLANNException('Index and query must have the same type') qpts = ensure_2d_array(qpts, default_flags) npts, dim = self.get_indexed_shape() if qpts.size == dim: qpts.reshape(1, dim) nqpts = qpts.shape[0] assert qpts.shape[1] == dim, 'data and query must have the same dims' assert npts >= num_neighbors, 'more neighbors than there are points' result = np.empty((nqpts, num_neighbors), dtype=index_type) if self.__curindex_type == np.float64: dists = np.empty((nqpts, num_neighbors), dtype=np.float64) else: dists = np.empty((nqpts, num_neighbors), dtype=np.float32) self.__flann_parameters.update(kwargs) flann.find_nearest_neighbors_index[ self.__curindex_type]( self.__curindex, qpts, nqpts, result, dists, num_neighbors, pointer(self.__flann_parameters)) if num_neighbors == 1: return (result.reshape(nqpts), dists.reshape(nqpts)) else: return (result, dists) ''') docstr = 'Searches for nearest neighbors using the index provided' return_doc = zero_success return_type = 'int' # optional_args = ['Distance d = Distance()'] binding_argnames = [ 'index_ptr', 'testset', 'tcount', 'result_ids', 'dists', 'nn', 'flann_params', ] py_alias = 'nn_index' py_args = ['qpts', 'num_neighbors=1', '**kwargs'] elif binding_name == 'find_nearest_neighbors': c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; typedef typename Distance::ResultType DistanceType; try {{ init_flann_parameters(flann_params); if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} Index<Distance>* index = (Index<Distance>*)index_ptr; Matrix<int> m_indices(result_ids,tcount, nn); Matrix<DistanceType> m_dists(dists, tcount, nn); SearchParams search_params = create_search_params(flann_params); index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()), m_indices, m_dists, nn, search_params ); return 0; }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return -1; }} return -1; ''') py_source = ut.codeblock(''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if qpts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) if pts.dtype != qpts.dtype: raise FLANNException('Data and query must have the same type') pts = ensure_2d_array(pts, default_flags) qpts = ensure_2d_array(qpts, default_flags) npts, dim = pts.shape nqpts = qpts.shape[0] assert qpts.shape[1] == dim, 'data and query must have the same dims' assert npts >= num_neighbors, 'more neighbors than there are points' result = np.empty((nqpts, num_neighbors), dtype=index_type) if pts.dtype == np.float64: dists = np.empty((nqpts, num_neighbors), dtype=np.float64) else: dists = np.empty((nqpts, num_neighbors), dtype=np.float32) self.__flann_parameters.update(kwargs) flann.find_nearest_neighbors[ pts.dtype.type]( pts, npts, dim, qpts, nqpts, result, dists, num_neighbors, pointer(self.__flann_parameters)) if num_neighbors == 1: return (result.reshape(nqpts), dists.reshape(nqpts)) else: return (result, dists) ''') docstr = 'Builds an index and uses it to find nearest neighbors.' return_doc = zero_success py_alias = 'nn' py_args = ['pts', 'qpts', 'num_neighbors=1', '**kwargs'] return_type = 'int' binding_argnames = [ 'dataset', 'rows', 'cols', 'testset', 'tcount', 'result_ids', 'dists', 'nn', 'flann_params' ] optional_args = ['Distance d = Distance()'] elif binding_name == 'load_index': c_source_part = ut.codeblock(r''' Index<Distance>* index = new Index<Distance>(Matrix<typename Distance::ElementType>(dataset,rows,cols), SavedIndexParams(filename), d); return index; ''') py_source = ut.codeblock(''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None self.__curindex_data = None self.__added_data = [] self.__curindex_type = None self.__curindex = flann.load_index[pts.dtype.type]( c_char_p(to_bytes(filename)), pts, npts, dim) if self.__curindex is None: raise FLANNException( ('Error loading the FLANN index with filename=%r.' ' C++ may have thrown more detailed errors') % (filename,)) self.__curindex_data = pts self.__added_data = [] self.__removed_ids = [] self.__curindex_type = pts.dtype.type ''') docstr = 'Loads a previously saved index from a file.' return_doc = 'index_ptr' cpp_param_doc['dataset'] = 'The dataset corresponding to the index' cpp_param_doc['filename'] = 'File to load the index from' py_args = ['filename', 'pts'] return_type = 'flann_index_t' binding_argnames = ['filename', 'dataset', 'rows', 'cols'] optional_args = ['Distance d = Distance()'] elif binding_name == 'save_index': docstr = 'Saves the index to a file. Only the index is saved into the file, the dataset corresponding to the index is not saved.' cpp_param_doc['index_ptr'] = 'The index that should be saved' cpp_param_doc['filename'] = 'The filename the index should be saved to' return_doc = 'Returns 0 on success, negative value on error' c_source_part = ut.codeblock(r''' Index<Distance>* index = (Index<Distance>*)index_ptr; index->save(filename); return 0; ''') py_source = ut.codeblock(''' if self.__curindex is not None: flann.save_index[self.__curindex_type]( self.__curindex, c_char_p(to_bytes(filename))) ''') return_type = 'int' binding_argnames = ['index_ptr', 'filename'] py_alias = None py_args = None elif binding_name == 'build_index': docstr = ut.codeblock(''' Builds and returns an index. It uses autotuning if the target_precision field of index_params is between 0 and 1, or the parameters specified if it's -1. ''') pydoc = ut.codeblock(''' This builds and internally stores an index to be used for future nearest neighbor matchings. It erases any previously stored indexes, so use multiple instances of this class to work with multiple stored indices. Use nn_index(...) to find the nearest neighbors in this index. pts is a 2d numpy array or matrix. All the computation is done in np.float32 type, but pts may be any type that is convertable to np.float32. ''') c_source = ut.codeblock(r''' typedef typename Distance::ElementType ElementType; try { init_flann_parameters(flann_params); if (flann_params == NULL) { throw FLANNException("The flann_params argument must be non-null"); } IndexParams params = create_parameters(flann_params); Index<Distance>* index = new Index<Distance>(Matrix<ElementType>(dataset,rows,cols), params, d); index->buildIndex(); if (flann_params->algorithm==FLANN_INDEX_AUTOTUNED) { IndexParams params = index->getParameters(); update_flann_parameters(params,flann_params); SearchParams search_params = get_param<SearchParams>(params,"search_params"); *speedup = get_param<float>(params,"speedup"); flann_params->checks = search_params.checks; flann_params->eps = search_params.eps; flann_params->cb_index = get_param<float>(params,"cb_index",0.0); } return index; } catch (std::runtime_error& e) { Logger::error("Caught exception: %s\n",e.what()); return NULL; } ''').replace('{', '{{').replace('}', '}}') py_source = ut.codeblock(''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) npts, dim = pts.shape self.__ensureRandomSeed(kwargs) self.__flann_parameters.update(kwargs) if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None speedup = c_float(0) self.__curindex = flann.build_index[pts.dtype.type]( pts, npts, dim, byref(speedup), pointer(self.__flann_parameters)) self.__curindex_data = pts self.__curindex_type = pts.dtype.type params = dict(self.__flann_parameters) params['speedup'] = speedup.value return params ''') # binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params'] return_doc = 'the newly created index or a number <0 for error' cpp_param_doc[ 'speedup'] = 'speedup over linear search, estimated if using autotuning, output parameter' optional_args = ['Distance d = Distance()'] return_type = 'flann_index_t' py_args = ['pts', '**kwargs'] binding_argnames = [ 'dataset', 'rows', 'cols', 'speedup', 'flann_params' ] elif binding_name == 'free_index': docstr = 'Deletes an index and releases the memory used by it.' pydoc = ut.codeblock(''' Deletes the current index freeing all the momory it uses. The memory used by the dataset that was indexed is not freed unless there are no other references to those numpy arrays. ''') c_source_part = ut.codeblock(r''' Index<Distance>* index = (Index<Distance>*)index_ptr; delete index; return 0; ''') py_source = ut.codeblock(''' self.__flann_parameters.update(kwargs) if self.__curindex is not None: flann.free_index[self.__curindex_type]( self.__curindex, pointer(self.__flann_parameters)) self.__curindex = None self.__curindex_data = None self.__added_data = [] self.__removed_ids = [] ''') return_doc = zero_success return_type = 'int' binding_argnames = ['index_ptr', 'flann_params'] cpp_param_doc['flann_params'] = ut.textblock( '''generic flann params (only used to specify verbosity)''') py_alias = 'delete_index' py_args = ['**kwargs'] elif binding_name == 'get_point': docstr = 'Gets a point from a given index position.' return_doc = 'pointer to datapoint or NULL on miss' binding_argnames = ['index_ptr', 'point_id'] cpp_param_doc['point_id'] = 'index of datapoint to get.' return_type = 'Distance::ElementType*' elif binding_name == 'flann_get_distance_order': docstr = ut.textblock( '''Gets the distance order in use throughout FLANN (only applicable if minkowski distance is in use).''') binding_argnames = [] return_type = 'int' else: dictdef = { '_template_new': { 'docstr': '', 'binding_argnames': [], 'return_type': 'int', }, 'flann_get_distance_type': { 'docstr': '', 'binding_argnames': [], 'return_type': 'int', }, 'flann_log_verbosity': { 'docstr': ut.codeblock(''' Sets the log level used for all flann functions (unless specified in FLANNParameters for each call '''), 'binding_argnames': ['level'], 'return_type': 'void', }, } if binding_name in dictdef: docstr = dictdef[binding_name].get('docstr', '') binding_argnames = dictdef[binding_name]['binding_argnames'] return_type = dictdef[binding_name]['return_type'] else: raise NotImplementedError('Unknown binding name %r' % (binding_name, )) if c_source is None: if c_source_part is not None: try_ = ut.codeblock(''' try {{ ''') throw_ = '\n' + ut.indent( ut.codeblock(''' if (index_ptr==NULL) {{ throw FLANNException("Invalid index"); }} '''), ' ' * 4) if 'index_ptr' not in binding_argnames: throw_ = '' if 'flann_params' in binding_argnames: part1 = try_ + '\n' + ' init_flann_parameters(flann_params);' + throw_ else: part1 = try_ + throw_ if return_type == 'int': default_return = '-1' else: default_return = 'NULL' part2 = ut.codeblock(r''' }} catch (std::runtime_error& e) {{ Logger::error("Caught exception: %s\n",e.what()); return ''' + default_return + '''; }} ''') c_source = part1 + '\n' + ut.indent(c_source_part, ' ' * 4) + '\n' + part2 else: c_source = ut.codeblock(''' TODO: IMPLEMENT THIS FUNCTION WRAPPER ''') try: docstr_cpp = docstr[:] if return_doc is not None: param_docs = ut.dict_take(cpp_param_doc, binding_argnames) cpp_param_docblock = '\n'.join([ '%s = %s' % (name, doc) for name, doc in zip(binding_argnames, param_docs) ]) docstr_cpp += '\n\n' + 'Params:\n' + ut.indent( cpp_param_docblock, ' ') docstr_cpp += '\n\n' + 'Returns: ' + return_doc if pydoc is None: docstr_py = docstr[:] else: docstr_py = pydoc[:] if py_args: py_param_doc = cpp_param_doc.copy() py_param_doc['pts'] = py_param_doc['dataset'].replace( 'pointer to ', '') py_param_doc['qpts'] = ( py_param_doc['testset'].replace('pointer to ', '') + ' (may be a single point)') py_param_doc['num_neighbors'] = py_param_doc['nn'] py_param_doc['**kwargs'] = py_param_doc['flann_params'] py_args_ = [a.split('=')[0] for a in py_args] param_docs = ut.dict_take(py_param_doc, py_args_, '') # py_types = py_param_docblock = '\n'.join([ '%s: %s' % (name, doc) for name, doc in zip(py_args_, param_docs) ]) docstr_py += '\n\n' + 'Params:\n' + ut.indent( py_param_docblock, ' ') except Exception as ex: ut.printex(ex, keys=['binding_name']) raise pass binding_def = { 'cpp_binding_name': cpp_binding_name, 'docstr_cpp': docstr_cpp, 'docstr_py': docstr_py, 'return_type': return_type, 'binding_argnames': binding_argnames, 'c_source': c_source, 'optional_args': optional_args, 'py_source': py_source, 'py_args': py_args, 'py_alias': py_alias, } return binding_def
def make_default_docstr(func, with_args=True, with_ret=True, with_commandline=True, with_example=True, with_header=False, with_debug=False, ): r""" Tries to make a sensible default docstr so the user can fill things in without typing too much # TODO: Interleave old documentation with new documentation Args: func (function): live python function with_args (bool): with_ret (bool): (default = True) with_commandline (bool): (default = True) with_example (bool): (default = True) with_header (bool): (default = False) with_debug (bool): (default = False) Returns: tuple: (argname, val) Ignore: pass CommandLine: python -m utool.util_autogen --exec-make_default_docstr --show Example: >>> # ENABLE_DOCTEST >>> from utool.util_autogen import * # NOQA >>> import utool as ut >>> func = ut.make_default_docstr >>> #func = ut.make_args_docstr >>> func = PythonStatement >>> default_docstr = make_default_docstr(func) >>> result = str(default_docstr) >>> print(result) """ import utool as ut #from utool import util_inspect funcinfo = ut.util_inspect.infer_function_info(func) argname_list = funcinfo.argname_list argtype_list = funcinfo.argtype_list argdesc_list = funcinfo.argdesc_list return_header = funcinfo.return_header return_type = funcinfo.return_type return_name = funcinfo.return_name return_desc = funcinfo.return_desc funcname = funcinfo.funcname modname = funcinfo.modname defaults = funcinfo.defaults num_indent = funcinfo.num_indent needs_surround = funcinfo.needs_surround funcname = funcinfo.funcname ismethod = funcinfo.ismethod kwarg_keys = funcinfo.kwarg_keys docstr_parts = [] # Header part if with_header: header_block = funcname docstr_parts.append(header_block) # Args part if with_args and len(argname_list) > 0: argheader = 'Args' arg_docstr = make_args_docstr(argname_list, argtype_list, argdesc_list, ismethod) argsblock = make_docstr_block(argheader, arg_docstr) docstr_parts.append(argsblock) with_kw = with_args if with_kw and len(kwarg_keys) > 0: #ut.embed() import textwrap kwargs_docstr = ', '.join(kwarg_keys) kwargs_docstr = '\n'.join(textwrap.wrap(kwargs_docstr)) kwargsblock = make_docstr_block('Kwargs', kwargs_docstr) docstr_parts.append(kwargsblock) # Return / Yeild part if with_ret and return_header is not None: if return_header is not None: return_doctr = make_returns_or_yeilds_docstr(return_type, return_name, return_desc) returnblock = make_docstr_block(return_header, return_doctr) docstr_parts.append(returnblock) # Example part # try to generate a simple and unit testable example if with_commandline: cmdlineheader = 'CommandLine' cmdlinecode = make_cmdline_docstr(funcname, modname) cmdlineblock = make_docstr_block(cmdlineheader, cmdlinecode) docstr_parts.append(cmdlineblock) if with_example: exampleheader = 'Example' examplecode = make_example_docstr(funcname, modname, argname_list, defaults, return_type, return_name, ismethod) examplecode_ = ut.indent(examplecode, '>>> ') exampleblock = make_docstr_block(exampleheader, examplecode_) docstr_parts.append(exampleblock) # DEBUG part (in case something goes wrong) if with_debug: debugheader = 'Debug' debugblock = ut.codeblock( ''' num_indent = {num_indent} ''' ).format(num_indent=num_indent) debugblock = make_docstr_block(debugheader, debugblock) docstr_parts.append(debugblock) # Enclosure / Indentation Parts if needs_surround: docstr_parts = ['r"""'] + ['\n\n'.join(docstr_parts)] + ['"""'] default_docstr = '\n'.join(docstr_parts) else: default_docstr = '\n\n'.join(docstr_parts) docstr_indent = ' ' * (num_indent + 4) default_docstr = ut.indent(default_docstr, docstr_indent) return default_docstr
def autogen_ibeis_runtest(): """ special case to generate tests script for IBEIS Example: >>> from autogen_test_script import * # NOQA >>> test_script = autogen_ibeis_runtest() >>> print(test_script) CommandLine: python -c "import utool; utool.autogen_ibeis_runtest()" python -c "import utool; print(utool.autogen_ibeis_runtest())" python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh chmod +x run_tests.sh """ quick_tests = [ 'ibeis/tests/assert_modules.py' ] #test_repos = [ # '~/code/ibeis' # '~/code/vtool' # '~/code/hesaff' # '~/code/guitool' #] #test_pattern = [ # '~/code/ibeis/test_ibs*.py' #] test_argvs = '--quiet --noshow' misc_pats = [ 'test_utool_parallel.py', 'test_pil_hash.py', ] repodir = '~/code/utool' exclude_list = [] # Verbosity to show which modules at least have some tests #untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs) #print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames)) #print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list)) implicit_build_modlist_str = ut.codeblock( ''' import sys exclude_doctests_fnames = set(['__init__.py']) exclude_dirs = [ '_broken', 'old', 'tests', 'timeits', '_scripts', '_timeits', '_doc', 'notebook', ] dpath_list = ['utool'] doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) for modname in doctest_modname_list: exec('import ' + modname, globals(), locals()) module_list = [sys.modules[name] for name in doctest_modname_list] ''' ) globals_ = globals() locals_ = locals() exec(implicit_build_modlist_str, globals_, locals_) module_list = locals_['module_list'] doctest_modname_list = locals_['doctest_modname_list'] import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list]) modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list])) explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str)) build_modlist_str = implicit_build_modlist_str #build_modlist_str = explicit_build_modlist_str pyscript_fmtstr = ut.codeblock( r''' #!/usr/bin/env python from __future__ import absolute_import, division, print_function import utool as ut def run_tests(): # Build module list and run tests {build_modlist_str} ut.doctest_module_list(module_list) if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() run_tests() ''' ) pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip()) pyscript_text = ut.autofix_codeblock(pyscript_text) def def_test(header, pat=None, dpath=None, modname=None, default=False, testcmds=None): """ interface to make test tuple """ return (header, default, modname, dpath, pat, testcmds) # BUILD OLD SHELL RUN TESTS HARNESS testcmds = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST') test_headers = [ # title, default, module, testpattern def_test('DOC', testcmds=testcmds, default=True) ] shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list) return shscript_text, pyscript_text
def get_dbinfo( ibs, verbose=True, with_imgsize=False, with_bytes=False, with_contrib=False, with_agesex=False, with_header=True, short=False, tag='dbinfo', aid_list=None, aids=None, ): """ Returns dictionary of digestable database information Infostr is a string summary of all the stats. Prints infostr in addition to returning locals Args: ibs (IBEISController): verbose (bool): with_imgsize (bool): with_bytes (bool): Returns: dict: SeeAlso: python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --use-hist=True --old=False --per_name_vpedge=False python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --all CommandLine: python -m wbia.other.dbinfo --exec-get_dbinfo:0 python -m wbia.other.dbinfo --test-get_dbinfo:1 python -m wbia.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3 python -m wbia.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1 python -m wbia.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1 python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db LF_Bajo_bonito -a default python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db DETECT_SEATURTLES -a default --readonly python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a ctrl python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0 python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0 Example1: >>> # SCRIPT >>> from wbia.other.dbinfo import * # NOQA >>> import wbia >>> defaultdb = 'testdb1' >>> ibs, aid_list = wbia.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1') >>> kwargs = ut.get_kwdefaults(get_dbinfo) >>> kwargs['verbose'] = False >>> kwargs['aid_list'] = aid_list >>> kwargs = ut.parse_dict_from_argv(kwargs) >>> output = get_dbinfo(ibs, **kwargs) >>> result = (output['info_str']) >>> print(result) >>> #ibs = wbia.opendb(defaultdb='testdb1') >>> # <HACK FOR FILTERING> >>> #from wbia.expt import cfghelpers >>> #from wbia.expt import annotation_configs >>> #from wbia.init import filter_annots >>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__, >>> # annotation_configs.TEST_NAMES) >>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES, >>> # ut.get_list_column(named_defaults_dict, 'qcfg'))) >>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0] >>> #aid_list = ibs.get_valid_aids() >>> # </HACK FOR FILTERING> Example1: >>> # ENABLE_DOCTEST >>> from wbia.other.dbinfo import * # NOQA >>> import wbia >>> verbose = True >>> short = True >>> #ibs = wbia.opendb(db='GZ_ALL') >>> #ibs = wbia.opendb(db='PZ_Master0') >>> ibs = wbia.opendb('testdb1') >>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS' >>> ibs.delete_contributors(ibs.get_valid_contributor_rowids()) >>> ibs.delete_empty_nids() >>> #ibs = wbia.opendb(db='PZ_MTEST') >>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True) >>> result = (output['info_str']) >>> print(result) +============================ DB Info: testdb1 DB Notes: None DB NumContrib: 0 ---------- # Names = 7 # Names (unassociated) = 0 # Names (singleton) = 5 # Names (multiton) = 2 ---------- # Annots = 13 # Annots (unknown) = 4 # Annots (singleton) = 5 # Annots (multiton) = 4 ---------- # Img = 13 L============================ """ # TODO Database size in bytes # TODO: occurrence, contributors, etc... if aids is not None: aid_list = aids # Basic variables request_annot_subset = False _input_aid_list = aid_list # NOQA if aid_list is None: valid_aids = ibs.get_valid_aids() valid_nids = ibs.get_valid_nids() valid_gids = ibs.get_valid_gids() else: if isinstance(aid_list, str): # Hack to get experiment stats on aids acfg_name_list = [aid_list] logger.info('Specified custom aids via acfgname %s' % (acfg_name_list,)) from wbia.expt import experiment_helpers acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list( ibs, acfg_name_list ) aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list))))) # aid_list = if verbose: logger.info('Specified %d custom aids' % (len(aid_list,))) request_annot_subset = True valid_aids = aid_list valid_nids = list( set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False)) - {const.UNKNOWN_NAME_ROWID} ) valid_gids = list(set(ibs.get_annot_gids(aid_list))) # associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation valid_images = ibs.images(valid_gids) valid_annots = ibs.annots(valid_aids) # Image info if verbose: logger.info('Checking Image Info') gx2_aids = valid_images.aids if request_annot_subset: # remove annots not in this subset valid_aids_set = set(valid_aids) gx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in gx2_aids] gx2_nAnnots = np.array(list(map(len, gx2_aids))) image_without_annots = len(np.where(gx2_nAnnots == 0)[0]) gx2_nAnnots_stats = ut.repr4( ut.get_stats(gx2_nAnnots, use_median=True), nl=0, precision=2, si=True ) image_reviewed_list = ibs.get_image_reviewed(valid_gids) # Name stats if verbose: logger.info('Checking Name Info') nx2_aids = ibs.get_name_aids(valid_nids) if request_annot_subset: # remove annots not in this subset valid_aids_set = set(valid_aids) nx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in nx2_aids] associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids))) ibs.check_name_mapping_consistency(nx2_aids) if False: # Occurrence Info def compute_annot_occurrence_ids(ibs, aid_list): from wbia.algo.preproc import preproc_occurrence gid_list = ibs.get_annot_gids(aid_list) gid2_aids = ut.group_items(aid_list, gid_list) config = {'seconds_thresh': 4 * 60 * 60} flat_imgsetids, flat_gids = preproc_occurrence.wbia_compute_occurrences( ibs, gid_list, config=config, verbose=False ) occurid2_gids = ut.group_items(flat_gids, flat_imgsetids) occurid2_aids = { oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items() } return occurid2_aids import utool with utool.embed_on_exception_context: occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids) occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values()) occur_unique_nids = [ut.unique(nids) for nids in occur_nids] nid2_occurxs = ut.ddict(list) for occurx, nids in enumerate(occur_unique_nids): for nid in nids: nid2_occurxs[nid].append(occurx) nid2_occurx_single = { nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1 } nid2_occurx_resight = { nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1 } singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys()) singlesight_annot_stats = ut.get_stats( list(map(len, singlesight_encounters)), use_median=True, use_sum=True ) resight_name_stats = ut.get_stats( list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True ) # Encounter Info def break_annots_into_encounters(aids): from wbia.algo.preproc import occurrence_blackbox import datetime thresh_sec = datetime.timedelta(minutes=30).seconds posixtimes = np.array(ibs.get_annot_image_unixtimes_asfloat(aids)) # latlons = ibs.get_annot_image_gps(aids) labels = occurrence_blackbox.cluster_timespace2( posixtimes, None, thresh_sec=thresh_sec ) return labels # ave_enc_time = [np.mean(times) for lbl, times in ut.group_items(posixtimes, labels).items()] # ut.square_pdist(ave_enc_time) try: am_rowids = ibs.get_annotmatch_rowids_between_groups([valid_aids], [valid_aids])[ 0 ] aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0, am_rowids=am_rowids) undirected_tags = ibs.get_aidpair_tags( aid_pairs.T[0], aid_pairs.T[1], directed=False ) tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags)) tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags) pair_tag_info = ut.map_dict_vals(len, tag_dict) except Exception: pair_tag_info = {} # logger.info(ut.repr2(pair_tag_info)) # Annot Stats # TODO: number of images where chips cover entire image # TODO: total image coverage of annotation # TODO: total annotation overlap """ ax2_unknown = ibs.is_aid_unknown(valid_aids) ax2_nid = ibs.get_annot_name_rowids(valid_aids) assert all([nid < 0 if unknown else nid > 0 for nid, unknown in zip(ax2_nid, ax2_unknown)]), 'bad annot nid' """ # if verbose: logger.info('Checking Annot Species') unknown_annots = valid_annots.compress(ibs.is_aid_unknown(valid_annots)) species_list = valid_annots.species_texts species2_annots = valid_annots.group_items(valid_annots.species_texts) species2_nAids = {key: len(val) for key, val in species2_annots.items()} if verbose: logger.info('Checking Multiton/Singleton Species') nx2_nAnnots = np.array(list(map(len, nx2_aids))) # Seperate singleton / multitons multiton_nxs = np.where(nx2_nAnnots > 1)[0] singleton_nxs = np.where(nx2_nAnnots == 1)[0] unassociated_nxs = np.where(nx2_nAnnots == 0)[0] assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names' valid_nxs = np.hstack([multiton_nxs, singleton_nxs]) num_names_with_gt = len(multiton_nxs) # Annot Info if verbose: logger.info('Checking Annot Info') multiton_aids_list = ut.take(nx2_aids, multiton_nxs) assert len(set(multiton_nxs)) == len(multiton_nxs) if len(multiton_aids_list) == 0: multiton_aids = np.array([], dtype=np.int) else: multiton_aids = np.hstack(multiton_aids_list) assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot' singleton_aids = ut.take(nx2_aids, singleton_nxs) multiton_nid2_nannots = list(map(len, multiton_aids_list)) # Image size stats if with_imgsize: if verbose: logger.info('Checking ImageSize Info') gpath_list = ibs.get_image_paths(valid_gids) def wh_print_stats(wh_list): if len(wh_list) == 0: return '{empty}' wh_list = np.asarray(wh_list) stat_dict = collections.OrderedDict( [ ('max', wh_list.max(0)), ('min', wh_list.min(0)), ('mean', wh_list.mean(0)), ('std', wh_list.std(0)), ] ) def arr2str(var): return '[' + (', '.join(list(map(lambda x: '%.1f' % x, var)))) + ']' ret = ',\n '.join( ['%s:%s' % (key, arr2str(val)) for key, val in stat_dict.items()] ) return '{\n ' + ret + '\n}' logger.info('reading image sizes') # Image size stats img_size_list = ibs.get_image_sizes(valid_gids) img_size_stats = wh_print_stats(img_size_list) # Chip size stats annotation_bbox_list = ibs.get_annot_bboxes(valid_aids) annotation_bbox_arr = np.array(annotation_bbox_list) if len(annotation_bbox_arr) == 0: annotation_size_list = [] else: annotation_size_list = annotation_bbox_arr[:, 2:4] chip_size_stats = wh_print_stats(annotation_size_list) imgsize_stat_lines = [ (' # Img in dir = %d' % len(gpath_list)), (' Image Size Stats = %s' % (img_size_stats,)), (' * Chip Size Stats = %s' % (chip_size_stats,)), ] else: imgsize_stat_lines = [] if verbose: logger.info('Building Stats String') multiton_stats = ut.repr3( ut.get_stats(multiton_nid2_nannots, use_median=True), nl=0, precision=2, si=True ) # Time stats unixtime_list = valid_images.unixtime2 # valid_unixtime_list = [time for time in unixtime_list if time != -1] # unixtime_statstr = ibs.get_image_time_statstr(valid_gids) if ut.get_argflag('--hackshow-unixtime'): show_time_distributions(ibs, unixtime_list) ut.show_if_requested() unixtime_statstr = ut.repr3(ut.get_timestats_dict(unixtime_list, full=True), si=True) # GPS stats gps_list_ = ibs.get_image_gps(valid_gids) gpsvalid_list = [gps != (-1, -1) for gps in gps_list_] gps_list = ut.compress(gps_list_, gpsvalid_list) def get_annot_age_stats(aid_list): annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list) annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list) age_dict = ut.ddict((lambda: 0)) for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max): if max_age is None: max_age = min_age if min_age is None: min_age = max_age if max_age is None and min_age is None: logger.info('Found UNKNOWN Age: %r, %r' % (min_age, max_age,)) age_dict['UNKNOWN'] += 1 elif (min_age is None or min_age < 12) and max_age < 12: age_dict['Infant'] += 1 elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36: age_dict['Juvenile'] += 1 elif 36 <= min_age and (max_age is None or 36 <= max_age): age_dict['Adult'] += 1 return age_dict def get_annot_sex_stats(aid_list): annot_sextext_list = ibs.get_annot_sex_texts(aid_list) sextext2_aids = ut.group_items(aid_list, annot_sextext_list) sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys()) assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str( set(annot_sextext_list) - set(sex_keys) ) sextext2_nAnnots = ut.odict( [(key, len(sextext2_aids.get(key, []))) for key in sex_keys] ) # Filter 0's sextext2_nAnnots = { key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0 } return sextext2_nAnnots def get_annot_qual_stats(ibs, aid_list): annots = ibs.annots(aid_list) qualtext2_nAnnots = ut.order_dict_by( ut.map_vals(len, annots.group_items(annots.quality_texts)), list(ibs.const.QUALITY_TEXT_TO_INT.keys()), ) return qualtext2_nAnnots def get_annot_viewpoint_stats(ibs, aid_list): annots = ibs.annots(aid_list) viewcode2_nAnnots = ut.order_dict_by( ut.map_vals(len, annots.group_items(annots.viewpoint_code)), list(ibs.const.VIEW.CODE_TO_INT.keys()) + [None], ) return viewcode2_nAnnots if verbose: logger.info('Checking Other Annot Stats') qualtext2_nAnnots = get_annot_qual_stats(ibs, valid_aids) viewcode2_nAnnots = get_annot_viewpoint_stats(ibs, valid_aids) agetext2_nAnnots = get_annot_age_stats(valid_aids) sextext2_nAnnots = get_annot_sex_stats(valid_aids) if verbose: logger.info('Checking Contrib Stats') # Contributor Statistics # hack remove colon for image alignment def fix_tag_list(tag_list): return [None if tag is None else tag.replace(':', ';') for tag in tag_list] image_contributor_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids)) annot_contributor_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids)) contributor_tag_to_gids = ut.group_items(valid_gids, image_contributor_tags) contributor_tag_to_aids = ut.group_items(valid_aids, annot_contributor_tags) contributor_tag_to_qualstats = { key: get_annot_qual_stats(ibs, aids) for key, aids in six.iteritems(contributor_tag_to_aids) } contributor_tag_to_viewstats = { key: get_annot_viewpoint_stats(ibs, aids) for key, aids in six.iteritems(contributor_tag_to_aids) } contributor_tag_to_nImages = { key: len(val) for key, val in six.iteritems(contributor_tag_to_gids) } contributor_tag_to_nAnnots = { key: len(val) for key, val in six.iteritems(contributor_tag_to_aids) } if verbose: logger.info('Summarizing') # Summarize stats num_names = len(valid_nids) num_names_unassociated = len(valid_nids) - len(associated_nids) num_names_singleton = len(singleton_nxs) num_names_multiton = len(multiton_nxs) num_singleton_annots = len(singleton_aids) num_multiton_annots = len(multiton_aids) num_unknown_annots = len(unknown_annots) num_annots = len(valid_aids) if with_bytes: if verbose: logger.info('Checking Disk Space') ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir())) dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir())) imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir())) cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir())) if True: if verbose: logger.info('Check asserts') try: bad_aids = np.intersect1d(multiton_aids, unknown_annots) _num_names_total_check = ( num_names_singleton + num_names_unassociated + num_names_multiton ) _num_annots_total_check = ( num_unknown_annots + num_singleton_annots + num_multiton_annots ) assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids' assert _num_names_total_check == num_names, 'inconsistent num names' # if not request_annot_subset: # dont check this if you have an annot subset assert _num_annots_total_check == num_annots, 'inconsistent num annots' except Exception as ex: ut.printex( ex, keys=[ '_num_names_total_check', 'num_names', '_num_annots_total_check', 'num_annots', 'num_names_singleton', 'num_names_multiton', 'num_unknown_annots', 'num_multiton_annots', 'num_singleton_annots', ], ) raise # Get contributor statistics contributor_rowids = ibs.get_valid_contributor_rowids() num_contributors = len(contributor_rowids) # print num_tabs = 5 def align2(str_): return ut.align(str_, ':', ' :') def align_dict2(dict_): str_ = ut.repr2(dict_, si=True) return align2(str_) header_block_lines = [('+============================')] + ( [ ('+ singleton := single sighting'), ('+ multiton := multiple sightings'), ('--' * num_tabs), ] if not short and with_header else [] ) source_block_lines = [ ('DB Info: ' + ibs.get_dbname()), ('DB Notes: ' + ibs.get_dbnotes()), ('DB NumContrib: %d' % num_contributors), ] bytes_block_lines = ( [ ('--' * num_tabs), ('DB Bytes: '), (' +- dbdir nBytes: ' + dbdir_space), (' | +- _ibsdb nBytes: ' + ibsdir_space), (' | | +-imgdir nBytes: ' + imgdir_space), (' | | +-cachedir nBytes: ' + cachedir_space), ] if with_bytes else [] ) name_block_lines = [ ('--' * num_tabs), ('# Names = %d' % num_names), ('# Names (unassociated) = %d' % num_names_unassociated), ('# Names (singleton) = %d' % num_names_singleton), ('# Names (multiton) = %d' % num_names_multiton), ] subset_str = ' ' if not request_annot_subset else '(SUBSET)' annot_block_lines = [ ('--' * num_tabs), ('# Annots %s = %d' % (subset_str, num_annots,)), ('# Annots (unknown) = %d' % num_unknown_annots), ('# Annots (singleton) = %d' % num_singleton_annots), ('# Annots (multiton) = %d' % num_multiton_annots), ] annot_per_basic_block_lines = ( [ ('--' * num_tabs), ('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)), ('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)), ('# Annots per Species = %s' % (align_dict2(species2_nAids),)), ] if not short else [] ) occurrence_block_lines = ( [ ('--' * num_tabs), # ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)), # ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)), ('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)), ] if not short else [] ) annot_per_qualview_block_lines = [ None if short else '# Annots per Viewpoint = %s' % align_dict2(viewcode2_nAnnots), None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots), ] annot_per_agesex_block_lines = ( [ '# Annots per Age = %s' % align_dict2(agetext2_nAnnots), '# Annots per Sex = %s' % align_dict2(sextext2_nAnnots), ] if not short and with_agesex else [] ) contributor_block_lines = ( [ '# Images per contributor = ' + align_dict2(contributor_tag_to_nImages), '# Annots per contributor = ' + align_dict2(contributor_tag_to_nAnnots), '# Quality per contributor = ' + ut.repr2(contributor_tag_to_qualstats, sorted_=True), '# Viewpoint per contributor = ' + ut.repr2(contributor_tag_to_viewstats, sorted_=True), ] if with_contrib else [] ) img_block_lines = [ ('--' * num_tabs), ('# Img = %d' % len(valid_gids)), None if short else ('# Img reviewed = %d' % sum(image_reviewed_list)), None if short else ('# Img with gps = %d' % len(gps_list)), # ('# Img with timestamp = %d' % len(valid_unixtime_list)), None if short else ('Img Time Stats = %s' % (align2(unixtime_statstr),)), ] info_str_lines = ( header_block_lines + bytes_block_lines + source_block_lines + name_block_lines + annot_block_lines + annot_per_basic_block_lines + occurrence_block_lines + annot_per_qualview_block_lines + annot_per_agesex_block_lines + img_block_lines + contributor_block_lines + imgsize_stat_lines + [('L============================')] ) info_str = '\n'.join(ut.filter_Nones(info_str_lines)) info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag)) if verbose: logger.info(info_str2) locals_ = locals() return locals_
def autogen_parts(binding_name=None): r""" CommandLine: python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=set_dataset python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=add_points python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_point --py python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=used_memory --py python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_points --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=veclen --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=size --py --c python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=clean_removed_points --py --c Ignore: # Logic goes here ~/code/flann/src/cpp/flann/algorithms/kdtree_index.h ~/code/flann/src/cpp/flann/util/serialization.h ~/code/flann/src/cpp/flann/util/dynamic_bitset.h # Bindings go here ~/code/flann/src/cpp/flann/flann.cpp ~/code/flann/src/cpp/flann/flann.h # Contains stuff for the flann namespace like flann::log_level # Also has Index with # Matrix<ElementType> features; SEEMS USEFUL ~/code/flann/src/cpp/flann/flann.hpp # Wrappers go here ~/code/flann/src/python/pyflann/flann_ctypes.py ~/code/flann/src/python/pyflann/index.py ~/local/build_scripts/flannscripts/autogen_bindings.py Example: >>> # ENABLE_DOCTEST >>> from autogen_bindings import * # NOQA >>> result = autogen_parts() >>> print(result) """ #ut.get_dynlib_exports(pyflann.flannlib._name) #flannlib if binding_name is None: binding_name = ut.get_argval('--bindingname', type_=str, default='add_points') # Variable names used in flann cpp source simple_c_to_ctypes = { 'void': None, 'char*': 'c_char_p', 'unsigned int': 'c_uint', 'int': 'c_int', 'float': 'c_float', 'float*': 'POINTER(c_float)', 'flann_index_t': 'FLANN_INDEX', 'FLANNParameters*': 'POINTER(FLANNParameters)', 'Distance::ResultType*': "ndpointer(%(restype)s, flags='aligned, c_contiguous, writeable')", 'Distance::ElementType*': "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')", 'typename Distance::ElementType*': "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')", 'typename Distance::ResultType*': "ndpointer(%(restype), ndim=2, flags='aligned, c_contiguous, writeable')", } templated_ctype_map = { 'filename': 'char*', 'level': 'int', 'rows': 'int', 'cols': 'int', 'point_id': 'unsigned int', 'num': 'int', 'max_nn': 'int', 'tcount': 'int', 'nn': 'int', 'radius': 'float', 'clusters': 'int', 'rebuild_threshold': 'float', 'index_ptr': 'flann_index_t', 'flann_params': 'FLANNParameters*', 'speedup': 'float*', 'id_list': 'int*', #'indices' : 'int*', 'dataset': 'typename Distance::ElementType*', 'points': 'typename Distance::ElementType*', 'query': 'typename Distance::ElementType*', 'query1d': 'typename Distance::ElementType*', 'testset': 'typename Distance::ElementType*', 'dists': 'typename Distance::ResultType*', 'dists1d': 'typename Distance::ResultType*', 'result_centers': 'typename Distance::ResultType*', 'result_ids': 'int*', } # Python ctype bindings python_ctype_map = { 'flann_params': 'POINTER(FLANNParameters)', 'id_list': "ndpointer(int32, ndim=1, flags='aligned, c_contiguous')", #'indices' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous, writeable')", 'query1d': "ndpointer(%(numpy)s, ndim=1, flags='aligned, c_contiguous')", 'dists1d': "ndpointer(%(restype), ndim=1, flags='aligned, c_contiguous, writeable')", 'result_ids': "ndpointer(int32, ndim=2, flags='aligned, c_contiguous, writeable')", #'query' : "ndpointer(float64, ndim=1, flags='aligned, c_contiguous')", } for key, val in templated_ctype_map.items(): if key not in python_ctype_map: python_ctype_map[key] = simple_c_to_ctypes[val] binding_def = define_flann_bindings(binding_name) docstr_cpp = binding_def['docstr_cpp'] docstr_py = binding_def['docstr_py'] cpp_binding_name = binding_def['cpp_binding_name'] return_type = binding_def['return_type'] binding_argnames = binding_def['binding_argnames'] c_source = binding_def['c_source'] py_source = binding_def['py_source'] optional_args = binding_def['optional_args'] py_alias = binding_def['py_alias'] py_args = binding_def['py_args'] binding_args = [ python_ctype_map[name] + ', # ' + name for name in binding_argnames ] binding_args_str = ' ' + '\n '.join(binding_args) callargs = ', '.join(binding_argnames) pycallargs = ', '.join( [name for name in ['self'] + binding_argnames if name != 'index_ptr']) if py_args is None: py_args = binding_argnames pyinputargs = pycallargs # FIXME else: pyinputargs = ', '.join(['self'] + py_args) pyrestype = simple_c_to_ctypes[return_type] if py_alias is None: py_binding_name = binding_name else: py_binding_name = py_alias binding_def['py_binding_name'] = py_binding_name #### flann_ctypes.py flann_ctypes_codeblock = ut.codeblock(''' flann.{binding_name} = {{}} define_functions(r""" flannlib.flann_{binding_name}_%(C)s.restype = {pyrestype} flannlib.flann_{binding_name}_%(C)s.argtypes = [ {binding_args_str} ] flann.{binding_name}[%(numpy)s] = flannlib.flann_{binding_name}_%(C)s """) ''').format(binding_name=binding_name, binding_args_str=binding_args_str, pyrestype=pyrestype) #### index.py default_py_source_parts = [] if 'pts' in py_args: default_py_source_parts.append( ut.codeblock(''' if pts.dtype.type not in allowed_types: raise FLANNException('Cannot handle type: %s' % pts.dtype) pts = ensure_2d_array(pts, default_flags) ''')) default_py_source_parts.append( ut.codeblock(''' rows = pts.shape[0] raise NotImplementedError('requires custom implementation') flann.{binding_name}[self.__curindex_type](self.__curindex, {pycallargs}) ''')) if py_source is None: py_source = '\n'.join(default_py_source_parts) flann_index_codeblock = ut.codeblock(r''' def {py_binding_name}({pyinputargs}): """''' + ut.indent('\n' + docstr_py, ' ') + ''' """''' + '\n' + ut.indent(py_source, ' ') + ''' ''').format(binding_name=binding_name, pycallargs=pycallargs, py_binding_name=py_binding_name, pyinputargs=pyinputargs, py_source=py_source) #### flann.cpp #// {binding_name} BEGIN CPP BINDING #template <typename Distance> #{return_type} __flann_{binding_name}({templated_args}) flann_cpp_code_fmtstr_ = ut.codeblock(r''' {{''' + '\n' + ut.indent(c_source, ' ' * (4 * 3)) + r''' }} ''') #implicit_type_bindings_fmtstr = ut.codeblock( # NOQA # ''' # DISTANCE_TYPE_BINDINGS({return_type}, {binding_name}, # SINGLE_ARG({T_typed_sigargs_cpp}), # SINGLE_ARG({callargs})) # ''' #) #type_bindings_fmtstr = implicit_type_bindings_fmtstr explicit_type_bindings_part3_fmtstr = ut.codeblock(r''' {{ if (flann_distance_type==FLANN_DIST_EUCLIDEAN) {{ return __flann_{binding_name}<L2<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_MANHATTAN) {{ return __flann_{binding_name}<L1<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_MINKOWSKI) {{ return __flann_{binding_name}<MinkowskiDistance<T> >({callargs}{minkowski_option}); }} else if (flann_distance_type==FLANN_DIST_HIST_INTERSECT) {{ return __flann_{binding_name}<HistIntersectionDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_HELLINGER) {{ return __flann_{binding_name}<HellingerDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_CHI_SQUARE) {{ return __flann_{binding_name}<ChiSquareDistance<T> >({callargs}); }} else if (flann_distance_type==FLANN_DIST_KULLBACK_LEIBLER) {{ return __flann_{binding_name}<KL_Divergence<T> >({callargs}); }} else {{ Logger::error( "Distance type unsupported in the C bindings, use the C++ bindings instead\n"); {errorhandle} }} }} ''') # c binding body c_bodyblock_fmtstr = ut.codeblock(''' {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2}) {{ {ifreturns}_flann_{binding_name}{iftemplate}({callargs}); }} ''') #### flann.h # c binding header #c_headersig_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2});' c_headersig_part1_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}(' #### format cpp parts binding_argtypes = ut.dict_take(templated_ctype_map, binding_argnames) _fix_T = { 'typename Distance::ElementType*': 'T*', 'typename Distance::ResultType*': 'R*', } templated_ctype_map_cpp = { name: _fix_T.get(type_, type_) for name, type_ in templated_ctype_map.items() } binding_argtypes_cpp = ut.dict_take(templated_ctype_map_cpp, binding_argnames) binding_sigargs_cpp = [ type_ + ' ' + name for type_, name in zip(binding_argtypes_cpp, binding_argnames) ] templated_bindings = [ templated_ctype_map[name] + ' ' + name for name in binding_argnames ] if optional_args is not None: templated_bindings += optional_args minkowski_option = ', MinkowskiDistance<T>(flann_distance_order)' else: minkowski_option = '' templated_args = ', '.join(templated_bindings) if binding_name == 'remove_point': # HACK templated_args += '_uint' cpp_sig_part1 = '{return_type} __flann_{binding_name}('.format( return_type=return_type, binding_name=binding_name) maxlen = 100 cpp_sig_ = ut.packstr(cpp_sig_part1 + templated_args, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(cpp_sig_part1)) cpp_sig = cpp_sig_[:-2] + ')' flann_cpp_code_fmtstr = 'template <typename Distance>\n' + cpp_sig + '\n' + flann_cpp_code_fmtstr_ #print(cpp_sig) T_typed_sigargs_cpp = ', '.join(binding_sigargs_cpp) used_template_list = [] used_template_list.append('typename T') if 'typename Distance::ResultType*' in binding_argtypes: used_template_list.append('typename R') used_templates = ', '.join(used_template_list) type_binding_part1 = 'template <{used_templates}>'.format( used_templates=used_templates) type_binding_part2_ = '{return_type} _flann_{binding_name}('.format( return_type=return_type, binding_name=binding_name) maxlen = 100 cpp_type_sig_ = ut.packstr(type_binding_part2_ + T_typed_sigargs_cpp, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(type_binding_part2_)) cpp_type_sig = cpp_type_sig_[:-2] + ')' type_binding_part12 = type_binding_part1 + '\n' + cpp_type_sig explicit_type_bindings_fmtstr = type_binding_part12 + '\n' + explicit_type_bindings_part3_fmtstr flann_cpp_codeblock_fmtstr = flann_cpp_code_fmtstr + '\n\n\n' + explicit_type_bindings_fmtstr + '\n' if return_type == 'int': errorhandle = 'return -1;' elif return_type == 'flann_index_t': errorhandle = 'return NULL;' else: errorhandle = 'throw 0;' # print('------') # print('flann_cpp_codeblock_fmtstr.format = %s' % (flann_cpp_codeblock_fmtstr,)) try: flann_cpp_codeblock = flann_cpp_codeblock_fmtstr.format( cpp_binding_name=cpp_binding_name, minkowski_option=minkowski_option, binding_name=binding_name, #templated_args=templated_args, callargs=callargs, #T_typed_sigargs_cpp=T_typed_sigargs_cpp, errorhandle=errorhandle, used_templates=used_templates, return_type=return_type) except KeyError as ex: ut.printex(ex, keys=['binding_name']) raise dataset_types = [ '', 'float', 'double', 'byte', 'int', ] #### format c parts c_header_sigs = [] c_body_blocks = [] templated_ctype_map_c = templated_ctype_map.copy() #templated_ctype_map_c['dataset'] = 'float' _fix_explicit_ctype = { '': 'float', 'byte': 'unsigned char', } # For each explicit c type for dataset_type in dataset_types: T_type = _fix_explicit_ctype.get(dataset_type, dataset_type) if dataset_type != '': signame_type = '_' + dataset_type else: signame_type = dataset_type R_type = 'float' if T_type != 'double' else 'double' dstype = T_type + '*' rstype = R_type + '*' # Overwrite template types with explicit c types needstemplate = True for type_, name in zip(binding_argtypes, binding_argnames): if type_ == 'typename Distance::ElementType*': templated_ctype_map_c[name] = dstype needstemplate = False if type_ == 'typename Distance::ResultType*': templated_ctype_map_c[name] = rstype needstemplate = False if type_ == 'Distance::ResultType*': templated_ctype_map_c[name] = rstype needstemplate = False #if type_ == 'struct FLANNParameters*': # # hack # templated_ctype_map_c[name] = 'FLANNParameters*' # HACK if binding_name == 'load_index' or binding_name == 'add_points': needstemplate = True if binding_name == 'build_index': needstemplate = True binding_argtypes2 = ut.dict_take(templated_ctype_map_c, binding_argnames) binding_sigargs2 = [ type_ + ' ' + name for type_, name in zip(binding_argtypes2, binding_argnames) ] T_typed_sigargs2 = ', '.join(binding_sigargs2) T_typed_sigargs2_nl = ',\n'.join(binding_sigargs2) if needstemplate: iftemplate = '<{T_type}>'.format(T_type=T_type) else: iftemplate = '' if return_type != 'void': ifreturns = 'return ' else: ifreturns = '' bodyblock = c_bodyblock_fmtstr.format( signame_type=signame_type, T_typed_sigargs2=T_typed_sigargs2, binding_name=binding_name, callargs=callargs, iftemplate=iftemplate, ifreturns=ifreturns, return_type=return_type) header_line_part1 = c_headersig_part1_fmtstr.format( signame_type=signame_type, binding_name=binding_name, return_type=return_type) header_line = header_line_part1 + ut.indent( T_typed_sigargs2_nl, ' ' * len(header_line_part1)).lstrip(' ') + ');' # Hack for header header_line = header_line.replace( 'FLANNParameters* flann_params', 'struct FLANNParameters* flann_params') #header_line = c_headersig_fmtstr.format(signame_type=signame_type, # T_typed_sigargs2=T_typed_sigargs2, # binding_name=binding_name, # return_type=return_type) c_header_sigs.append(header_line) c_body_blocks.append(bodyblock) flann_cpp_codeblock += '\n' + '\n'.join(c_body_blocks) #flann_cpp_codeblock += '\n' + '// {binding_name} END'.format(binding_name=binding_name) #BEGIN {binding_name} flann_h_codeblock = ut.codeblock(r''' /** {docstr_cpp} */ ''').format(docstr_cpp=docstr_cpp, binding_name=binding_name) flann_h_codeblock += '\n\n' + '\n\n'.join(c_header_sigs) blocks_dict = {} import re flann_index_codeblock = ut.indent(flann_index_codeblock, ' ') blocks_dict['flann_ctypes.py'] = flann_ctypes_codeblock blocks_dict['index.py'] = flann_index_codeblock blocks_dict['flann.h'] = flann_h_codeblock blocks_dict['flann.cpp'] = flann_cpp_codeblock for key in blocks_dict.keys(): blocks_dict[key] = re.sub('\n\s+\n', '\n\n', blocks_dict[key]) # , flags=re.MULTILINE) blocks_dict[key] = re.sub('\s\s+\n', '\n', blocks_dict[key]) pass if ut.get_argflag('--py'): print('\n\n# ---------------\n\n') print('GOES IN flann_ctypes.py') print('\n\n# ---------------\n\n') print(flann_ctypes_codeblock) print('\n\n# ---------------\n\n') print('GOES IN index.py') print('\n\n# ---------------\n\n') print(flann_index_codeblock) if ut.get_argflag('--c'): print('\n\n# ---------------\n\n') print('GOES IN flann.h') print('\n\n# ---------------\n\n') print(flann_h_codeblock) print('\n\n# ---------------\n\n') print('GOES IN flann.cpp') print('\n\n# ---------------\n\n') print(flann_cpp_codeblock) return blocks_dict, binding_def
def autogen_explicit_injectable_metaclass(classname, regen_command=None, conditional_imports=None): r""" Args: classname (?): Returns: ?: CommandLine: python -m utool.util_class --exec-autogen_explicit_injectable_metaclass Example: >>> # DISABLE_DOCTEST >>> from utool.util_class import * # NOQA >>> from utool.util_class import __CLASSTYPE_ATTRIBUTES__ # NOQA >>> import ibeis >>> import ibeis.control.IBEISControl >>> classname = ibeis.control.controller_inject.CONTROLLER_CLASSNAME >>> result = autogen_explicit_injectable_metaclass(classname) >>> print(result) """ import utool as ut vals_list = [] def make_redirect(func): # PRESERVES ALL SIGNATURES WITH EXECS src_fmt = r''' def {funcname}{defsig}: """ {orig_docstr}""" return {orig_funcname}{callsig} ''' from utool._internal import meta_util_six orig_docstr = meta_util_six.get_funcdoc(func) funcname = meta_util_six.get_funcname(func) orig_funcname = modname.split('.')[-1] + '.' + funcname orig_docstr = '' if orig_docstr is None else orig_docstr import textwrap # Put wrapped function into a scope import inspect argspec = inspect.getargspec(func) (args, varargs, varkw, defaults) = argspec defsig = inspect.formatargspec(*argspec) callsig = inspect.formatargspec(*argspec[0:3]) src_fmtdict = dict(funcname=funcname, orig_funcname=orig_funcname, defsig=defsig, callsig=callsig, orig_docstr=orig_docstr) src = textwrap.dedent(src_fmt).format(**src_fmtdict) return src src_list = [] for classkey, vals in __CLASSTYPE_ATTRIBUTES__.items(): modname = classkey[1] if classkey[0] == classname: vals_list.append(vals) for func in vals: src = make_redirect(func) src = ut.indent(src) src = '\n'.join([_.rstrip() for _ in src.split('\n')]) src_list.append(src) if regen_command is None: regen_command = 'FIXME None given' module_header = ut.codeblock( """ # -*- coding: utf-8 -*- """ + ut.TRIPLE_DOUBLE_QUOTE + """ Static file containing autogenerated functions for {classname} Autogenerated on {autogen_time} RegenCommand: {regen_command} """ + ut.TRIPLE_DOUBLE_QUOTE + """ from __future__ import absolute_import, division, print_function import utool as ut """).format( autogen_time=ut.get_timestamp(), regen_command=regen_command, classname=classname) depends_module_block = autogen_import_list(classname, conditional_imports) inject_statement_fmt = ("print, rrr, profile = " "ut.inject2(__name__, '[autogen_explicit_inject_{classname}]')") inject_statement = inject_statement_fmt.format(classname=classname) source_block_lines = [ module_header, depends_module_block, inject_statement, '\n', 'class ExplicitInject' + classname + '(object):', ] + src_list source_block = '\n'.join(source_block_lines) source_block = ut.autoformat_pep8(source_block, aggressive=2) return source_block
def make_docstr_block(header, block): import utool as ut indented_block = '\n' + ut.indent(block) docstr_block = ''.join([header, ':', indented_block]) return docstr_block
def autogen_ibeis_runtest(): """ special case to generate tests script for IBEIS Example: >>> from autogen_test_script import * # NOQA >>> test_script = autogen_ibeis_runtest() >>> print(test_script) CommandLine: python -c "import utool; utool.autogen_ibeis_runtest()" python -c "import utool; print(utool.autogen_ibeis_runtest())" python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh chmod +x run_tests.sh """ quick_tests = [ 'ibeis/tests/assert_modules.py' ] #test_repos = [ # '~/code/ibeis' # '~/code/vtool' # '~/code/hesaff' # '~/code/guitool' #] #test_pattern = [ # '~/code/ibeis/test_ibs*.py' #] test_argvs = '--quiet --noshow' misc_pats = [ 'test_utool_parallel.py', 'test_pil_hash.py', ] repodir = '~/code/ibeis' testdir = 'ibeis/tests' exclude_list = [] # Hacky, but not too bad way of getting in doctests # Test to see if doctest_funcs appears after main # Do not doctest these modules #exclude_doctests_fnames = set(['template_definitions.py', # 'autogen_test_script.py']) #exclude_dirs = [ # '_broken', # 'old', # 'tests', # 'timeits', # '_scripts', # '_timeits', # '_doc', # 'notebook', #] #dpath_list = ['ibeis'] #doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) # Verbosity to show which modules at least have some tests #untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs) #print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames)) #print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list)) # The implict list is exactly the code we will use to make the implicit list #module_list = None #doctest_modname_list = None #'export_subset.py', #'_autogen_ibeiscontrol_funcs.py', #'randomforest.py', implicit_build_modlist_str = ut.codeblock( ''' import sys exclude_doctests_fnames = set([ 'template_definitions.py', 'autogen_test_script.py', ]) exclude_dirs = [ '_broken', 'old', 'tests', 'timeits', '_scripts', '_timeits', '_doc', 'notebook', ] dpath_list = ['ibeis'] doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs) for modname in doctest_modname_list: exec('import ' + modname, globals(), locals()) module_list = [sys.modules[name] for name in doctest_modname_list] ''' ) globals_ = globals() locals_ = locals() exec(implicit_build_modlist_str, globals_, locals_) module_list = locals_['module_list'] doctest_modname_list = locals_['doctest_modname_list'] #module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list] #for modname in doctest_modname_list: # exec('import ' + modname, globals(), locals()) #module_list = [sys.modules[name] for name in doctest_modname_list] #print('\n'.join(testcmds)) #print('\n'.join(['python -m ' + modname for modname in doctest_modname_list])) import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list]) modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list])) explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str)) build_modlist_str = implicit_build_modlist_str #build_modlist_str = explicit_build_modlist_str pyscript_fmtstr = ut.codeblock( r''' #!/usr/bin/env python from __future__ import absolute_import, division, print_function import utool as ut def run_tests(): # Build module list and run tests {build_modlist_str} ut.doctest_module_list(module_list) if __name__ == '__main__': import multiprocessing multiprocessing.freeze_support() run_tests() ''' ) pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip()) pyscript_text = ut.autofix_codeblock(pyscript_text) # BUILD OLD SHELL RUN TESTS HARNESS testcmds_ = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST', testslow=True) testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_] test_headers = [ # title, default, module, testpattern ut.def_test('VTOOL', dpath='vtool/tests', pat=['test*.py'], modname='vtool'), ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']), ut.def_test('IBEIS', dpath=testdir, pat=['test_ibs*.py', 'test_delete*.py'], default=False), ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']), ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']), ut.def_test('MISC', dpath=testdir, pat=misc_pats), ut.def_test('OTHER', dpath=testdir, pat='OTHER'), ut.def_test('HESAFF', dpath='pyhesaff/tests', pat=['test_*.py'], modname='pyhesaff'), ut.def_test('DOC', testcmds=testcmds, default=True) ] # Referencs: https://docs.python.org/2/library/runpy.html shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list) #print(pyscript_text) return shscript_text, pyscript_text
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None, repodir=None, exclude_list=[]): """ Autogeneration function TODO move to util_autogen or just depricate Examples: >>> from utool.util_tests import * # NOQA >>> import utool # NOQA >>> testdirs = ['~/code/ibeis/test_ibs*.py'] """ import utool as ut from os.path import relpath, join, dirname # NOQA exclude_list += ['__init__.py'] # General format of the testing script script_fmtstr = ut.codeblock(r''' #!/bin/bash # Runs all tests # Win32 path hacks export CWD=$(pwd) export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")" # <CORRECT_PYTHON> # GET CORRECT PYTHON ON ALL PLATFORMS export SYSNAME="$(expr substr $(uname -s) 1 10)" if [ "$SYSNAME" = "MINGW32_NT" ]; then export PYEXE=python else if [ "$PYMAJOR" = "3" ]; then # virtual env? export PYEXE=python else export PYEXE=python2.7 fi fi # </CORRECT_PYTHON> PRINT_DELIMETER() {{ printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n" }} export TEST_ARGV="{test_argvs} $@" {dirdef_block} # Default tests to run set_test_flags() {{ export DEFAULT=$1 {testdefault_block} }} set_test_flags OFF {testdefaulton_block} # Parse for bash commandline args for i in "$@" do case $i in --testall) set_test_flags ON ;; esac {testcmdline_block} done BEGIN_TESTS() {{ cat <<EOF {runtests_bubbletext} EOF echo "BEGIN: TEST_ARGV=$TEST_ARGV" PRINT_DELIMETER num_passed=0 num_ran=0 export FAILED_TESTS='' }} RUN_TEST() {{ echo "RUN_TEST: $@" export TEST="$PYEXE $@ $TEST_ARGV" $TEST export RETURN_CODE=$? echo "RETURN_CODE=$RETURN_CODE" PRINT_DELIMETER num_ran=$(($num_ran + 1)) if [ "$RETURN_CODE" == "0" ] ; then num_passed=$(($num_passed + 1)) fi if [ "$RETURN_CODE" != "0" ] ; then export FAILED_TESTS="$FAILED_TESTS\n$TEST" fi }} END_TESTS() {{ echo "RUN_TESTS: DONE" if [ "$FAILED_TESTS" != "" ] ; then echo "-----" printf "Failed Tests:" printf "$FAILED_TESTS\n" printf "$FAILED_TESTS\n" >> failed_shelltests.txt echo "-----" fi echo "$num_passed / $num_ran tests passed" }} #--------------------------------------------- # START TESTS BEGIN_TESTS {quicktest_block} {test_block} #--------------------------------------------- # END TESTING END_TESTS ''') testcmdline_fmtstr = ut.codeblock(r''' case $i in --notest{header_lower}) export {testflag}=OFF ;; esac case $i in --test{header_lower}) export {testflag}=ON ;; esac ''') header_test_block_fmstr = ut.codeblock(r''' #--------------------------------------------- #{header_text} if [ "${testflag}" = "ON" ] ; then cat <<EOF {header_bubble_text} EOF {testlines_block} fi ''') #specialargv = '--noshow' specialargv = '' testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}' testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}' def format_testline(fpath, dirvar): if dirvar is None: return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv) else: return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv) default_flag_line_list = [] defaulton_flag_line_list = [] testcmdline_list = [] dirdef_list = [] header_test_block_list = [] known_tests = ut.ddict(list) # Tests to always run if quick_tests is not None: quicktest_block = '\n'.join( ['# Quick Tests (always run)'] + ['RUN_TEST ' + testline for testline in quick_tests]) else: quicktest_block = '# No quick tests' # Loop over different test types for testdef_tup in test_headers: header, default, modname, dpath, pats, testcmds = testdef_tup # Build individual test type information header_upper = header.upper() header_lower = header.lower() testflag = header_upper + '_TEST' if modname is not None: dirvar = header_upper + '_DIR' dirdef = ''.join([ 'export {dirvar}=$($PYEXE -c "', 'import os, {modname};', 'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))', '")' ]).format(dirvar=dirvar, modname=modname) dirdef_list.append(dirdef) else: dirvar = None # Build test dir #dirvar = header_upper + '_DIR' #dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname) #dirdef_list.append(dirdef) # Build command line flags default_flag_line = 'export {testflag}=$DEFAULT'.format( testflag=testflag) if default: defaulton_flag_line = 'export {testflag}=ON'.format( testflag=testflag) defaulton_flag_line_list.append(defaulton_flag_line) testcmdline_fmtdict = dict( header_lower=header_lower, testflag=testflag, ) testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict) #ut.ls(dpath) # VERY HACK BIT OF CODE # Get list of tests from patterns if testcmds is None: if modname is not None: module = __import__(modname) repo_path = dirname(dirname(module.__file__)) else: repo_path = repodir dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath)) if header_upper == 'OTHER': # Hacky way to grab any other tests not explicitly seen in this directory _testfpath_list = list( set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_])) #_testfpath_list = ut.glob(dpath_, '*.py') #set(known_tests[dpath_]) else: _testfpath_list = ut.flatten( [ut.glob(dpath_, pat) for pat in pats]) def not_excluded(x): return not any( [x.find(exclude) > -1 for exclude in exclude_list]) _testfpath_list = list(filter(not_excluded, _testfpath_list)) known_tests[dpath_].extend(_testfpath_list) #print(_testfpath_list) testfpath_list = [ util_path.unixjoin(dpath, relpath(fpath, dpath_)) for fpath in _testfpath_list ] testline_list = [ format_testline(fpath, dirvar) for fpath in testfpath_list ] else: testline_list = testcmds testlines_block = ut.indentjoin(testline_list).strip('\n') # Construct test block for this type header_text = header_upper + ' TESTS' headerfont = 'cybermedium' header_bubble_text = ut.indent( ut.bubbletext(header_text, headerfont).strip()) header_test_block_dict = dict( testflag=testflag, header_text=header_text, testlines_block=testlines_block, header_bubble_text=header_bubble_text, ) header_test_block = header_test_block_fmstr.format( **header_test_block_dict) # Append to script lists header_test_block_list.append(header_test_block) default_flag_line_list.append(default_flag_line) testcmdline_list.append(testcmdline) runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge') test_block = '\n'.join(header_test_block_list) dirdef_block = '\n'.join(dirdef_list) testdefault_block = ut.indent('\n'.join(default_flag_line_list)) testdefaulton_block = '\n'.join(defaulton_flag_line_list) testcmdline_block = '\n'.join(testcmdline_list) script_fmtdict = dict( quicktest_block=quicktest_block, runtests_bubbletext=runtests_bubbletext, test_argvs=test_argvs, dirdef_block=dirdef_block, testdefault_block=testdefault_block, testdefaulton_block=testdefaulton_block, testcmdline_block=testcmdline_block, test_block=test_block, ) script_text = script_fmtstr.format(**script_fmtdict) return script_text