def get_inspect_str(qres):
assert_qres(qres)
nFeatMatch_list = get_num_feats_in_matches(qres)
nFeatMatch_stats = utool.mystats(nFeatMatch_list)
top_lbl = utool.unindent('''
top aids
scores
ranks''').strip()
top_aids = qres.get_top_aids(num=5)
top_scores = qres.get_aid_scores(top_aids)
top_ranks = qres.get_aid_ranks(top_aids)
top_stack = np.vstack((top_aids, top_scores, top_ranks))
top_stack = np.array(top_stack, dtype=np.int32)
top_str = str(top_stack)
inspect_str = '\n'.join([
'QueryResult',
'qaid=%r ' % qres.qaid,
utool.horiz_string(top_lbl, ' ', top_str),
'num Feat Matches stats:',
utool.indent(utool.dict_str(nFeatMatch_stats)),
])
inspect_str = utool.indent(inspect_str, '[INSPECT] ')
return inspect_str
def check_importable(rman):
import utool as ut
label = " %s" % rman.label if rman.label else rman.label
missing = []
print("Checking if%s modules are importable" % (label,))
msg_list = []
recommended_fixes = []
for repo in rman.repos:
flag, msg, errors = repo.check_importable()
if not flag:
msg_list.append(" * !!!%s REPO %s HAS IMPORT ISSUES" % (label.upper(), repo))
if any([str(ex).find("undefined symbol") > -1 for ex in errors]):
recommended_fixes.append("rebuild")
else:
recommended_fixes.append(None)
if ut.VERBOSE:
msg_list.append(ut.indent(msg, " "))
missing.append(repo)
else:
if ut.VERBOSE:
msg_list.append(ut.indent(msg, " "))
print("\n".join(msg_list))
problems = list(zip(missing, recommended_fixes))
return problems
def _build_typed_params_kwargs_docstr_block(typed_params):
r"""
Args:
typed_params (dict):
CommandLine:
python -m pyhesaff build_typed_params_docstr
Example:
>>> # DISABLE_DOCTEST
>>> from pyhesaff._pyhesaff import * # NOQA
>>> typed_params = HESAFF_TYPED_PARAMS
>>> result = build_typed_params_docstr(typed_params)
>>> print(result)
"""
kwargs_lines = []
for tup in typed_params:
type_, name, default = tup
typestr = str(type_).replace('<class \'ctypes.c_',
'').replace('\'>', '')
line_fmtstr = '{name} ({typestr}): default={default}'
line = line_fmtstr.format(name=name, typestr=typestr, default=default)
kwargs_lines.append(line)
kwargs_docstr_block = ('Kwargs:\n' +
ut.indent('\n'.join(kwargs_lines), ' '))
return ut.indent(kwargs_docstr_block, ' ')
def print_tree_struct(*args, **kwargs):
tree_str = (ut.indent(ut.repr3(get_tree_info(*args, **kwargs), nl=1)))
print(tree_str)
#bytes_str = ut.byte_str2(drive.get_total_nbytes(dpath_to_unique_fidx[path]))
#print('path = %r, %s' % (path, bytes_str))
#print(ut.repr3(key_list))
return tree_str
def print_tree_struct(*args, **kwargs):
tree_str = (ut.indent(ut.repr3(get_tree_info(*args, **kwargs), nl=1)))
print(tree_str)
#bytes_str = ut.byte_str2(drive.get_total_nbytes(dpath_to_unique_fidx[path]))
#print('path = %r, %s' % (path, bytes_str))
#print(ut.repr3(key_list))
return tree_str
def markdown_cell(markdown):
r"""
Args:
markdown (str):
Returns:
str: json formatted ipython notebook markdown cell
CommandLine:
python -m ibeis.templates.generate_notebook --exec-markdown_cell
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.templates.generate_notebook import * # NOQA
>>> markdown = '# Title'
>>> result = markdown_cell(markdown)
>>> print(result)
"""
markdown_header = ut.codeblock(
"""
{
"cell_type": "markdown",
"metadata": {},
"source": [
"""
)
markdown_footer = ut.codeblock(
"""
]
}
"""
)
return (markdown_header + '\n' +
ut.indent(repr_single(markdown), ' ' * 2) +
'\n' + markdown_footer)
def dictinfo(dict_):
if not isinstance(dict_, dict):
return 'expected dict got %r' % type(dict_)
keys = list(dict_.keys())
vals = list(dict_.values())
num_keys = len(keys)
key_types = list(set(map(type, keys)))
val_types = list(set(map(type, vals)))
fmtstr_ = '\n' + ut.unindent('''
* num_keys = {num_keys}
* key_types = {key_types}
* val_types = {val_types}
'''.strip('\n'))
if len(val_types) == 1:
if val_types[0] == np.ndarray:
# each key holds an ndarray
val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0)
val_shape_stats_str = ut.dict_str(val_shape_stats,
strvals=True,
newlines=False)
val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* val_shape_stats = {val_shape_stats_str}
* val_dtypes = {val_dtypes}
'''.strip('\n'))
elif val_types[0] == list:
# each key holds a list
val_len_stats = ut.get_stats(set(map(len, vals)))
val_len_stats_str = ut.dict_str(val_len_stats,
strvals=True,
newlines=False)
depth = ut.list_depth(vals)
deep_val_types = list(set(ut.list_deep_types(vals)))
fmtstr_ += ut.unindent('''
* list_depth = {depth}
* val_len_stats = {val_len_stats_str}
* deep_types = {deep_val_types}
'''.strip('\n'))
if len(deep_val_types) == 1:
if deep_val_types[0] == np.ndarray:
deep_val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* deep_val_dtypes = {deep_val_dtypes}
''').strip('\n')
elif val_types[0] in [
np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32,
np.float64
]:
# each key holds a scalar
val_stats = ut.get_stats(vals)
fmtstr_ += ut.unindent('''
* val_stats = {val_stats}
''').strip('\n')
fmtstr = fmtstr_.format(**locals())
return ut.indent(fmtstr)
def parse_benchmarks(funcname, docstring, py_modname):
test_tuples_, setup_script_ = make_benchmarks(funcname, docstring,
py_modname)
if len(test_tuples_) == 0:
test_tuples = '[]'
else:
test_tuples = '[\n' + (' ' * 8) + '\n '.join(
list(map(str, test_tuples_))) + '\n ]' # NOQA
setup_script = utool.indent(setup_script_).strip() # NOQA
benchmark_name = utool.quasiquote('run_benchmark_{funcname}')
#test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''')
# http://en.wikipedia.org/wiki/Relative_change_and_difference
bench_code_fmt_ = r'''
def {benchmark_name}(iterations):
test_tuples = {test_tuples}
setup_script = textwrap.dedent("""
{setup_script}
""")
time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations)
time_pair = lambda (x, y): (time_line(x), time_line(y))
def print_timing_info(tup):
from math import log
test_lines = []
def test_print(str):
if not utool.QUIET:
print(str)
test_lines.append(str)
test_print('\n---------------')
test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations))
test_print('[bench] tests:')
test_print(' ' + str(tup))
(pyth_time, cyth_time) = time_pair(tup)
test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time))
test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time))
time_delta = cyth_time - pyth_time
#pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100
#pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100
pyth_per_cyth = (pyth_time / cyth_time) * 100
inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100
nepers = log(cyth_time / pyth_time)
if time_delta < 0:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,))
#test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,))
test_print('[bench.result] cython was faster by %f seconds' % -time_delta)
else:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,))
#test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers slower' % (nepers,))
test_print('[bench.result] python was faster by %f seconds' % time_delta)
pyth_call, cyth_call = tup
run_doctest(pyth_call, cyth_call, setup_script)
return (pyth_time, cyth_time, test_lines)
test_results = list(map(print_timing_info, test_tuples))
# results are lists of (time1, time2, strlist)
return test_results'''
bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n')
bench_code = utool.quasiquote(bench_code_fmt)
return (benchmark_name, bench_code)
def parse_benchmarks(funcname, docstring, py_modname):
test_tuples_, setup_script_ = make_benchmarks(funcname, docstring, py_modname)
if len(test_tuples_) == 0:
test_tuples = '[]'
else:
test_tuples = '[\n' + (' ' * 8) + '\n '.join(list(map(str, test_tuples_))) + '\n ]' # NOQA
setup_script = utool.indent(setup_script_).strip() # NOQA
benchmark_name = utool.quasiquote('run_benchmark_{funcname}')
#test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''')
# http://en.wikipedia.org/wiki/Relative_change_and_difference
bench_code_fmt_ = r'''
def {benchmark_name}(iterations):
test_tuples = {test_tuples}
setup_script = textwrap.dedent("""
{setup_script}
""")
time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations)
time_pair = lambda (x, y): (time_line(x), time_line(y))
def print_timing_info(tup):
from math import log
test_lines = []
def test_print(str):
if not utool.QUIET:
print(str)
test_lines.append(str)
test_print('\n---------------')
test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations))
test_print('[bench] tests:')
test_print(' ' + str(tup))
(pyth_time, cyth_time) = time_pair(tup)
test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time))
test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time))
time_delta = cyth_time - pyth_time
#pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100
#pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100
pyth_per_cyth = (pyth_time / cyth_time) * 100
inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100
nepers = log(cyth_time / pyth_time)
if time_delta < 0:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,))
#test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,))
test_print('[bench.result] cython was faster by %f seconds' % -time_delta)
else:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,))
#test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers slower' % (nepers,))
test_print('[bench.result] python was faster by %f seconds' % time_delta)
pyth_call, cyth_call = tup
run_doctest(pyth_call, cyth_call, setup_script)
return (pyth_time, cyth_time, test_lines)
test_results = list(map(print_timing_info, test_tuples))
# results are lists of (time1, time2, strlist)
return test_results'''
bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n')
bench_code = utool.quasiquote(bench_code_fmt)
return (benchmark_name, bench_code)
def convert_tests_from_ibeis_to_nose(module_list):
# PARSE OUT TESTABLE DOCTESTTUPS
#import utool as ut
testtup_list = []
seen_ = set()
topimport_list = []
for module in module_list:
mod_doctest_tup = ut.get_module_doctest_tup(module=module,
verbose=False,
allexamples=True)
enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup
flags = [tup.src not in seen_ for tup in enabled_testtup_list]
enabled_testtup_list = ut.compress(enabled_testtup_list, flags)
testtup_list.extend(enabled_testtup_list)
if len(enabled_testtup_list) > 0:
topimport_list.append('from %s import * # NOQA' % (module.__name__,))
print('Found %d test tups' % (len(testtup_list)))
autogen_test_src_funcs = []
#import redbaron
for testtup in testtup_list:
name = testtup.name
num = testtup.num
src = testtup.src
want = testtup.want
import re
src = re.sub('# ENABLE_DOCTEST\n', '', src)
src = re.sub('from [^*]* import \* *# NOQA\n', '', src)
src = re.sub(r'from [^*]* import \*\n', '', src)
src = ut.str_between(src, None, 'ut.quit_if_noshow').rstrip('\n')
src = ut.str_between(src, None, 'ut.show_if_requested').rstrip('\n')
# import utool
# utool.embed()
"""
"""
#flag = testtup.flag
if want.endswith('\n'):
want = want[:-1]
if want:
#src_node = redbaron.RedBaron(src)
#if len(src_node.find_all('name', 'result')) > 0:
# src_node.append('assert result == %r' % (want,))
if '\nresult = ' in src:
src += '\nassert str(result) == %r' % (want,)
func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src)
autogen_test_src_funcs.append(func_src)
autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n'
from ibeis import tests
from os.path import join
moddir = ut.get_module_dir(tests)
ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
def make_bench_text(benchmark_codes, benchmark_names, py_modname):
# TODO: let each function individually specify number
codes = '\n\n\n'.join(benchmark_codes) # NOQA
list_ = [utool.quasiquote('results.extend({benchfunc}(iterations))')
for benchfunc in benchmark_names]
all_benchmarks = utool.indent('\n'.join(list_), ' ' * 8).strip() # NOQA
timestamp = utool.get_timestamp() # NOQA
bench_text_fmt = get_bench_text_fmt()
bench_text = utool.quasiquote(bench_text_fmt)
return bench_text
def expand_closure_source(funcname, func):
source = ut.get_func_sourcecode(func)
closure_vars = [(k, v.cell_contents) for k, v in
zip(func.func_code.co_freevars, func.func_closure)]
source = ut.unindent(source)
import re
for k, v in closure_vars:
source = re.sub('\\b' + k + '\\b', ut.repr2(v), source)
source = re.sub(r'def .*\(self', 'def ' + funcname + '(self', source)
source = ut.indent(source.strip(), ' ') + '\n'
return source
def make_bench_text(benchmark_codes, benchmark_names, py_modname):
# TODO: let each function individually specify number
codes = '\n\n\n'.join(benchmark_codes) # NOQA
list_ = [
utool.quasiquote('results.extend({benchfunc}(iterations))')
for benchfunc in benchmark_names
]
all_benchmarks = utool.indent('\n'.join(list_), ' ' * 8).strip() # NOQA
timestamp = utool.get_timestamp() # NOQA
bench_text_fmt = get_bench_text_fmt()
bench_text = utool.quasiquote(bench_text_fmt)
return bench_text
def dictinfo(dict_):
if not isinstance(dict_, dict):
return 'expected dict got %r' % type(dict_)
keys = list(dict_.keys())
vals = list(dict_.values())
num_keys = len(keys)
key_types = list(set(map(type, keys)))
val_types = list(set(map(type, vals)))
fmtstr_ = '\n' + ut.unindent('''
* num_keys = {num_keys}
* key_types = {key_types}
* val_types = {val_types}
'''.strip('\n'))
if len(val_types) == 1:
if val_types[0] == np.ndarray:
# each key holds an ndarray
val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0)
val_shape_stats_str = ut.dict_str(val_shape_stats, strvals=True, newlines=False)
val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* val_shape_stats = {val_shape_stats_str}
* val_dtypes = {val_dtypes}
'''.strip('\n'))
elif val_types[0] == list:
# each key holds a list
val_len_stats = ut.get_stats(set(map(len, vals)))
val_len_stats_str = ut.dict_str(val_len_stats, strvals=True, newlines=False)
depth = ut.list_depth(vals)
deep_val_types = list(set(ut.list_deep_types(vals)))
fmtstr_ += ut.unindent('''
* list_depth = {depth}
* val_len_stats = {val_len_stats_str}
* deep_types = {deep_val_types}
'''.strip('\n'))
if len(deep_val_types) == 1:
if deep_val_types[0] == np.ndarray:
deep_val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* deep_val_dtypes = {deep_val_dtypes}
''').strip('\n')
elif val_types[0] in [np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32, np.float64]:
# each key holds a scalar
val_stats = ut.get_stats(vals)
fmtstr_ += ut.unindent('''
* val_stats = {val_stats}
''').strip('\n')
fmtstr = fmtstr_.format(**locals())
return ut.indent(fmtstr)
def check_cpp_build(rman):
import utool as ut
label = ' %s' % rman.label if rman.label else rman.label
missing = []
print('Checking if%s modules are built' % (label,))
for repo in rman.repos:
flag, msg = repo.check_cpp_build()
if not flag:
print(' * !!!%s REPO %s NEEDS TO BE BUILT' % (label.upper(), repo,))
if ut.VERBOSE:
print(ut.indent(msg, ' '))
missing.append(repo)
return missing
def check_cpp_build(rman):
import utool as ut
label = ' %s' % rman.label if rman.label else rman.label
missing = []
print('Checking if%s modules are built' % (label,))
for repo in rman.repos:
flag, msg = repo.check_cpp_build()
if not flag:
print(' * !!!%s REPO %s NEEDS TO BE BUILT' % (label.upper(), repo,))
if ut.VERBOSE:
print(ut.indent(msg, ' '))
missing.append(repo)
return missing
def get_inspect_str(qres, ibs=None, name_scoring=False):
qres.assert_self()
#ut.embed()
top_lbls = [' top aids', ' scores', ' rawscores', ' ranks']
top_aids = np.array(qres.get_top_aids(num=6, name_scoring=name_scoring, ibs=ibs), dtype=np.int32)
top_scores = np.array(qres.get_aid_scores(top_aids), dtype=np.float64)
top_rawscores = np.array(qres.get_aid_scores(top_aids, rawscore=True), dtype=np.float64)
top_ranks = np.array(qres.get_aid_ranks(top_aids), dtype=np.int32)
top_list = [top_aids, top_scores, top_rawscores, top_ranks]
if ibs is not None:
top_lbls += [' isgt']
istrue = qres.get_aid_truth(ibs, top_aids)
top_list.append(np.array(istrue, dtype=np.int32))
if name_scoring:
top_lbls = ['top nid'] + top_lbls
top_list = [ibs.get_annot_name_rowids(top_aids)] + top_list
top_stack = np.vstack(top_list)
#top_stack = np.array(top_stack, dtype=object)
top_stack = np.array(top_stack, dtype=np.float32)
#np.int32)
top_str = np.array_str(top_stack, precision=3, suppress_small=True, max_line_width=200)
top_lbl = '\n'.join(top_lbls)
inspect_list = ['QueryResult',
qres.cfgstr,
]
if ibs is not None:
gt_ranks = qres.get_gt_ranks(ibs=ibs)
gt_scores = qres.get_gt_scores(ibs=ibs)
inspect_list.append('gt_ranks = %r' % gt_ranks)
inspect_list.append('gt_scores = %r' % gt_scores)
nFeatMatch_list = get_num_feats_in_matches(qres)
nFeatMatch_stats_str = ut.get_stats_str(nFeatMatch_list, newlines=True, exclude_keys=('nMin', 'nMax'))
inspect_list.extend([
'qaid=%r ' % qres.qaid,
ut.hz_str(top_lbl, ' ', top_str),
'num feat matches per annotation stats:',
#ut.indent(ut.dict_str(nFeatMatch_stats)),
ut.indent(nFeatMatch_stats_str),
])
inspect_str = '\n'.join(inspect_list)
#inspect_str = ut.indent(inspect_str, '[INSPECT] ')
return inspect_str
def get_inspect_str(qres, ibs=None, name_scoring=False):
qres.assert_self()
#ut.embed()
top_lbls = [' top aids', ' scores', ' rawscores', ' ranks']
top_aids = np.array(qres.get_top_aids(num=6, name_scoring=name_scoring, ibs=ibs), dtype=np.int32)
top_scores = np.array(qres.get_aid_scores(top_aids), dtype=np.float64)
top_rawscores = np.array(qres.get_aid_scores(top_aids, rawscore=True), dtype=np.float64)
top_ranks = np.array(qres.get_aid_ranks(top_aids), dtype=np.int32)
top_list = [top_aids, top_scores, top_rawscores, top_ranks]
if ibs is not None:
top_lbls += [' isgt']
istrue = qres.get_aid_truth(ibs, top_aids)
top_list.append(np.array(istrue, dtype=np.int32))
if name_scoring:
top_lbls = ['top nid'] + top_lbls
top_list = [ibs.get_annot_name_rowids(top_aids)] + top_list
top_stack = np.vstack(top_list)
#top_stack = np.array(top_stack, dtype=object)
top_stack = np.array(top_stack, dtype=np.float32)
#np.int32)
top_str = np.array_str(top_stack, precision=3, suppress_small=True, max_line_width=200)
top_lbl = '\n'.join(top_lbls)
inspect_list = ['QueryResult',
qres.cfgstr,
]
if ibs is not None:
gt_ranks = qres.get_gt_ranks(ibs=ibs)
gt_scores = qres.get_gt_scores(ibs=ibs)
inspect_list.append('gt_ranks = %r' % gt_ranks)
inspect_list.append('gt_scores = %r' % gt_scores)
nFeatMatch_list = get_num_feats_in_matches(qres)
nFeatMatch_stats_str = ut.get_stats_str(nFeatMatch_list, newlines=True, exclude_keys=('nMin', 'nMax'))
inspect_list.extend([
'qaid=%r ' % qres.qaid,
ut.hz_str(top_lbl, ' ', top_str),
'num feat matches per annotation stats:',
#ut.indent(ut.dict_str(nFeatMatch_stats)),
ut.indent(nFeatMatch_stats_str),
])
inspect_str = '\n'.join(inspect_list)
#inspect_str = ut.indent(inspect_str, '[INSPECT] ')
return inspect_str
def make_notebook(cell_list):
"""
References:
# Change cell width
http://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-ipython-notebook-in-my-browser/24207353#24207353
"""
import utool as ut
header = ut.codeblock(
'''
{
"cells": [
'''
)
footer = ut.codeblock(
'''
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
''')
cell_body = ut.indent(',\n'.join(cell_list), ' ')
notebook_str = header + '\n' + cell_body + '\n' + footer
try:
import json
json.loads(notebook_str)
except ValueError as ex:
ut.printex(ex, 'Invalid notebook JSON')
raise
return notebook_str
def convert_tests_from_utool_to_nose(module_list):
# PARSE OUT TESTABLE DOCTESTTUPS
#import utool as ut
testtup_list = []
seen_ = set()
topimport_list = []
for module in module_list:
mod_doctest_tup = ut.get_module_doctest_tup(module=module,
verbose=False,
allexamples=True)
enabled_testtup_list, frame_fpath, all_testflags, module = mod_doctest_tup
flags = [tup.src not in seen_ for tup in enabled_testtup_list]
enabled_testtup_list = ut.compress(enabled_testtup_list, flags)
testtup_list.extend(enabled_testtup_list)
if len(enabled_testtup_list) > 0:
topimport_list.append('from %s import * # NOQA' % (module.__name__,))
print('Found %d test tups' % (len(testtup_list)))
autogen_test_src_funcs = []
#import redbaron
for testtup in testtup_list:
name = testtup.name
num = testtup.num
src = testtup.src
want = testtup.want
import re
src = re.sub('# ENABLE_DOCTEST\n', '', src)
src = re.sub('from [^*]* import \* *# NOQA\n', '', src)
src = re.sub('from [^*]* import \*\n', '', src)
#flag = testtup.flag
if want.endswith('\n'):
want = want[:-1]
if want:
#src_node = redbaron.RedBaron(src)
#if len(src_node.find_all('name', 'result')) > 0:
# src_node.append('assert result == %r' % (want,))
if '\nresult = ' in src:
src += '\nassert str(result) == %r' % (want,)
func_src = 'def test_%s_%d():\n' % (name.replace('.', '_'), num,) + ut.indent(src)
autogen_test_src_funcs.append(func_src)
autogen_test_src = '\n'.join(topimport_list) + '\n\n\n' + '\n\n\n'.join(autogen_test_src_funcs) + '\n'
from utool import tests
from os.path import join
moddir = ut.get_module_dir(tests)
ut.writeto(join(moddir, 'test_autogen_nose_tests.py'), autogen_test_src)
def _build_typed_params_kwargs_docstr_block(typed_params):
r"""
Args:
typed_params (dict):
CommandLine:
python -m pyhesaff build_typed_params_docstr
Example:
>>> # DISABLE_DOCTEST
>>> from pyhesaff._pyhesaff import * # NOQA
>>> typed_params = HESAFF_TYPED_PARAMS
>>> result = build_typed_params_docstr(typed_params)
>>> print(result)
"""
kwargs_lines = []
for tup in typed_params:
type_, name, default = tup
typestr = str(type_).replace('<class \'ctypes.c_', '').replace('\'>', '')
line_fmtstr = '{name} ({typestr}): default={default}'
line = line_fmtstr.format(name=name, typestr=typestr, default=default)
kwargs_lines.append(line)
kwargs_docstr_block = ('Kwargs:\n' + ut.indent('\n'.join(kwargs_lines), ' '))
return ut.indent(kwargs_docstr_block, ' ')
def as_table(self, caption=None):
if caption is None:
caption = self.caption
tabular = self.as_tabular()
table = ut.codeblock(r"""
\begin{{table}}[h]
\centering
\caption{{{caption}}}
""").format(caption=caption)
if tabular:
table += '\n' + ut.indent(tabular)
table += ('\n' + ut.codeblock("""
\\end{{table}}
""").format())
return table
def check_importable(rman):
import utool as ut
label = ' %s' % rman.label if rman.label else rman.label
missing = []
print('Checking if%s modules are importable' % (label,))
msg_list = []
recommended_fixes = []
for repo in rman.repos:
flag, msg, errors = repo.check_importable()
if not flag:
msg_list.append(' * !!!%s REPO %s HAS IMPORT ISSUES' % (label.upper(), repo,))
if any([str(ex).find('undefined symbol') > -1 for ex in errors]):
recommended_fixes.append('rebuild')
else:
recommended_fixes.append(None)
if ut.VERBOSE:
msg_list.append(ut.indent(msg, ' '))
missing.append(repo)
else:
if ut.VERBOSE:
msg_list.append(ut.indent(msg, ' '))
print('\n'.join(msg_list))
problems = list(zip(missing, recommended_fixes))
return problems
def code_cell(sourcecode):
r"""
Args:
sourcecode (str):
Returns:
str: json formatted ipython notebook code cell
CommandLine:
python -m ibeis.templates.generate_notebook --exec-code_cell
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.templates.generate_notebook import * # NOQA
>>> sourcecode = notebook_cells.timestamp_distribution[1]
>>> sourcecode = notebook_cells.initialize[1]
>>> result = code_cell(sourcecode)
>>> print(result)
"""
import utool as ut
sourcecode = ut.remove_codeblock_syntax_sentinals(sourcecode)
cell_header = ut.codeblock(
'''
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source":
''')
cell_footer = ut.codeblock(
'''
}
''')
if sourcecode is None:
source_line_repr = ' []\n'
else:
lines = sourcecode.split('\n')
line_list = [line + '\n' if count < len(lines) else line
for count, line in enumerate(lines, start=1)]
#repr_line_list = [repr_single_for_md(line) for line in line_list]
repr_line_list = [repr_single_for_md(line) for line in line_list]
source_line_repr = ut.indent(',\n'.join(repr_line_list), ' ' * 2)
source_line_repr = ' [\n' + source_line_repr + '\n ]\n'
return (cell_header + source_line_repr + cell_footer)
def check_installed(rman):
import utool as ut
label = ' %s' % rman.label if rman.label else rman.label
missing = []
msg_list = []
print('Checking if%s modules are installed' % (label,))
for repo in rman.repos:
flag, msg = repo.check_installed()
if not flag:
msg_list.append(' * !!!%s REPO %s NEEDS TO BE INSTALLED' % (label.upper(), repo,))
if ut.VERBOSE:
msg_list.append(ut.indent(msg, ' '))
missing.append(repo)
# else:
# print(' * found%s module = %s' % (label, repo,))
print('\n'.join(msg_list))
return missing
def check_installed(rman):
import utool as ut
label = ' %s' % rman.label if rman.label else rman.label
missing = []
msg_list = []
print('Checking if%s modules are installed' % (label,))
for repo in rman.repos:
flag, msg = repo.check_installed()
if not flag:
msg_list.append(' * !!!%s REPO %s NEEDS TO BE INSTALLED' % (label.upper(), repo,))
if ut.VERBOSE:
msg_list.append(ut.indent(msg, ' '))
missing.append(repo)
# else:
# print(' * found%s module = %s' % (label, repo,))
print('\n'.join(msg_list))
return missing
def test_body(count, logmode, backspace):
ut.colorprint('\n---- count = %r -----' % (count,), 'yellow')
ut.colorprint('backspace = %r' % (backspace,), 'yellow')
ut.colorprint('logmode = %r' % (logmode,), 'yellow')
if logmode:
ut.delete('test.log')
ut.start_logging('test.log')
print('Start main loop')
import time
for count in ut.ProgressIter(range(20), freq=3, backspace=backspace):
time.sleep(.01)
print('Done with main loop work')
print('Exiting main body')
if logmode:
ut.stop_logging()
#print('-----DONE LOGGING----')
testlog_text = ut.readfrom('test.log')
print(ut.indent(testlog_text.replace('\r', '\n'), ' '))
def test_body(count, logmode, backspace):
ut.colorprint('\n---- count = %r -----' % (count, ), 'yellow')
ut.colorprint('backspace = %r' % (backspace, ), 'yellow')
ut.colorprint('logmode = %r' % (logmode, ), 'yellow')
if logmode:
ut.delete('test.log')
ut.start_logging('test.log')
print('Start main loop')
import time
for count in ut.ProgressIter(range(20), freq=3, backspace=backspace):
time.sleep(.01)
print('Done with main loop work')
print('Exiting main body')
if logmode:
ut.stop_logging()
#print('-----DONE LOGGING----')
testlog_text = ut.readfrom('test.log')
print(ut.indent(testlog_text.replace('\r', '\n'), ' '))
def __str__(allres):
#print = tores.append
ibs = allres.ibs
toret = ('+======================\n')
scalar_summary = str(allres.scalar_summary).strip()
toret += ('| All Results: %s \n' % ibs.get_db_name())
toret += ('| title_suffix=%s\n' % str(allres.title_suffix))
toret += ('| scalar_summary=\n%s\n' % utool.indent(scalar_summary, '| '))
toret += ('| ' + str(allres.scalar_mAP_str))
toret += ('|---\n')
toret += ('| greater5_%s \n' % (ibs.cidstr(allres.greater5_rids),))
toret += ('|---\n')
toret += ('| greater1_%s \n' % (ibs.cidstr(allres.greater1_rids),))
toret += ('|---\n')
toret += ('+======================.\n')
#toret+=('| problem_false_pairs=\n%r' % allres.problem_false_pairs)
#toret+=('| problem_true_pairs=\n%r' % allres.problem_true_pairs)
return toret
def infer_monitor_specs(res_w, res_h, inches_diag):
"""
monitors = [
dict(name='work1', inches_diag=23, res_w=1920, res_h=1080),
dict(name='work2', inches_diag=24, res_w=1920, res_h=1200),
dict(name='hp-129', inches_diag=25, res_w=1920, res_h=1080),
dict(name='?-26', inches_diag=26, res_w=1920, res_h=1080),
dict(name='?-27', inches_diag=27, res_w=1920, res_h=1080),
]
for info in monitors:
name = info['name']
inches_diag = info['inches_diag']
res_h = info['res_h']
res_w = info['res_w']
print('---')
print(name)
inches_w = inches_diag * res_w / np.sqrt(res_h**2 + res_w**2)
inches_h = inches_diag * res_h / np.sqrt(res_h**2 + res_w**2)
print('inches diag = %.2f' % (inches_diag))
print('inches WxH = %.2f x %.2f' % (inches_w, inches_h))
#inches_w = inches_diag * res_w/sqrt(res_h**2 + res_w**2)
"""
import sympy
# Build a system of equations and solve it
inches_w, inches_h = sympy.symbols(
'inches_w inches_h'.split(), real=True, positive=True
)
res_w, res_h = sympy.symbols('res_w res_h'.split(), real=True, positive=True)
(inches_diag,) = sympy.symbols('inches_diag'.split(), real=True, positive=True)
equations = [
sympy.Eq(inches_diag, (inches_w ** 2 + inches_h ** 2) ** 0.5),
sympy.Eq(res_w / res_h, inches_w / inches_h),
]
print('Possible solutions:')
query_vars = [inches_w, inches_h]
for solution in sympy.solve(equations, query_vars):
print('Solution:')
reprstr = ut.repr3(
ut.odict(zip(query_vars, solution)), explicit=True, nobr=1, with_comma=False
)
print(ut.indent(ut.autopep8_format(reprstr)))
def make_notebook(cell_list):
"""
References:
# Change cell width
http://stackoverflow.com/questions/21971449/how-do-i-increase-the-cell-width-of-the-ipython-notebook-in-my-browser/24207353#24207353
"""
header = ut.codeblock(
"""
{
"cells": [
"""
)
footer = ut.codeblock(
"""
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
""")
cell_body = ut.indent(',\n'.join(cell_list), ' ')
notebook_str = header + '\n' + cell_body + '\n' + footer
return notebook_str
def __str__(allres):
#print = tores.append
ibs = allres.ibs
toret = ('+======================\n')
scalar_summary = str(allres.scalar_summary).strip()
toret += ('| All Results: %s \n' % ibs.get_db_name())
toret += ('| title_suffix=%s\n' % str(allres.title_suffix))
toret += ('| scalar_summary=\n%s\n' %
utool.indent(scalar_summary, '| '))
toret += ('| ' + str(allres.scalar_mAP_str))
toret += ('|---\n')
toret += ('| greater5_%s \n' % (ibs.cidstr(allres.greater5_rids), ))
toret += ('|---\n')
toret += ('| greater1_%s \n' % (ibs.cidstr(allres.greater1_rids), ))
toret += ('|---\n')
toret += ('+======================.\n')
#toret+=('| problem_false_pairs=\n%r' % allres.problem_false_pairs)
#toret+=('| problem_true_pairs=\n%r' % allres.problem_true_pairs)
return toret
def handle_endmacro(matcher):
""" this is quite a bit hacky, but this is the
most straightforward way to implement them until the
parse_cyth_preproc_markup/visit_FunctionDef 'coroutine'
blob is refactored """
(macro_name,) = suspended_macro_context_ptr[0]
lines = macro_input_buffer_ptr[0]
#print('macro invokation of "%s" on lines %r' % (macro_name, lines))
expander = MACRO_EXPANDERS_DICT.get(macro_name, None)
if expander:
expanded_lines = ['\n'] + expander(self.gensym, lines) + ['\n']
else:
errmsg = 'No macro named %r has been registered via the cyth.macro decorator'
raise NotImplementedError(errmsg % macro_name)
indented_expanded_lines = [utool.indent(x, indent=self.indent_with * (self.indentation + 1))
for x in expanded_lines]
#print('output is %r' % expanded_lines)
source_lines.extend(indented_expanded_lines)
collect_macro_input_ptr[0] = False
macro_input_buffer_ptr[0] = []
def handle_endmacro(matcher):
""" this is quite a bit hacky, but this is the
most straightforward way to implement them until the
parse_cyth_preproc_markup/visit_FunctionDef 'coroutine'
blob is refactored """
(macro_name, ) = suspended_macro_context_ptr[0]
lines = macro_input_buffer_ptr[0]
#print('macro invokation of "%s" on lines %r' % (macro_name, lines))
expander = MACRO_EXPANDERS_DICT.get(macro_name, None)
if expander:
expanded_lines = ['\n'] + expander(self.gensym, lines) + ['\n']
else:
errmsg = 'No macro named %r has been registered via the cyth.macro decorator'
raise NotImplementedError(errmsg % macro_name)
indented_expanded_lines = [
utool.indent(x,
indent=self.indent_with * (self.indentation + 1))
for x in expanded_lines
]
#print('output is %r' % expanded_lines)
source_lines.extend(indented_expanded_lines)
collect_macro_input_ptr[0] = False
macro_input_buffer_ptr[0] = []
def markdown_cell(markdown):
r"""
Args:
markdown (str):
Returns:
str: json formatted ipython notebook markdown cell
CommandLine:
python -m ibeis.templates.generate_notebook --exec-markdown_cell
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.templates.generate_notebook import * # NOQA
>>> markdown = '# Title'
>>> result = markdown_cell(markdown)
>>> print(result)
"""
import utool as ut
markdown_header = ut.codeblock(
'''
{
"cell_type": "markdown",
"metadata": {},
"source": [
'''
)
markdown_footer = ut.codeblock(
'''
]
}
'''
)
return (markdown_header + '\n' +
ut.indent(repr_single_for_md(markdown), ' ' * 2) +
'\n' + markdown_footer)
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None,
repodir=None, exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(
r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(
r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(
r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "',
'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")']).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(header_lower=header_lower,
testflag=testflag,)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten([ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any([x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list]
testline_list = [format_testline(fpath, dirvar) for fpath in testfpath_list]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,)
header_test_block = header_test_block_fmstr.format(**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs, dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
def make_default_module_maintest(modname, modpath=None):
"""
make_default_module_maintest
TODO: use path relative to home dir if the file is a script
Args:
modname (str): module name
Returns:
str: text source code
CommandLine:
python -m utool.util_autogen --test-make_default_module_maintest
References:
http://legacy.python.org/dev/peps/pep-0338/
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_autogen import * # NOQA
>>> modname = 'utool.util_autogen'
>>> text = make_default_module_maintest(modname)
>>> result = str(text)
>>> print(result)
"""
import utool as ut
# Need to use python -m to run a module
# otherwise their could be odd platform specific errors.
#python -c "import utool, {modname};
# ut.doctest_funcs({modname}, allexamples=True)"
#in_pythonpath, module_type, path = find_modname_in_pythonpath(modname)
# only use the -m if it is part of a package directory
if modpath is not None:
moddir = dirname(modpath)
pkginit_fpath = join(moddir, '__init__.py')
use_modrun = exists(pkginit_fpath)
else:
use_modrun = True
if use_modrun:
pyargs = '-m ' + modname
else:
if ut.WIN32:
modpath = normpath(modpath).replace(expanduser('~'), '%HOME%')
pyargs = '-B ' + ut.ensure_unixslash(modpath)
else:
modpath = normpath(modpath).replace(expanduser('~'), '~')
pyargs = modpath
cmdline = ut.codeblock(
'''
python {pyargs}
python {pyargs} --allexamples
# REM python {pyargs} --allexamples --noface --nosrc
''')
if not use_modrun:
if ut.WIN32:
augpath = 'set PYTHONPATH=%PYTHONPATH%' + os.pathsep + moddir
else:
augpath = 'export PYTHONPATH=$PYTHONPATH' + os.pathsep + moddir
cmdline = augpath + '\n' + cmdline
cmdline = ut.indent(cmdline, ' ' * 8).lstrip(' ').format(pyargs=pyargs)
text = ut.codeblock(
r'''
# STARTBLOCK
if __name__ == '__main__':
r"""
CommandLine:
{cmdline}
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
import utool as ut # NOQA
ut.doctest_funcs()
# ENDBLOCK
'''
).format(cmdline=cmdline)
text = remove_codeblock_syntax_sentinals(text)
return text
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = ['ibeis/tests/assert_modules.py']
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/ibeis'
testdir = 'ibeis/tests'
exclude_list = []
# Hacky, but not too bad way of getting in doctests
# Test to see if doctest_funcs appears after main
# Do not doctest these modules
#exclude_doctests_fnames = set(['template_definitions.py',
# 'autogen_test_script.py'])
#exclude_dirs = [
# '_broken',
# 'old',
# 'tests',
# 'timeits',
# '_scripts',
# '_timeits',
# '_doc',
# 'notebook',
#]
#dpath_list = ['ibeis']
#doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
# The implict list is exactly the code we will use to make the implicit list
#module_list = None
#doctest_modname_list = None
#'export_subset.py',
#'_autogen_ibeiscontrol_funcs.py',
#'randomforest.py',
implicit_build_modlist_str = ut.codeblock('''
import sys
exclude_doctests_fnames = set([
'template_definitions.py',
'autogen_test_script.py',
])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['ibeis']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
''')
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
#module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list]
#for modname in doctest_modname_list:
# exec('import ' + modname, globals(), locals())
#module_list = [sys.modules[name] for name in doctest_modname_list]
#print('\n'.join(testcmds))
#print('\n'.join(['python -m ' + modname for modname in doctest_modname_list]))
import_str = '\n'.join(
['import ' + modname for modname in doctest_modname_list])
modlist_str = (
'module_list = [%s\n]' %
ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
''')
pyscript_text = pyscript_fmtstr.format(
build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds_ = ut.get_module_testlines(module_list,
remove_pyc=True,
verbose=False,
pythoncmd='RUN_TEST',
testslow=True)
testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_]
test_headers = [
# title, default, module, testpattern
ut.def_test('VTOOL',
dpath='vtool/tests',
pat=['test*.py'],
modname='vtool'),
ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']),
ut.def_test('IBEIS',
dpath=testdir,
pat=['test_ibs*.py', 'test_delete*.py'],
default=False),
ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']),
ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']),
ut.def_test('MISC', dpath=testdir, pat=misc_pats),
ut.def_test('OTHER', dpath=testdir, pat='OTHER'),
ut.def_test('HESAFF',
dpath='pyhesaff/tests',
pat=['test_*.py'],
modname='pyhesaff'),
ut.def_test('DOC', testcmds=testcmds, default=True)
]
# Referencs: https://docs.python.org/2/library/runpy.html
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs,
quick_tests, repodir,
exclude_list)
#print(pyscript_text)
return shscript_text, pyscript_text
def print_layer_info(output_layer):
str_ = get_layer_info_str(output_layer)
str_ = ut.indent('[info] ' + str_)
print('\n' + str_)
def autogen_parts(binding_name=None):
r"""
CommandLine:
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=set_dataset
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=add_points
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_point --py
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=used_memory --py
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_points --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=veclen --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=size --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=clean_removed_points --py --c
Ignore:
# Logic goes here
~/code/flann/src/cpp/flann/algorithms/kdtree_index.h
~/code/flann/src/cpp/flann/util/serialization.h
~/code/flann/src/cpp/flann/util/dynamic_bitset.h
# Bindings go here
~/code/flann/src/cpp/flann/flann.cpp
~/code/flann/src/cpp/flann/flann.h
# Contains stuff for the flann namespace like flann::log_level
# Also has Index with
# Matrix<ElementType> features; SEEMS USEFUL
~/code/flann/src/cpp/flann/flann.hpp
# Wrappers go here
~/code/flann/src/python/pyflann/flann_ctypes.py
~/code/flann/src/python/pyflann/index.py
~/local/build_scripts/flannscripts/autogen_bindings.py
Example:
>>> # ENABLE_DOCTEST
>>> from autogen_bindings import * # NOQA
>>> result = autogen_parts()
>>> print(result)
"""
#ut.get_dynlib_exports(pyflann.flannlib._name)
#flannlib
if binding_name is None:
binding_name = ut.get_argval('--bindingname', type_=str, default='add_points')
# Variable names used in flann cpp source
simple_c_to_ctypes = {
'void' : None,
'char*' : 'c_char_p',
'unsigned int' : 'c_uint',
'int' : 'c_int',
'float' : 'c_float',
'float*' : 'POINTER(c_float)',
'flann_index_t' : 'FLANN_INDEX',
'FLANNParameters*' : 'POINTER(FLANNParameters)',
'Distance::ResultType*' : "ndpointer(%(restype)s, flags='aligned, c_contiguous, writeable')",
'Distance::ElementType*' : "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')",
'typename Distance::ElementType*' : "ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')",
'typename Distance::ResultType*' : "ndpointer(%(restype), ndim=2, flags='aligned, c_contiguous, writeable')",
}
templated_ctype_map = {
'filename' : 'char*',
'level' : 'int',
'rows' : 'int',
'cols' : 'int',
'point_id' : 'unsigned int',
'num' : 'int',
'max_nn' : 'int',
'tcount' : 'int',
'nn' : 'int',
'radius' : 'float',
'clusters' : 'int',
'rebuild_threshold' : 'float',
'index_ptr' : 'flann_index_t',
'flann_params' : 'FLANNParameters*',
'speedup' : 'float*',
'id_list' : 'int*',
#'indices' : 'int*',
'dataset' : 'typename Distance::ElementType*',
'points' : 'typename Distance::ElementType*',
'query' : 'typename Distance::ElementType*',
'query1d' : 'typename Distance::ElementType*',
'testset' : 'typename Distance::ElementType*',
'dists' : 'typename Distance::ResultType*',
'dists1d' : 'typename Distance::ResultType*',
'result_centers' : 'typename Distance::ResultType*',
'result_ids' : 'int*',
}
# Python ctype bindings
python_ctype_map = {
'flann_params' : 'POINTER(FLANNParameters)',
'id_list' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous')",
#'indices' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous, writeable')",
'query1d' : "ndpointer(%(numpy)s, ndim=1, flags='aligned, c_contiguous')",
'dists1d' : "ndpointer(%(restype), ndim=1, flags='aligned, c_contiguous, writeable')",
'result_ids' : "ndpointer(int32, ndim=2, flags='aligned, c_contiguous, writeable')",
#'query' : "ndpointer(float64, ndim=1, flags='aligned, c_contiguous')",
}
for key, val in templated_ctype_map.items():
if key not in python_ctype_map:
python_ctype_map[key] = simple_c_to_ctypes[val]
binding_def = define_flann_bindings(binding_name)
docstr_cpp = binding_def['docstr_cpp']
docstr_py = binding_def['docstr_py']
cpp_binding_name = binding_def['cpp_binding_name']
return_type = binding_def['return_type']
binding_argnames = binding_def['binding_argnames']
c_source = binding_def['c_source']
py_source = binding_def['py_source']
optional_args = binding_def['optional_args']
py_alias = binding_def['py_alias']
py_args = binding_def['py_args']
binding_args = [python_ctype_map[name] + ', # ' + name for name in binding_argnames]
binding_args_str = ' ' + '\n '.join(binding_args)
callargs = ', '.join(binding_argnames)
pycallargs = ', '.join([name for name in ['self'] + binding_argnames if name != 'index_ptr'])
if py_args is None:
py_args = binding_argnames
pyinputargs = pycallargs # FIXME
else:
pyinputargs = ', '.join(['self'] + py_args)
pyrestype = simple_c_to_ctypes[return_type]
if py_alias is None:
py_binding_name = binding_name
else:
py_binding_name = py_alias
binding_def['py_binding_name'] = py_binding_name
#### flann_ctypes.py
flann_ctypes_codeblock = ut.codeblock(
'''
flann.{binding_name} = {{}}
define_functions(r"""
flannlib.flann_{binding_name}_%(C)s.restype = {pyrestype}
flannlib.flann_{binding_name}_%(C)s.argtypes = [
{binding_args_str}
]
flann.{binding_name}[%(numpy)s] = flannlib.flann_{binding_name}_%(C)s
""")
'''
).format(binding_name=binding_name, binding_args_str=binding_args_str, pyrestype=pyrestype)
#### index.py
default_py_source_parts = []
if 'pts' in py_args:
default_py_source_parts.append(ut.codeblock(
'''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
'''))
default_py_source_parts.append(ut.codeblock(
'''
rows = pts.shape[0]
raise NotImplementedError('requires custom implementation')
flann.{binding_name}[self.__curindex_type](self.__curindex, {pycallargs})
'''
))
if py_source is None:
py_source = '\n'.join(default_py_source_parts)
flann_index_codeblock = ut.codeblock(
r'''
def {py_binding_name}({pyinputargs}):
"""''' + ut.indent('\n' + docstr_py, ' ') + '''
"""''' + '\n' + ut.indent(py_source, ' ') + '''
'''
).format(binding_name=binding_name, pycallargs=pycallargs,
py_binding_name=py_binding_name,
pyinputargs=pyinputargs, py_source=py_source)
#### flann.cpp
#// {binding_name} BEGIN CPP BINDING
#template <typename Distance>
#{return_type} __flann_{binding_name}({templated_args})
flann_cpp_code_fmtstr_ = ut.codeblock(
r'''
{{''' + '\n' + ut.indent(c_source, ' ' * (4 * 3)) + r'''
}}
'''
)
#implicit_type_bindings_fmtstr = ut.codeblock( # NOQA
# '''
# DISTANCE_TYPE_BINDINGS({return_type}, {binding_name},
# SINGLE_ARG({T_typed_sigargs_cpp}),
# SINGLE_ARG({callargs}))
# '''
#)
#type_bindings_fmtstr = implicit_type_bindings_fmtstr
explicit_type_bindings_part3_fmtstr = ut.codeblock(
r'''
{{
if (flann_distance_type==FLANN_DIST_EUCLIDEAN) {{
return __flann_{binding_name}<L2<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_MANHATTAN) {{
return __flann_{binding_name}<L1<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_MINKOWSKI) {{
return __flann_{binding_name}<MinkowskiDistance<T> >({callargs}{minkowski_option});
}}
else if (flann_distance_type==FLANN_DIST_HIST_INTERSECT) {{
return __flann_{binding_name}<HistIntersectionDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_HELLINGER) {{
return __flann_{binding_name}<HellingerDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_CHI_SQUARE) {{
return __flann_{binding_name}<ChiSquareDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_KULLBACK_LEIBLER) {{
return __flann_{binding_name}<KL_Divergence<T> >({callargs});
}}
else {{
Logger::error( "Distance type unsupported in the C bindings, use the C++ bindings instead\n");
{errorhandle}
}}
}}
'''
)
# c binding body
c_bodyblock_fmtstr = ut.codeblock(
'''
{return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2})
{{
{ifreturns}_flann_{binding_name}{iftemplate}({callargs});
}}
'''
)
#### flann.h
# c binding header
#c_headersig_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2});'
c_headersig_part1_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}('
#### format cpp parts
binding_argtypes = ut.dict_take(templated_ctype_map, binding_argnames)
_fix_T = {
'typename Distance::ElementType*': 'T*',
'typename Distance::ResultType*': 'R*',
}
templated_ctype_map_cpp = {name: _fix_T.get(type_, type_) for name, type_ in templated_ctype_map.items()}
binding_argtypes_cpp = ut.dict_take(templated_ctype_map_cpp, binding_argnames)
binding_sigargs_cpp = [type_ + ' ' + name
for type_, name in zip(binding_argtypes_cpp, binding_argnames)]
templated_bindings = [templated_ctype_map[name] + ' ' + name for name in binding_argnames]
if optional_args is not None:
templated_bindings += optional_args
minkowski_option = ', MinkowskiDistance<T>(flann_distance_order)'
else:
minkowski_option = ''
templated_args = ', '.join(templated_bindings)
if binding_name == 'remove_point':
# HACK
templated_args += '_uint'
cpp_sig_part1 = '{return_type} __flann_{binding_name}('.format(return_type=return_type, binding_name=binding_name)
maxlen = 100
cpp_sig_ = ut.packstr(cpp_sig_part1 + templated_args, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(cpp_sig_part1))
cpp_sig = cpp_sig_[:-2] + ')'
flann_cpp_code_fmtstr = 'template <typename Distance>\n' + cpp_sig + '\n' + flann_cpp_code_fmtstr_
#print(cpp_sig)
T_typed_sigargs_cpp = ', '.join(binding_sigargs_cpp)
used_template_list = []
used_template_list.append('typename T')
if 'typename Distance::ResultType*' in binding_argtypes:
used_template_list.append('typename R')
used_templates = ', '.join(used_template_list)
type_binding_part1 = 'template <{used_templates}>'.format(used_templates=used_templates)
type_binding_part2_ = '{return_type} _flann_{binding_name}('.format(return_type=return_type, binding_name=binding_name)
maxlen = 100
cpp_type_sig_ = ut.packstr(type_binding_part2_ + T_typed_sigargs_cpp, textwidth=maxlen, breakchars=', ', wordsep=', ', break_words=False, newline_prefix=' ' * len(type_binding_part2_))
cpp_type_sig = cpp_type_sig_[:-2] + ')'
type_binding_part12 = type_binding_part1 + '\n' + cpp_type_sig
explicit_type_bindings_fmtstr = type_binding_part12 + '\n' + explicit_type_bindings_part3_fmtstr
flann_cpp_codeblock_fmtstr = flann_cpp_code_fmtstr + '\n\n\n' + explicit_type_bindings_fmtstr + '\n'
if return_type == 'int':
errorhandle = 'return -1;'
elif return_type == 'flann_index_t':
errorhandle = 'return NULL;'
else:
errorhandle = 'throw 0;'
# print('------')
# print('flann_cpp_codeblock_fmtstr.format = %s' % (flann_cpp_codeblock_fmtstr,))
try:
flann_cpp_codeblock = flann_cpp_codeblock_fmtstr.format(
cpp_binding_name=cpp_binding_name,
minkowski_option=minkowski_option,
binding_name=binding_name,
#templated_args=templated_args,
callargs=callargs,
#T_typed_sigargs_cpp=T_typed_sigargs_cpp,
errorhandle=errorhandle,
used_templates=used_templates, return_type=return_type)
except KeyError as ex:
ut.printex(ex, keys=['binding_name'])
raise
dataset_types = [
'',
'float',
'double',
'byte',
'int',
]
#### format c parts
c_header_sigs = []
c_body_blocks = []
templated_ctype_map_c = templated_ctype_map.copy()
#templated_ctype_map_c['dataset'] = 'float'
_fix_explicit_ctype = {
'' : 'float',
'byte' : 'unsigned char',
}
# For each explicit c type
for dataset_type in dataset_types:
T_type = _fix_explicit_ctype.get(dataset_type, dataset_type)
if dataset_type != '':
signame_type = '_' + dataset_type
else:
signame_type = dataset_type
R_type = 'float' if T_type != 'double' else 'double'
dstype = T_type + '*'
rstype = R_type + '*'
# Overwrite template types with explicit c types
needstemplate = True
for type_, name in zip(binding_argtypes, binding_argnames):
if type_ == 'typename Distance::ElementType*':
templated_ctype_map_c[name] = dstype
needstemplate = False
if type_ == 'typename Distance::ResultType*':
templated_ctype_map_c[name] = rstype
needstemplate = False
if type_ == 'Distance::ResultType*':
templated_ctype_map_c[name] = rstype
needstemplate = False
#if type_ == 'struct FLANNParameters*':
# # hack
# templated_ctype_map_c[name] = 'FLANNParameters*'
# HACK
if binding_name == 'load_index' or binding_name == 'add_points':
needstemplate = True
if binding_name == 'build_index':
needstemplate = True
binding_argtypes2 = ut.dict_take(templated_ctype_map_c, binding_argnames)
binding_sigargs2 = [type_ + ' ' + name for type_, name in
zip(binding_argtypes2, binding_argnames)]
T_typed_sigargs2 = ', '.join(binding_sigargs2)
T_typed_sigargs2_nl = ',\n'.join(binding_sigargs2)
if needstemplate:
iftemplate = '<{T_type}>'.format(T_type=T_type)
else:
iftemplate = ''
if return_type != 'void':
ifreturns = 'return '
else:
ifreturns = ''
bodyblock = c_bodyblock_fmtstr.format(signame_type=signame_type,
T_typed_sigargs2=T_typed_sigargs2,
binding_name=binding_name,
callargs=callargs,
iftemplate=iftemplate,
ifreturns=ifreturns,
return_type=return_type)
header_line_part1 = c_headersig_part1_fmtstr.format(signame_type=signame_type,
binding_name=binding_name,
return_type=return_type)
header_line = header_line_part1 + ut.indent(T_typed_sigargs2_nl, ' ' * len(header_line_part1)).lstrip(' ') + ');'
# Hack for header
header_line = header_line.replace('FLANNParameters* flann_params', 'struct FLANNParameters* flann_params')
#header_line = c_headersig_fmtstr.format(signame_type=signame_type,
# T_typed_sigargs2=T_typed_sigargs2,
# binding_name=binding_name,
# return_type=return_type)
c_header_sigs.append(header_line)
c_body_blocks.append(bodyblock)
flann_cpp_codeblock += '\n' + '\n'.join(c_body_blocks)
#flann_cpp_codeblock += '\n' + '// {binding_name} END'.format(binding_name=binding_name)
#BEGIN {binding_name}
flann_h_codeblock = ut.codeblock(
r'''
/**
{docstr_cpp}
*/
'''
).format(docstr_cpp=docstr_cpp, binding_name=binding_name)
flann_h_codeblock += '\n\n' + '\n\n'.join(c_header_sigs)
blocks_dict = {}
import re
flann_index_codeblock = ut.indent(flann_index_codeblock, ' ')
blocks_dict['flann_ctypes.py'] = flann_ctypes_codeblock
blocks_dict['index.py'] = flann_index_codeblock
blocks_dict['flann.h'] = flann_h_codeblock
blocks_dict['flann.cpp'] = flann_cpp_codeblock
for key in blocks_dict.keys():
blocks_dict[key] = re.sub('\n\s+\n', '\n\n', blocks_dict[key])
# , flags=re.MULTILINE)
blocks_dict[key] = re.sub('\s\s+\n', '\n', blocks_dict[key])
pass
if ut.get_argflag('--py'):
print('\n\n# ---------------\n\n')
print('GOES IN flann_ctypes.py')
print('\n\n# ---------------\n\n')
print(flann_ctypes_codeblock)
print('\n\n# ---------------\n\n')
print('GOES IN index.py')
print('\n\n# ---------------\n\n')
print(flann_index_codeblock)
if ut.get_argflag('--c'):
print('\n\n# ---------------\n\n')
print('GOES IN flann.h')
print('\n\n# ---------------\n\n')
print(flann_h_codeblock)
print('\n\n# ---------------\n\n')
print('GOES IN flann.cpp')
print('\n\n# ---------------\n\n')
print(flann_cpp_codeblock)
return blocks_dict, binding_def
def get_dbinfo(ibs, verbose=True,
with_imgsize=False,
with_bytes=False,
with_contrib=False,
with_agesex=False,
with_header=True,
short=False,
tag='dbinfo',
aid_list=None):
"""
Returns dictionary of digestable database information
Infostr is a string summary of all the stats. Prints infostr in addition to
returning locals
Args:
ibs (IBEISController):
verbose (bool):
with_imgsize (bool):
with_bytes (bool):
Returns:
dict:
CommandLine:
python -m ibeis.other.dbinfo --exec-get_dbinfo:0
python -m ibeis.other.dbinfo --test-get_dbinfo:1
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
python -m ibeis.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a ctrl
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
python -m ibeis.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0
Example1:
>>> # SCRIPT
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> defaultdb = 'testdb1'
>>> ibs, aid_list = ibeis.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
>>> kwargs = ut.get_kwdefaults(get_dbinfo)
>>> kwargs['verbose'] = False
>>> kwargs['aid_list'] = aid_list
>>> kwargs = ut.parse_dict_from_argv(kwargs)
>>> output = get_dbinfo(ibs, **kwargs)
>>> result = (output['info_str'])
>>> print(result)
>>> #ibs = ibeis.opendb(defaultdb='testdb1')
>>> # <HACK FOR FILTERING>
>>> #from ibeis.expt import cfghelpers
>>> #from ibeis.expt import annotation_configs
>>> #from ibeis.init import filter_annots
>>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
>>> # annotation_configs.TEST_NAMES)
>>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
>>> # ut.get_list_column(named_defaults_dict, 'qcfg')))
>>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
>>> #aid_list = ibs.get_valid_aids()
>>> # </HACK FOR FILTERING>
Example1:
>>> # ENABLE_DOCTEST
>>> from ibeis.other.dbinfo import * # NOQA
>>> import ibeis
>>> verbose = True
>>> short = True
>>> #ibs = ibeis.opendb(db='GZ_ALL')
>>> #ibs = ibeis.opendb(db='PZ_Master0')
>>> ibs = ibeis.opendb('testdb1')
>>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
>>> ibs.delete_contributors(ibs.get_valid_contrib_rowids())
>>> ibs.delete_empty_nids()
>>> #ibs = ibeis.opendb(db='PZ_MTEST')
>>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
>>> result = (output['info_str'])
>>> print(result)
+============================
DB Info: testdb1
DB Notes: None
DB NumContrib: 0
----------
# Names = 7
# Names (unassociated) = 0
# Names (singleton) = 5
# Names (multiton) = 2
----------
# Annots = 13
# Annots (unknown) = 4
# Annots (singleton) = 5
# Annots (multiton) = 4
----------
# Img = 13
L============================
"""
# TODO Database size in bytes
# TODO: occurrence, contributors, etc...
# Basic variables
request_annot_subset = False
_input_aid_list = aid_list # NOQA
if aid_list is None:
valid_aids = ibs.get_valid_aids()
valid_nids = ibs.get_valid_nids()
valid_gids = ibs.get_valid_gids()
else:
if isinstance(aid_list, str):
# Hack to get experiment stats on aids
acfg_name_list = [aid_list]
print('Specified custom aids via acfgname %s' % (acfg_name_list,))
from ibeis.expt import experiment_helpers
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs, acfg_name_list)
aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
#aid_list =
if verbose:
print('Specified %d custom aids' % (len(aid_list,)))
request_annot_subset = True
valid_aids = aid_list
valid_nids = list(
set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False)) -
{const.UNKNOWN_NAME_ROWID}
)
valid_gids = list(set(ibs.get_annot_gids(aid_list)))
#associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation
FILTER_HACK = True
if FILTER_HACK:
# HUGE HACK - get only images and names with filtered aids
valid_aids_ = ibs.filter_aids_custom(valid_aids)
valid_nids_ = ibs.filter_nids_custom(valid_nids)
valid_gids_ = ibs.filter_gids_custom(valid_gids)
if verbose:
print('Filtered %d names' % (len(valid_nids) - len(valid_nids_)))
print('Filtered %d images' % (len(valid_gids) - len(valid_gids_)))
print('Filtered %d annots' % (len(valid_aids) - len(valid_aids_)))
valid_gids = valid_gids_
valid_nids = valid_nids_
valid_aids = valid_aids_
#associated_nids = ut.compress(associated_nids, map(any,
#ibs.unflat_map(ibs.get_annot_custom_filterflags,
# ibs.get_name_aids(associated_nids))))
# Image info
if verbose:
print('Checking Image Info')
gx2_aids = ibs.get_image_aids(valid_gids)
if FILTER_HACK:
gx2_aids = [ibs.filter_aids_custom(aids) for aids in gx2_aids] # HACK FOR FILTER
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
gx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in gx2_aids]
gx2_nAnnots = np.array(list(map(len, gx2_aids)))
image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
gx2_nAnnots_stats = ut.get_stats_str(gx2_nAnnots, newlines=True, use_median=True)
image_reviewed_list = ibs.get_image_reviewed(valid_gids)
# Name stats
if verbose:
print('Checking Name Info')
nx2_aids = ibs.get_name_aids(valid_nids)
if FILTER_HACK:
nx2_aids = [ibs.filter_aids_custom(aids) for aids in nx2_aids] # HACK FOR FILTER
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
nx2_aids = [list(set(aids).intersection(valid_aids_set)) for aids in nx2_aids]
associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))
ibs.check_name_mapping_consistency(nx2_aids)
# Occurrence Info
def compute_annot_occurrence_ids(ibs, aid_list):
from ibeis.algo.preproc import preproc_occurrence
gid_list = ibs.get_annot_gids(aid_list)
gid2_aids = ut.group_items(aid_list, gid_list)
flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False)
occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()}
return occurid2_aids
import utool
with utool.embed_on_exception_context:
occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
nid2_occurxs = ut.ddict(list)
for occurx, nids in enumerate(occur_unique_nids):
for nid in nids:
nid2_occurxs[nid].append(occurx)
nid2_occurx_single = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1}
nid2_occurx_resight = {nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1}
singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())
singlesight_annot_stats = ut.get_stats(list(map(len, singlesight_encounters)), use_median=True, use_sum=True)
resight_name_stats = ut.get_stats(list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True)
try:
aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0)
undirected_tags = ibs.get_aidpair_tags(aid_pairs.T[0], aid_pairs.T[1], directed=False)
tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
pair_tag_info = ut.map_dict_vals(len, tag_dict)
num_reviewed_pairs = sum(ibs.get_annot_pair_is_reviewed(aid_pairs.T[0], aid_pairs.T[1]))
pair_tag_info['num_reviewed'] = num_reviewed_pairs
except Exception:
pair_tag_info = {}
#print(ut.dict_str(pair_tag_info))
# Annot Stats
# TODO: number of images where chips cover entire image
# TODO: total image coverage of annotation
# TODO: total annotation overlap
"""
ax2_unknown = ibs.is_aid_unknown(valid_aids)
ax2_nid = ibs.get_annot_name_rowids(valid_aids)
assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
"""
#
if verbose:
print('Checking Annot Species')
unknown_aids = ut.compress(valid_aids, ibs.is_aid_unknown(valid_aids))
species_list = ibs.get_annot_species_texts(valid_aids)
species2_aids = ut.group_items(valid_aids, species_list)
species2_nAids = {key: len(val) for key, val in species2_aids.items()}
if verbose:
print('Checking Multiton/Singleton Species')
nx2_nAnnots = np.array(list(map(len, nx2_aids)))
# Seperate singleton / multitons
multiton_nxs = np.where(nx2_nAnnots > 1)[0]
singleton_nxs = np.where(nx2_nAnnots == 1)[0]
unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
valid_nxs = np.hstack([multiton_nxs, singleton_nxs])
num_names_with_gt = len(multiton_nxs)
# Annot Info
if verbose:
print('Checking Annot Info')
multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
assert len(set(multiton_nxs)) == len(multiton_nxs)
if len(multiton_aids_list) == 0:
multiton_aids = np.array([], dtype=np.int)
else:
multiton_aids = np.hstack(multiton_aids_list)
assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
singleton_aids = ut.take(nx2_aids, singleton_nxs)
multiton_nid2_nannots = list(map(len, multiton_aids_list))
# Image size stats
if with_imgsize:
if verbose:
print('Checking ImageSize Info')
gpath_list = ibs.get_image_paths(valid_gids)
def wh_print_stats(wh_list):
if len(wh_list) == 0:
return '{empty}'
wh_list = np.asarray(wh_list)
stat_dict = OrderedDict(
[( 'max', wh_list.max(0)),
( 'min', wh_list.min(0)),
('mean', wh_list.mean(0)),
( 'std', wh_list.std(0))])
def arr2str(var):
return ('[' + (
', '.join(list(map(lambda x: '%.1f' % x, var)))
) + ']')
ret = (',\n '.join([
'%s:%s' % (key, arr2str(val))
for key, val in stat_dict.items()
]))
return '{\n ' + ret + '\n}'
print('reading image sizes')
# Image size stats
img_size_list = ibs.get_image_sizes(valid_gids)
img_size_stats = wh_print_stats(img_size_list)
# Chip size stats
annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
annotation_bbox_arr = np.array(annotation_bbox_list)
if len(annotation_bbox_arr) == 0:
annotation_size_list = []
else:
annotation_size_list = annotation_bbox_arr[:, 2:4]
chip_size_stats = wh_print_stats(annotation_size_list)
imgsize_stat_lines = [
(' # Img in dir = %d' % len(gpath_list)),
(' Image Size Stats = %s' % (img_size_stats,)),
(' * Chip Size Stats = %s' % (chip_size_stats,)),
]
else:
imgsize_stat_lines = []
if verbose:
print('Building Stats String')
multiton_stats = ut.get_stats_str(multiton_nid2_nannots, newlines=True, use_median=True)
# Time stats
unixtime_list = ibs.get_image_unixtime(valid_gids)
unixtime_list = ut.list_replace(unixtime_list, -1, float('nan'))
#valid_unixtime_list = [time for time in unixtime_list if time != -1]
#unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
if ut.get_argflag('--hackshow-unixtime'):
show_time_distributions(ibs, unixtime_list)
ut.show_if_requested()
unixtime_statstr = ut.get_timestats_str(unixtime_list, newlines=True, full=True)
# GPS stats
gps_list_ = ibs.get_image_gps(valid_gids)
gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
gps_list = ut.compress(gps_list_, gpsvalid_list)
def get_annot_age_stats(aid_list):
annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
age_dict = ut.ddict((lambda : 0))
for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
if (min_age is None or min_age < 12) and max_age < 12:
age_dict['Infant'] += 1
elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
age_dict['Juvenile'] += 1
elif 36 <= min_age and (36 <= max_age or max_age is None):
age_dict['Adult'] += 1
else:
print('Found UNKNOWN Age: %r, %r' % (min_age, max_age, ))
age_dict['UNKNOWN'] += 1
return age_dict
def get_annot_sex_stats(aid_list):
annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(set(annot_sextext_list) - set(sex_keys))
sextext2_nAnnots = ut.odict([(key, len(sextext2_aids.get(key, []))) for key in sex_keys])
# Filter 0's
sextext2_nAnnots = {key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0}
return sextext2_nAnnots
if verbose:
print('Checking Other Annot Stats')
qualtext2_nAnnots = ibs.get_annot_qual_stats(valid_aids)
yawtext2_nAnnots = ibs.get_annot_yaw_stats(valid_aids)
agetext2_nAnnots = get_annot_age_stats(valid_aids)
sextext2_nAnnots = get_annot_sex_stats(valid_aids)
if verbose:
print('Checking Contrib Stats')
# Contributor Statistics
# hack remove colon for image alignment
def fix_tag_list(tag_list):
return [None if tag is None else tag.replace(':', ';') for tag in tag_list]
image_contrib_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
annot_contrib_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
contrib_tag_to_gids = ut.group_items(valid_gids, image_contrib_tags)
contrib_tag_to_aids = ut.group_items(valid_aids, annot_contrib_tags)
contrib_tag_to_qualstats = {key: ibs.get_annot_qual_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}
contrib_tag_to_viewstats = {key: ibs.get_annot_yaw_stats(aids) for key, aids in six.iteritems(contrib_tag_to_aids)}
contrib_tag_to_nImages = {key: len(val) for key, val in six.iteritems(contrib_tag_to_gids)}
contrib_tag_to_nAnnots = {key: len(val) for key, val in six.iteritems(contrib_tag_to_aids)}
if verbose:
print('Summarizing')
# Summarize stats
num_names = len(valid_nids)
num_names_unassociated = len(valid_nids) - len(associated_nids)
num_names_singleton = len(singleton_nxs)
num_names_multiton = len(multiton_nxs)
num_singleton_annots = len(singleton_aids)
num_multiton_annots = len(multiton_aids)
num_unknown_annots = len(unknown_aids)
num_annots = len(valid_aids)
if with_bytes:
if verbose:
print('Checking Disk Space')
ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))
if True:
if verbose:
print('Check asserts')
try:
bad_aids = np.intersect1d(multiton_aids, unknown_aids)
_num_names_total_check = num_names_singleton + num_names_unassociated + num_names_multiton
_num_annots_total_check = num_unknown_annots + num_singleton_annots + num_multiton_annots
assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
assert _num_names_total_check == num_names, 'inconsistent num names'
#if not request_annot_subset:
# dont check this if you have an annot subset
assert _num_annots_total_check == num_annots, 'inconsistent num annots'
except Exception as ex:
ut.printex(ex, keys=[
'_num_names_total_check',
'num_names',
'_num_annots_total_check',
'num_annots',
'num_names_singleton',
'num_names_multiton',
'num_unknown_annots',
'num_multiton_annots',
'num_singleton_annots',
])
raise
# Get contributor statistics
contrib_rowids = ibs.get_valid_contrib_rowids()
num_contributors = len(contrib_rowids)
# print
num_tabs = 5
def align2(str_):
return ut.align(str_, ':', ' :')
def align_dict2(dict_):
str_ = ut.dict_str(dict_)
return align2(str_)
header_block_lines = (
[('+============================'), ] + (
[
('+ singleton := single sighting'),
('+ multiton := multiple sightings'),
('--' * num_tabs),
] if not short and with_header else []
)
)
source_block_lines = [
('DB Info: ' + ibs.get_dbname()),
('DB Notes: ' + ibs.get_dbnotes()),
('DB NumContrib: %d' % num_contributors),
]
bytes_block_lines = [
('--' * num_tabs),
('DB Bytes: '),
(' +- dbdir nBytes: ' + dbdir_space),
(' | +- _ibsdb nBytes: ' + ibsdir_space),
(' | | +-imgdir nBytes: ' + imgdir_space),
(' | | +-cachedir nBytes: ' + cachedir_space),
] if with_bytes else []
name_block_lines = [
('--' * num_tabs),
('# Names = %d' % num_names),
('# Names (unassociated) = %d' % num_names_unassociated),
('# Names (singleton) = %d' % num_names_singleton),
('# Names (multiton) = %d' % num_names_multiton),
]
subset_str = ' ' if not request_annot_subset else '(SUBSET)'
annot_block_lines = [
('--' * num_tabs),
('# Annots %s = %d' % (subset_str, num_annots,)),
('# Annots (unknown) = %d' % num_unknown_annots),
('# Annots (singleton) = %d' % num_singleton_annots),
('# Annots (multiton) = %d' % num_multiton_annots),
]
annot_per_basic_block_lines = [
('--' * num_tabs),
('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)),
('# Annots per Species = %s' % (align_dict2(species2_nAids),)),
] if not short else []
occurrence_block_lines = [
('--' * num_tabs),
('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
] if not short else []
annot_per_qualview_block_lines = [
None if short else '# Annots per Viewpoint = %s' % align_dict2(yawtext2_nAnnots),
None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
]
annot_per_agesex_block_lines = [
'# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
'# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
] if not short and with_agesex else []
contrib_block_lines = [
'# Images per contributor = ' + align_dict2(contrib_tag_to_nImages),
'# Annots per contributor = ' + align_dict2(contrib_tag_to_nAnnots),
'# Quality per contributor = ' + ut.dict_str(contrib_tag_to_qualstats, sorted_=True),
'# Viewpoint per contributor = ' + ut.dict_str(contrib_tag_to_viewstats, sorted_=True),
] if with_contrib else []
img_block_lines = [
('--' * num_tabs),
('# Img = %d' % len(valid_gids)),
None if short else ('# Img reviewed = %d' % sum(image_reviewed_list)),
None if short else ('# Img with gps = %d' % len(gps_list)),
#('# Img with timestamp = %d' % len(valid_unixtime_list)),
None if short else ('Img Time Stats = %s' % (align2(unixtime_statstr),)),
]
info_str_lines = (
header_block_lines +
bytes_block_lines +
source_block_lines +
name_block_lines +
annot_block_lines +
annot_per_basic_block_lines +
occurrence_block_lines +
annot_per_qualview_block_lines +
annot_per_agesex_block_lines +
img_block_lines +
contrib_block_lines +
imgsize_stat_lines +
[('L============================'), ]
)
info_str = '\n'.join(ut.filter_Nones(info_str_lines))
info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
if verbose:
print(info_str2)
locals_ = locals()
return locals_
def outline(self, indent):
import utool as ut
yield indent + self.__nice__()
indent = indent + ' ' * 4
yield ut.indent(ut.format_single_paragraph_sentences(self.resolve()),
indent)
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = ['ibeis/tests/assert_modules.py']
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/utool'
exclude_list = []
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
implicit_build_modlist_str = ut.codeblock('''
import sys
exclude_doctests_fnames = set(['__init__.py'])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['utool']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
''')
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
import_str = '\n'.join(
['import ' + modname for modname in doctest_modname_list])
modlist_str = (
'module_list = [%s\n]' %
ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
''')
pyscript_text = pyscript_fmtstr.format(
build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
def def_test(header,
pat=None,
dpath=None,
modname=None,
default=False,
testcmds=None):
""" interface to make test tuple """
return (header, default, modname, dpath, pat, testcmds)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds = ut.get_module_testlines(module_list,
remove_pyc=True,
verbose=False,
pythoncmd='RUN_TEST')
test_headers = [
# title, default, module, testpattern
def_test('DOC', testcmds=testcmds, default=True)
]
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs,
quick_tests, repodir,
exclude_list)
return shscript_text, pyscript_text
def define_flann_bindings(binding_name):
"""
Define the binding names for flann
"""
# default c source
c_source = None
optional_args = None
c_source_part = None
py_source = None
py_alias = None
py_args = None
pydoc = None
cpp_param_doc = {
'cols': 'number of columns in the dataset (feature dimensionality)',
'dataset': 'pointer to a data set stored in row major order',
'dists': ut.packtext(
'''pointer to matrix for the distances of the nearest neighbors of
the testset features in the dataset'''),
'flann_params': 'generic flann parameters',
'index_ptr': 'the index (constructed previously using flann_build_index)',
'nn': 'how many nearest neighbors to return',
'rebuild_threshold': ut.packtext(
'''reallocs index when it grows by factor of `rebuild_threshold`.
A smaller value results is more space efficient but less
computationally efficient. Must be greater than 1.'''),
'result_ids': ut.packtext(
'''pointer to matrix for the indices of the nearest neighbors of
the testset features in the dataset (must have tcount number of
rows and nn number of columns)'''),
'rows': 'number of rows (features) in the dataset',
'tcount': ut.packtext(
'''number of rows (features) in the query dataset (same
dimensionality as features in the dataset)'''),
'testset': 'pointer to a query set stored in row major order',
'level': 'verbosity level'
}
standard_csource = ut.codeblock(
r'''
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
return index->{cpp_binding_name}();
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
throw;
}}
'''
)
return_doc = None
cpp_binding_name = binding_name
zero_success = 'zero or a number <0 for error'
if binding_name == 'clean_removed_points':
cpp_binding_name = ut.to_camel_case(binding_name)
return_type = 'void'
docstr = 'Deletes removed points in index?'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'veclen':
return_type = 'int'
docstr = 'Returns number of features in this index'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'size':
return_type = 'int'
docstr = 'returns The dimensionality of the features in this index.'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'getType':
return_type = 'flann_algorithm_t'
docstr = 'returns The index type (kdtree, kmeans,...)'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'used_memory':
docstr = ut.codeblock(
'''
Returns the amount of memory (in bytes) used by the index
index_ptr = pointer to pre-built index.
Returns: int
'''
)
c_source = ut.codeblock(
r'''
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
return index->usedMemory();
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
'''
)
py_source = ut.codeblock(
'''
if self.__curindex is None:
return 0
return flann.used_memory[self.__curindex_type](self.__curindex)
''')
binding_argnames = ['index_ptr']
return_type = 'int'
elif binding_name == 'add_points':
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<ElementType> points = Matrix<ElementType>(points, rows, index->veclen());
index->addPoints(points, rebuild_threshold);
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
'''
)
py_source = ut.codeblock(
'''
if new_pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % new_pts.dtype)
if new_pts.dtype != self.__curindex_type:
raise FLANNException('New points must have the same type')
new_pts = ensure_2d_array(new_pts, default_flags)
rows = new_pts.shape[0]
flann.add_points[self.__curindex_type](self.__curindex, new_pts, rows, rebuild_threshold)
return self.__added_data.append(new_pts)
''')
#return_type = 'void'
return_type = 'int'
docstr = 'Adds points to pre-built index.'
if False:
binding_argnames = [
'index_ptr',
'points',
'rows',
'cols', # TODO: can remove
'rebuild_threshold',
]
else:
binding_argnames = [
'index_ptr',
'points',
'rows',
'rebuild_threshold',
]
return_doc = '0 if success otherwise -1'
py_args = ['new_pts', 'rebuild_threshold=2.']
cpp_param_doc['points'] = 'pointer to array of points'
elif binding_name == 'remove_point':
c_source = ut.codeblock(
r'''
size_t point_id(point_id_uint);
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->removePoint(point_id);
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
'''
)
py_source = ut.codeblock(
'''
flann.remove_point[self.__curindex_type](self.__curindex, point_id)
self.__removed_ids.append(point_id)
''')
#return_type = 'void'
return_type = 'int'
docstr = 'Removes a point from the index'
return_doc = zero_success
cpp_param_doc['point_id'] = 'point id to be removed'
cpp_param_doc['index_ptr'] = 'The index that should be modified'
binding_argnames = ['index_ptr', 'point_id']
elif binding_name == 'remove_points':
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
try {{
if (index_ptr==NULL) {{
thow FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->removePoints(id_list, num);
return;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return;
}}
'''
)
py_source = ut.codeblock(
'''
id_list = np.array(id_list, dtype=np.int32)
num = len(id_list)
flann.remove_points[self.__curindex_type](self.__curindex, id_list, num)
self.__removed_ids.extend(id_list)
''')
cpp_param_doc['index_ptr'] = 'The index that should be modified'
cpp_param_doc['id_list'] = 'list of point ids to be removed'
cpp_param_doc['num'] = 'number of points in id_list'
docstr = 'Removes multiple points from the index'
return_doc = 'void'
py_args = ['id_list']
return_type = 'void'
binding_argnames = ['index_ptr', 'id_list', 'num']
elif binding_name == 'compute_cluster_centers':
docstr = ut.textblock(
r'''
Clusters the features in the dataset using a hierarchical kmeans
clustering approach. This is significantly faster than using a
flat kmeans clustering for a large number of clusters.
''')
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {
init_flann_parameters(flann_params);
Matrix<ElementType> inputData(dataset,rows,cols);
KMeansIndexParams params(flann_params->branching, flann_params->iterations, flann_params->centers_init, flann_params->cb_index);
Matrix<DistanceType> centers(result_centers, clusters,cols);
int clusterNum = hierarchicalClustering<Distance>(inputData, centers, params, d);
return clusterNum;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return -1;
}
'''.replace('{', '{{').replace('}', '}}')
)
py_source = ut.codeblock(
'''
# First verify the paremeters are sensible.
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if int(branch_size) != branch_size or branch_size < 2:
raise FLANNException('branch_size must be an integer >= 2.')
branch_size = int(branch_size)
if int(num_branches) != num_branches or num_branches < 1:
raise FLANNException('num_branches must be an integer >= 1.')
num_branches = int(num_branches)
if max_iterations is None:
max_iterations = -1
else:
max_iterations = int(max_iterations)
# init the arrays and starting values
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
num_clusters = (branch_size - 1) * num_branches + 1
if pts.dtype.type == np.float64:
result = np.empty((num_clusters, dim), dtype=np.float64)
else:
result = np.empty((num_clusters, dim), dtype=np.float32)
# set all the parameters appropriately
self.__ensureRandomSeed(kwargs)
params = {'iterations': max_iterations,
'algorithm': 'kmeans',
'branching': branch_size,
'random_seed': kwargs['random_seed']}
self.__flann_parameters.update(params)
numclusters = flann.compute_cluster_centers[pts.dtype.type](
pts, npts, dim, num_clusters, result,
pointer(self.__flann_parameters))
if numclusters <= 0:
raise FLANNException('Error occured during clustering procedure.')
if dtype is None:
return result
else:
return dtype(result)
''').replace('}', '}}').replace('{', '{{')
return_doc = ut.packtext(
'''number of clusters computed or a number <0 for error. This
number can be different than the number of clusters requested, due
to the way hierarchical clusters are computed. The number of
clusters returned will be the highest number of the form
(branch_size-1)*K+1 smaller than the number of clusters
requested.''')
cpp_param_doc['clusters'] = 'number of cluster to compute'
cpp_param_doc['result_centers'] = 'memory buffer where the output cluster centers are stored'
cpp_param_doc['flann_params'] = 'generic flann parameters and index_params used to specify the kmeans tree parameters (branching factor, max number of iterations to use)'
return_type = 'int'
binding_argnames = ['dataset', 'rows', 'cols', 'clusters', 'result_centers', 'flann_params']
optional_args = ['Distance d = Distance()']
py_alias = 'hierarchical_kmeans'
py_args = 'pts, branch_size, num_branches, max_iterations=None, dtype=None, **kwargs'.split(', ')
elif binding_name == 'radius_search':
docstr = ut.codeblock(
r'''
Performs an radius search using an already constructed index.
In case of radius search, instead of always returning a predetermined
number of nearest neighbours (for example the 10 nearest neighbours), the
search will return all the neighbours found within a search radius
of the query point.
The check parameter in the FLANNParameters below sets the level of approximation
for the search by only visiting "checks" number of features in the index
(the same way as for the KNN search). A lower value for checks will give
a higher search speedup at the cost of potentially not returning all the
neighbours in the specified radius.
''')
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {{
init_flann_parameters(flann_params);
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_result_ids(result_ids, 1, max_nn);
Matrix<DistanceType> m_dists(dists1d, 1, max_nn);
SearchParams search_params = create_search_params(flann_params);
int count = index->radiusSearch(Matrix<ElementType>(query1d, 1, index->veclen()),
m_result_ids,
m_dists, radius, search_params );
return count;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
''')
py_source = ut.codeblock(
'''
if self.__curindex is None:
raise FLANNException(
'build_index(...) method not called first or current index deleted.')
if query.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % query.dtype)
if self.__curindex_type != query.dtype.type:
raise FLANNException('Index and query must have the same type')
npts, dim = self.get_indexed_shape()
assert(query.shape[0] == dim), 'data and query must have the same dims'
result = np.empty(npts, dtype=index_type)
if self.__curindex_type == np.float64:
dists = np.empty(npts, dtype=np.float64)
else:
dists = np.empty(npts, dtype=np.float32)
self.__flann_parameters.update(kwargs)
nn = flann.radius_search[
self.__curindex_type](
self.__curindex, query, result, dists, npts, radius,
pointer(self.__flann_parameters))
return (result[0:nn], dists[0:nn])
''')
cpp_param_doc['index_ptr'] = 'the index'
cpp_param_doc['query1d'] = 'query point'
cpp_param_doc['dists1d'] = 'similar, but for storing distances'
cpp_param_doc['result_ids'] = 'array for storing the indices found (will be modified)'
cpp_param_doc['max_nn'] = 'size of arrays result_ids and dists1d'
cpp_param_doc['radius'] = 'search radius (squared radius for euclidian metric)'
return_doc = 'number of neighbors found or <0 for an error'
return_type = 'int'
binding_argnames = ['index_ptr', 'query1d', 'result_ids', 'dists1d', 'max_nn',
'radius', 'flann_params', ]
py_alias = 'nn_radius'
py_args = 'query, radius, **kwargs'.split(', ')
elif binding_name == 'find_nearest_neighbors_index':
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {
init_flann_parameters(flann_params);
if (index_ptr==NULL) {
throw FLANNException("Invalid index");
}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_indices(result_ids,tcount, nn);
Matrix<DistanceType> m_dists(dists, tcount, nn);
SearchParams search_params = create_search_params(flann_params);
index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()),
m_indices,
m_dists, nn, search_params );
return 0;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return -1;
}
return -1;
'''
).replace('{', '{{').replace('}', '}}')
py_source = ut.codeblock(
'''
if self.__curindex is None:
raise FLANNException(
'build_index(...) method not called first or current index deleted.')
if qpts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % qpts.dtype)
if self.__curindex_type != qpts.dtype.type:
raise FLANNException('Index and query must have the same type')
qpts = ensure_2d_array(qpts, default_flags)
npts, dim = self.get_indexed_shape()
if qpts.size == dim:
qpts.reshape(1, dim)
nqpts = qpts.shape[0]
assert qpts.shape[1] == dim, 'data and query must have the same dims'
assert npts >= num_neighbors, 'more neighbors than there are points'
result = np.empty((nqpts, num_neighbors), dtype=index_type)
if self.__curindex_type == np.float64:
dists = np.empty((nqpts, num_neighbors), dtype=np.float64)
else:
dists = np.empty((nqpts, num_neighbors), dtype=np.float32)
self.__flann_parameters.update(kwargs)
flann.find_nearest_neighbors_index[
self.__curindex_type](
self.__curindex, qpts, nqpts, result, dists, num_neighbors,
pointer(self.__flann_parameters))
if num_neighbors == 1:
return (result.reshape(nqpts), dists.reshape(nqpts))
else:
return (result, dists)
'''
)
docstr = 'Searches for nearest neighbors using the index provided'
return_doc = zero_success
return_type = 'int'
# optional_args = ['Distance d = Distance()']
binding_argnames = ['index_ptr', 'testset', 'tcount', 'result_ids',
'dists', 'nn', 'flann_params', ]
py_alias = 'nn_index'
py_args = ['qpts', 'num_neighbors=1', '**kwargs']
elif binding_name == 'find_nearest_neighbors':
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {{
init_flann_parameters(flann_params);
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_indices(result_ids,tcount, nn);
Matrix<DistanceType> m_dists(dists, tcount, nn);
SearchParams search_params = create_search_params(flann_params);
index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()),
m_indices,
m_dists, nn, search_params );
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
return -1;
'''
)
py_source = ut.codeblock(
'''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if qpts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if pts.dtype != qpts.dtype:
raise FLANNException('Data and query must have the same type')
pts = ensure_2d_array(pts, default_flags)
qpts = ensure_2d_array(qpts, default_flags)
npts, dim = pts.shape
nqpts = qpts.shape[0]
assert qpts.shape[1] == dim, 'data and query must have the same dims'
assert npts >= num_neighbors, 'more neighbors than there are points'
result = np.empty((nqpts, num_neighbors), dtype=index_type)
if pts.dtype == np.float64:
dists = np.empty((nqpts, num_neighbors), dtype=np.float64)
else:
dists = np.empty((nqpts, num_neighbors), dtype=np.float32)
self.__flann_parameters.update(kwargs)
flann.find_nearest_neighbors[
pts.dtype.type](
pts, npts, dim, qpts, nqpts, result, dists, num_neighbors,
pointer(self.__flann_parameters))
if num_neighbors == 1:
return (result.reshape(nqpts), dists.reshape(nqpts))
else:
return (result, dists)
''')
docstr = 'Builds an index and uses it to find nearest neighbors.'
return_doc = zero_success
py_alias = 'nn'
py_args = ['pts', 'qpts', 'num_neighbors=1', '**kwargs']
return_type = 'int'
binding_argnames = ['dataset', 'rows', 'cols', 'testset', 'tcount',
'result_ids', 'dists', 'nn', 'flann_params']
optional_args = ['Distance d = Distance()']
elif binding_name == 'load_index':
c_source_part = ut.codeblock(
r'''
Index<Distance>* index = new Index<Distance>(Matrix<typename Distance::ElementType>(dataset,rows,cols), SavedIndexParams(filename), d);
return index;
'''
)
py_source = ut.codeblock(
'''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
self.__curindex_data = None
self.__added_data = []
self.__curindex_type = None
self.__curindex = flann.load_index[pts.dtype.type](
c_char_p(to_bytes(filename)), pts, npts, dim)
if self.__curindex is None:
raise FLANNException(
('Error loading the FLANN index with filename=%r.'
' C++ may have thrown more detailed errors') % (filename,))
self.__curindex_data = pts
self.__added_data = []
self.__removed_ids = []
self.__curindex_type = pts.dtype.type
''')
docstr = 'Loads a previously saved index from a file.'
return_doc = 'index_ptr'
cpp_param_doc['dataset'] = 'The dataset corresponding to the index'
cpp_param_doc['filename'] = 'File to load the index from'
py_args = ['filename', 'pts']
return_type = 'flann_index_t'
binding_argnames = ['filename', 'dataset', 'rows', 'cols']
optional_args = ['Distance d = Distance()']
elif binding_name == 'save_index':
docstr = 'Saves the index to a file. Only the index is saved into the file, the dataset corresponding to the index is not saved.'
cpp_param_doc['index_ptr'] = 'The index that should be saved'
cpp_param_doc['filename'] = 'The filename the index should be saved to'
return_doc = 'Returns 0 on success, negative value on error'
c_source_part = ut.codeblock(
r'''
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->save(filename);
return 0;
''')
py_source = ut.codeblock(
'''
if self.__curindex is not None:
flann.save_index[self.__curindex_type](
self.__curindex, c_char_p(to_bytes(filename)))
''')
return_type = 'int'
binding_argnames = ['index_ptr', 'filename']
py_alias = None
py_args = None
elif binding_name == 'build_index':
docstr = ut.codeblock(
'''
Builds and returns an index. It uses autotuning if the target_precision field of index_params
is between 0 and 1, or the parameters specified if it's -1.
''')
pydoc = ut.codeblock(
'''
This builds and internally stores an index to be used for
future nearest neighbor matchings. It erases any previously
stored indexes, so use multiple instances of this class to
work with multiple stored indices. Use nn_index(...) to find
the nearest neighbors in this index.
pts is a 2d numpy array or matrix. All the computation is done
in np.float32 type, but pts may be any type that is convertable
to np.float32.
''')
c_source = ut.codeblock(
r'''
typedef typename Distance::ElementType ElementType;
try {
init_flann_parameters(flann_params);
if (flann_params == NULL) {
throw FLANNException("The flann_params argument must be non-null");
}
IndexParams params = create_parameters(flann_params);
Index<Distance>* index = new Index<Distance>(Matrix<ElementType>(dataset,rows,cols), params, d);
index->buildIndex();
if (flann_params->algorithm==FLANN_INDEX_AUTOTUNED) {
IndexParams params = index->getParameters();
update_flann_parameters(params,flann_params);
SearchParams search_params = get_param<SearchParams>(params,"search_params");
*speedup = get_param<float>(params,"speedup");
flann_params->checks = search_params.checks;
flann_params->eps = search_params.eps;
flann_params->cb_index = get_param<float>(params,"cb_index",0.0);
}
return index;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return NULL;
}
''').replace('{', '{{').replace('}', '}}')
py_source = ut.codeblock(
'''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
self.__ensureRandomSeed(kwargs)
self.__flann_parameters.update(kwargs)
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
speedup = c_float(0)
self.__curindex = flann.build_index[pts.dtype.type](
pts, npts, dim, byref(speedup), pointer(self.__flann_parameters))
self.__curindex_data = pts
self.__curindex_type = pts.dtype.type
params = dict(self.__flann_parameters)
params['speedup'] = speedup.value
return params
''')
# binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params']
return_doc = 'the newly created index or a number <0 for error'
cpp_param_doc['speedup'] = 'speedup over linear search, estimated if using autotuning, output parameter'
optional_args = ['Distance d = Distance()']
return_type = 'flann_index_t'
py_args = ['pts', '**kwargs']
binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params']
elif binding_name == 'free_index':
docstr = 'Deletes an index and releases the memory used by it.'
pydoc = ut.codeblock(
'''
Deletes the current index freeing all the momory it uses.
The memory used by the dataset that was indexed is not freed
unless there are no other references to those numpy arrays.
''')
c_source_part = ut.codeblock(
r'''
Index<Distance>* index = (Index<Distance>*)index_ptr;
delete index;
return 0;
''')
py_source = ut.codeblock(
'''
self.__flann_parameters.update(kwargs)
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
self.__curindex_data = None
self.__added_data = []
self.__removed_ids = []
''')
return_doc = zero_success
return_type = 'int'
binding_argnames = ['index_ptr', 'flann_params']
cpp_param_doc['flann_params'] = ut.textblock(
'''generic flann params (only used to specify verbosity)''')
py_alias = 'delete_index'
py_args = ['**kwargs']
elif binding_name == 'get_point':
docstr = 'Gets a point from a given index position.'
return_doc = 'pointer to datapoint or NULL on miss'
binding_argnames = ['index_ptr', 'point_id']
cpp_param_doc['point_id'] = 'index of datapoint to get.'
return_type = 'Distance::ElementType*'
elif binding_name == 'flann_get_distance_order':
docstr = ut.textblock(
'''Gets the distance order in use throughout FLANN (only applicable
if minkowski distance is in use).''')
binding_argnames = []
return_type = 'int'
else:
dictdef = {
'_template_new': {
'docstr': '',
'binding_argnames': [],
'return_type': 'int',
},
'flann_get_distance_type': {
'docstr': '',
'binding_argnames': [],
'return_type': 'int',
},
'flann_log_verbosity': {
'docstr': ut.codeblock(
'''
Sets the log level used for all flann functions (unless
specified in FLANNParameters for each call
'''
),
'binding_argnames': ['level'],
'return_type': 'void',
},
}
if binding_name in dictdef:
docstr = dictdef[binding_name].get('docstr', '')
binding_argnames = dictdef[binding_name]['binding_argnames']
return_type = dictdef[binding_name]['return_type']
else:
raise NotImplementedError('Unknown binding name %r' % (binding_name,))
if c_source is None:
if c_source_part is not None:
try_ = ut.codeblock(
'''
try {{
'''
)
throw_ = '\n' + ut.indent(ut.codeblock(
'''
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
'''
), ' ' * 4)
if 'index_ptr' not in binding_argnames:
throw_ = ''
if 'flann_params' in binding_argnames:
part1 = try_ + '\n' + ' init_flann_parameters(flann_params);' + throw_
else:
part1 = try_ + throw_
if return_type == 'int':
default_return = '-1'
else:
default_return = 'NULL'
part2 = ut.codeblock(
r'''
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return ''' + default_return + ''';
}}
'''
)
c_source = part1 + '\n' + ut.indent(c_source_part, ' ' * 4) + '\n' + part2
else:
c_source = ut.codeblock(
'''
TODO: IMPLEMENT THIS FUNCTION WRAPPER
'''
)
try:
docstr_cpp = docstr[:]
if return_doc is not None:
param_docs = ut.dict_take(cpp_param_doc, binding_argnames)
cpp_param_docblock = '\n'.join(
['%s = %s' % (name, doc)
for name, doc in zip(binding_argnames, param_docs)])
docstr_cpp += '\n\n' + 'Params:\n' + ut.indent(cpp_param_docblock, ' ')
docstr_cpp += '\n\n' + 'Returns: ' + return_doc
if pydoc is None:
docstr_py = docstr[:]
else:
docstr_py = pydoc[:]
if py_args:
py_param_doc = cpp_param_doc.copy()
py_param_doc['pts'] = py_param_doc['dataset'].replace('pointer to ', '')
py_param_doc['qpts'] = (py_param_doc['testset'].replace(
'pointer to ', '') + ' (may be a single point)')
py_param_doc['num_neighbors'] = py_param_doc['nn']
py_param_doc['**kwargs'] = py_param_doc['flann_params']
py_args_ = [a.split('=')[0] for a in py_args]
param_docs = ut.dict_take(py_param_doc, py_args_, '')
# py_types =
py_param_docblock = '\n'.join(['%s: %s' % (name, doc)
for name, doc in zip(py_args_, param_docs)])
docstr_py += '\n\n' + 'Params:\n' + ut.indent(py_param_docblock, ' ')
except Exception as ex:
ut.printex(ex, keys=['binding_name'])
raise
pass
binding_def = {
'cpp_binding_name': cpp_binding_name,
'docstr_cpp': docstr_cpp,
'docstr_py': docstr_py,
'return_type': return_type,
'binding_argnames': binding_argnames,
'c_source': c_source,
'optional_args': optional_args,
'py_source': py_source,
'py_args': py_args,
'py_alias': py_alias,
}
return binding_def
def define_flann_bindings(binding_name):
"""
Define the binding names for flann
"""
# default c source
c_source = None
optional_args = None
c_source_part = None
py_source = None
py_alias = None
py_args = None
pydoc = None
cpp_param_doc = {
'cols':
'number of columns in the dataset (feature dimensionality)',
'dataset':
'pointer to a data set stored in row major order',
'dists':
ut.packtext(
'''pointer to matrix for the distances of the nearest neighbors of
the testset features in the dataset'''),
'flann_params':
'generic flann parameters',
'index_ptr':
'the index (constructed previously using flann_build_index)',
'nn':
'how many nearest neighbors to return',
'rebuild_threshold':
ut.packtext(
'''reallocs index when it grows by factor of `rebuild_threshold`.
A smaller value results is more space efficient but less
computationally efficient. Must be greater than 1.'''),
'result_ids':
ut.packtext(
'''pointer to matrix for the indices of the nearest neighbors of
the testset features in the dataset (must have tcount number of
rows and nn number of columns)'''),
'rows':
'number of rows (features) in the dataset',
'tcount':
ut.packtext('''number of rows (features) in the query dataset (same
dimensionality as features in the dataset)'''),
'testset':
'pointer to a query set stored in row major order',
'level':
'verbosity level'
}
standard_csource = ut.codeblock(r'''
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
return index->{cpp_binding_name}();
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
throw;
}}
''')
return_doc = None
cpp_binding_name = binding_name
zero_success = 'zero or a number <0 for error'
if binding_name == 'clean_removed_points':
cpp_binding_name = ut.to_camel_case(binding_name)
return_type = 'void'
docstr = 'Deletes removed points in index?'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'veclen':
return_type = 'int'
docstr = 'Returns number of features in this index'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'size':
return_type = 'int'
docstr = 'returns The dimensionality of the features in this index.'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'getType':
return_type = 'flann_algorithm_t'
docstr = 'returns The index type (kdtree, kmeans,...)'
binding_argnames = ['index_ptr']
c_source = standard_csource
elif binding_name == 'used_memory':
docstr = ut.codeblock('''
Returns the amount of memory (in bytes) used by the index
index_ptr = pointer to pre-built index.
Returns: int
''')
c_source = ut.codeblock(r'''
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
return index->usedMemory();
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
''')
py_source = ut.codeblock('''
if self.__curindex is None:
return 0
return flann.used_memory[self.__curindex_type](self.__curindex)
''')
binding_argnames = ['index_ptr']
return_type = 'int'
elif binding_name == 'add_points':
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<ElementType> points = Matrix<ElementType>(points, rows, index->veclen());
index->addPoints(points, rebuild_threshold);
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
''')
py_source = ut.codeblock('''
if new_pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % new_pts.dtype)
if new_pts.dtype != self.__curindex_type:
raise FLANNException('New points must have the same type')
new_pts = ensure_2d_array(new_pts, default_flags)
rows = new_pts.shape[0]
flann.add_points[self.__curindex_type](self.__curindex, new_pts, rows, rebuild_threshold)
return self.__added_data.append(new_pts)
''')
#return_type = 'void'
return_type = 'int'
docstr = 'Adds points to pre-built index.'
if False:
binding_argnames = [
'index_ptr',
'points',
'rows',
'cols', # TODO: can remove
'rebuild_threshold',
]
else:
binding_argnames = [
'index_ptr',
'points',
'rows',
'rebuild_threshold',
]
return_doc = '0 if success otherwise -1'
py_args = ['new_pts', 'rebuild_threshold=2.']
cpp_param_doc['points'] = 'pointer to array of points'
elif binding_name == 'remove_point':
c_source = ut.codeblock(r'''
size_t point_id(point_id_uint);
try {{
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->removePoint(point_id);
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
''')
py_source = ut.codeblock('''
flann.remove_point[self.__curindex_type](self.__curindex, point_id)
self.__removed_ids.append(point_id)
''')
#return_type = 'void'
return_type = 'int'
docstr = 'Removes a point from the index'
return_doc = zero_success
cpp_param_doc['point_id'] = 'point id to be removed'
cpp_param_doc['index_ptr'] = 'The index that should be modified'
binding_argnames = ['index_ptr', 'point_id']
elif binding_name == 'remove_points':
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
try {{
if (index_ptr==NULL) {{
thow FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->removePoints(id_list, num);
return;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return;
}}
''')
py_source = ut.codeblock('''
id_list = np.array(id_list, dtype=np.int32)
num = len(id_list)
flann.remove_points[self.__curindex_type](self.__curindex, id_list, num)
self.__removed_ids.extend(id_list)
''')
cpp_param_doc['index_ptr'] = 'The index that should be modified'
cpp_param_doc['id_list'] = 'list of point ids to be removed'
cpp_param_doc['num'] = 'number of points in id_list'
docstr = 'Removes multiple points from the index'
return_doc = 'void'
py_args = ['id_list']
return_type = 'void'
binding_argnames = ['index_ptr', 'id_list', 'num']
elif binding_name == 'compute_cluster_centers':
docstr = ut.textblock(r'''
Clusters the features in the dataset using a hierarchical kmeans
clustering approach. This is significantly faster than using a
flat kmeans clustering for a large number of clusters.
''')
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {
init_flann_parameters(flann_params);
Matrix<ElementType> inputData(dataset,rows,cols);
KMeansIndexParams params(flann_params->branching, flann_params->iterations, flann_params->centers_init, flann_params->cb_index);
Matrix<DistanceType> centers(result_centers, clusters,cols);
int clusterNum = hierarchicalClustering<Distance>(inputData, centers, params, d);
return clusterNum;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return -1;
}
'''.replace('{', '{{').replace('}', '}}'))
py_source = ut.codeblock('''
# First verify the paremeters are sensible.
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if int(branch_size) != branch_size or branch_size < 2:
raise FLANNException('branch_size must be an integer >= 2.')
branch_size = int(branch_size)
if int(num_branches) != num_branches or num_branches < 1:
raise FLANNException('num_branches must be an integer >= 1.')
num_branches = int(num_branches)
if max_iterations is None:
max_iterations = -1
else:
max_iterations = int(max_iterations)
# init the arrays and starting values
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
num_clusters = (branch_size - 1) * num_branches + 1
if pts.dtype.type == np.float64:
result = np.empty((num_clusters, dim), dtype=np.float64)
else:
result = np.empty((num_clusters, dim), dtype=np.float32)
# set all the parameters appropriately
self.__ensureRandomSeed(kwargs)
params = {'iterations': max_iterations,
'algorithm': 'kmeans',
'branching': branch_size,
'random_seed': kwargs['random_seed']}
self.__flann_parameters.update(params)
numclusters = flann.compute_cluster_centers[pts.dtype.type](
pts, npts, dim, num_clusters, result,
pointer(self.__flann_parameters))
if numclusters <= 0:
raise FLANNException('Error occured during clustering procedure.')
if dtype is None:
return result
else:
return dtype(result)
''').replace('}', '}}').replace('{', '{{')
return_doc = ut.packtext(
'''number of clusters computed or a number <0 for error. This
number can be different than the number of clusters requested, due
to the way hierarchical clusters are computed. The number of
clusters returned will be the highest number of the form
(branch_size-1)*K+1 smaller than the number of clusters
requested.''')
cpp_param_doc['clusters'] = 'number of cluster to compute'
cpp_param_doc[
'result_centers'] = 'memory buffer where the output cluster centers are stored'
cpp_param_doc[
'flann_params'] = 'generic flann parameters and index_params used to specify the kmeans tree parameters (branching factor, max number of iterations to use)'
return_type = 'int'
binding_argnames = [
'dataset', 'rows', 'cols', 'clusters', 'result_centers',
'flann_params'
]
optional_args = ['Distance d = Distance()']
py_alias = 'hierarchical_kmeans'
py_args = 'pts, branch_size, num_branches, max_iterations=None, dtype=None, **kwargs'.split(
', ')
elif binding_name == 'radius_search':
docstr = ut.codeblock(r'''
Performs an radius search using an already constructed index.
In case of radius search, instead of always returning a predetermined
number of nearest neighbours (for example the 10 nearest neighbours), the
search will return all the neighbours found within a search radius
of the query point.
The check parameter in the FLANNParameters below sets the level of approximation
for the search by only visiting "checks" number of features in the index
(the same way as for the KNN search). A lower value for checks will give
a higher search speedup at the cost of potentially not returning all the
neighbours in the specified radius.
''')
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {{
init_flann_parameters(flann_params);
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_result_ids(result_ids, 1, max_nn);
Matrix<DistanceType> m_dists(dists1d, 1, max_nn);
SearchParams search_params = create_search_params(flann_params);
int count = index->radiusSearch(Matrix<ElementType>(query1d, 1, index->veclen()),
m_result_ids,
m_dists, radius, search_params );
return count;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
''')
py_source = ut.codeblock('''
if self.__curindex is None:
raise FLANNException(
'build_index(...) method not called first or current index deleted.')
if query.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % query.dtype)
if self.__curindex_type != query.dtype.type:
raise FLANNException('Index and query must have the same type')
npts, dim = self.get_indexed_shape()
assert(query.shape[0] == dim), 'data and query must have the same dims'
result = np.empty(npts, dtype=index_type)
if self.__curindex_type == np.float64:
dists = np.empty(npts, dtype=np.float64)
else:
dists = np.empty(npts, dtype=np.float32)
self.__flann_parameters.update(kwargs)
nn = flann.radius_search[
self.__curindex_type](
self.__curindex, query, result, dists, npts, radius,
pointer(self.__flann_parameters))
return (result[0:nn], dists[0:nn])
''')
cpp_param_doc['index_ptr'] = 'the index'
cpp_param_doc['query1d'] = 'query point'
cpp_param_doc['dists1d'] = 'similar, but for storing distances'
cpp_param_doc[
'result_ids'] = 'array for storing the indices found (will be modified)'
cpp_param_doc['max_nn'] = 'size of arrays result_ids and dists1d'
cpp_param_doc[
'radius'] = 'search radius (squared radius for euclidian metric)'
return_doc = 'number of neighbors found or <0 for an error'
return_type = 'int'
binding_argnames = [
'index_ptr',
'query1d',
'result_ids',
'dists1d',
'max_nn',
'radius',
'flann_params',
]
py_alias = 'nn_radius'
py_args = 'query, radius, **kwargs'.split(', ')
elif binding_name == 'find_nearest_neighbors_index':
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {
init_flann_parameters(flann_params);
if (index_ptr==NULL) {
throw FLANNException("Invalid index");
}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_indices(result_ids,tcount, nn);
Matrix<DistanceType> m_dists(dists, tcount, nn);
SearchParams search_params = create_search_params(flann_params);
index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()),
m_indices,
m_dists, nn, search_params );
return 0;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return -1;
}
return -1;
''').replace('{', '{{').replace('}', '}}')
py_source = ut.codeblock('''
if self.__curindex is None:
raise FLANNException(
'build_index(...) method not called first or current index deleted.')
if qpts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % qpts.dtype)
if self.__curindex_type != qpts.dtype.type:
raise FLANNException('Index and query must have the same type')
qpts = ensure_2d_array(qpts, default_flags)
npts, dim = self.get_indexed_shape()
if qpts.size == dim:
qpts.reshape(1, dim)
nqpts = qpts.shape[0]
assert qpts.shape[1] == dim, 'data and query must have the same dims'
assert npts >= num_neighbors, 'more neighbors than there are points'
result = np.empty((nqpts, num_neighbors), dtype=index_type)
if self.__curindex_type == np.float64:
dists = np.empty((nqpts, num_neighbors), dtype=np.float64)
else:
dists = np.empty((nqpts, num_neighbors), dtype=np.float32)
self.__flann_parameters.update(kwargs)
flann.find_nearest_neighbors_index[
self.__curindex_type](
self.__curindex, qpts, nqpts, result, dists, num_neighbors,
pointer(self.__flann_parameters))
if num_neighbors == 1:
return (result.reshape(nqpts), dists.reshape(nqpts))
else:
return (result, dists)
''')
docstr = 'Searches for nearest neighbors using the index provided'
return_doc = zero_success
return_type = 'int'
# optional_args = ['Distance d = Distance()']
binding_argnames = [
'index_ptr',
'testset',
'tcount',
'result_ids',
'dists',
'nn',
'flann_params',
]
py_alias = 'nn_index'
py_args = ['qpts', 'num_neighbors=1', '**kwargs']
elif binding_name == 'find_nearest_neighbors':
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
typedef typename Distance::ResultType DistanceType;
try {{
init_flann_parameters(flann_params);
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
Index<Distance>* index = (Index<Distance>*)index_ptr;
Matrix<int> m_indices(result_ids,tcount, nn);
Matrix<DistanceType> m_dists(dists, tcount, nn);
SearchParams search_params = create_search_params(flann_params);
index->knnSearch(Matrix<ElementType>(testset, tcount, index->veclen()),
m_indices,
m_dists, nn, search_params );
return 0;
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return -1;
}}
return -1;
''')
py_source = ut.codeblock('''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if qpts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
if pts.dtype != qpts.dtype:
raise FLANNException('Data and query must have the same type')
pts = ensure_2d_array(pts, default_flags)
qpts = ensure_2d_array(qpts, default_flags)
npts, dim = pts.shape
nqpts = qpts.shape[0]
assert qpts.shape[1] == dim, 'data and query must have the same dims'
assert npts >= num_neighbors, 'more neighbors than there are points'
result = np.empty((nqpts, num_neighbors), dtype=index_type)
if pts.dtype == np.float64:
dists = np.empty((nqpts, num_neighbors), dtype=np.float64)
else:
dists = np.empty((nqpts, num_neighbors), dtype=np.float32)
self.__flann_parameters.update(kwargs)
flann.find_nearest_neighbors[
pts.dtype.type](
pts, npts, dim, qpts, nqpts, result, dists, num_neighbors,
pointer(self.__flann_parameters))
if num_neighbors == 1:
return (result.reshape(nqpts), dists.reshape(nqpts))
else:
return (result, dists)
''')
docstr = 'Builds an index and uses it to find nearest neighbors.'
return_doc = zero_success
py_alias = 'nn'
py_args = ['pts', 'qpts', 'num_neighbors=1', '**kwargs']
return_type = 'int'
binding_argnames = [
'dataset', 'rows', 'cols', 'testset', 'tcount', 'result_ids',
'dists', 'nn', 'flann_params'
]
optional_args = ['Distance d = Distance()']
elif binding_name == 'load_index':
c_source_part = ut.codeblock(r'''
Index<Distance>* index = new Index<Distance>(Matrix<typename Distance::ElementType>(dataset,rows,cols), SavedIndexParams(filename), d);
return index;
''')
py_source = ut.codeblock('''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
self.__curindex_data = None
self.__added_data = []
self.__curindex_type = None
self.__curindex = flann.load_index[pts.dtype.type](
c_char_p(to_bytes(filename)), pts, npts, dim)
if self.__curindex is None:
raise FLANNException(
('Error loading the FLANN index with filename=%r.'
' C++ may have thrown more detailed errors') % (filename,))
self.__curindex_data = pts
self.__added_data = []
self.__removed_ids = []
self.__curindex_type = pts.dtype.type
''')
docstr = 'Loads a previously saved index from a file.'
return_doc = 'index_ptr'
cpp_param_doc['dataset'] = 'The dataset corresponding to the index'
cpp_param_doc['filename'] = 'File to load the index from'
py_args = ['filename', 'pts']
return_type = 'flann_index_t'
binding_argnames = ['filename', 'dataset', 'rows', 'cols']
optional_args = ['Distance d = Distance()']
elif binding_name == 'save_index':
docstr = 'Saves the index to a file. Only the index is saved into the file, the dataset corresponding to the index is not saved.'
cpp_param_doc['index_ptr'] = 'The index that should be saved'
cpp_param_doc['filename'] = 'The filename the index should be saved to'
return_doc = 'Returns 0 on success, negative value on error'
c_source_part = ut.codeblock(r'''
Index<Distance>* index = (Index<Distance>*)index_ptr;
index->save(filename);
return 0;
''')
py_source = ut.codeblock('''
if self.__curindex is not None:
flann.save_index[self.__curindex_type](
self.__curindex, c_char_p(to_bytes(filename)))
''')
return_type = 'int'
binding_argnames = ['index_ptr', 'filename']
py_alias = None
py_args = None
elif binding_name == 'build_index':
docstr = ut.codeblock('''
Builds and returns an index. It uses autotuning if the target_precision field of index_params
is between 0 and 1, or the parameters specified if it's -1.
''')
pydoc = ut.codeblock('''
This builds and internally stores an index to be used for
future nearest neighbor matchings. It erases any previously
stored indexes, so use multiple instances of this class to
work with multiple stored indices. Use nn_index(...) to find
the nearest neighbors in this index.
pts is a 2d numpy array or matrix. All the computation is done
in np.float32 type, but pts may be any type that is convertable
to np.float32.
''')
c_source = ut.codeblock(r'''
typedef typename Distance::ElementType ElementType;
try {
init_flann_parameters(flann_params);
if (flann_params == NULL) {
throw FLANNException("The flann_params argument must be non-null");
}
IndexParams params = create_parameters(flann_params);
Index<Distance>* index = new Index<Distance>(Matrix<ElementType>(dataset,rows,cols), params, d);
index->buildIndex();
if (flann_params->algorithm==FLANN_INDEX_AUTOTUNED) {
IndexParams params = index->getParameters();
update_flann_parameters(params,flann_params);
SearchParams search_params = get_param<SearchParams>(params,"search_params");
*speedup = get_param<float>(params,"speedup");
flann_params->checks = search_params.checks;
flann_params->eps = search_params.eps;
flann_params->cb_index = get_param<float>(params,"cb_index",0.0);
}
return index;
}
catch (std::runtime_error& e) {
Logger::error("Caught exception: %s\n",e.what());
return NULL;
}
''').replace('{', '{{').replace('}', '}}')
py_source = ut.codeblock('''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
npts, dim = pts.shape
self.__ensureRandomSeed(kwargs)
self.__flann_parameters.update(kwargs)
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
speedup = c_float(0)
self.__curindex = flann.build_index[pts.dtype.type](
pts, npts, dim, byref(speedup), pointer(self.__flann_parameters))
self.__curindex_data = pts
self.__curindex_type = pts.dtype.type
params = dict(self.__flann_parameters)
params['speedup'] = speedup.value
return params
''')
# binding_argnames = ['dataset', 'rows', 'cols', 'speedup', 'flann_params']
return_doc = 'the newly created index or a number <0 for error'
cpp_param_doc[
'speedup'] = 'speedup over linear search, estimated if using autotuning, output parameter'
optional_args = ['Distance d = Distance()']
return_type = 'flann_index_t'
py_args = ['pts', '**kwargs']
binding_argnames = [
'dataset', 'rows', 'cols', 'speedup', 'flann_params'
]
elif binding_name == 'free_index':
docstr = 'Deletes an index and releases the memory used by it.'
pydoc = ut.codeblock('''
Deletes the current index freeing all the momory it uses.
The memory used by the dataset that was indexed is not freed
unless there are no other references to those numpy arrays.
''')
c_source_part = ut.codeblock(r'''
Index<Distance>* index = (Index<Distance>*)index_ptr;
delete index;
return 0;
''')
py_source = ut.codeblock('''
self.__flann_parameters.update(kwargs)
if self.__curindex is not None:
flann.free_index[self.__curindex_type](
self.__curindex, pointer(self.__flann_parameters))
self.__curindex = None
self.__curindex_data = None
self.__added_data = []
self.__removed_ids = []
''')
return_doc = zero_success
return_type = 'int'
binding_argnames = ['index_ptr', 'flann_params']
cpp_param_doc['flann_params'] = ut.textblock(
'''generic flann params (only used to specify verbosity)''')
py_alias = 'delete_index'
py_args = ['**kwargs']
elif binding_name == 'get_point':
docstr = 'Gets a point from a given index position.'
return_doc = 'pointer to datapoint or NULL on miss'
binding_argnames = ['index_ptr', 'point_id']
cpp_param_doc['point_id'] = 'index of datapoint to get.'
return_type = 'Distance::ElementType*'
elif binding_name == 'flann_get_distance_order':
docstr = ut.textblock(
'''Gets the distance order in use throughout FLANN (only applicable
if minkowski distance is in use).''')
binding_argnames = []
return_type = 'int'
else:
dictdef = {
'_template_new': {
'docstr': '',
'binding_argnames': [],
'return_type': 'int',
},
'flann_get_distance_type': {
'docstr': '',
'binding_argnames': [],
'return_type': 'int',
},
'flann_log_verbosity': {
'docstr':
ut.codeblock('''
Sets the log level used for all flann functions (unless
specified in FLANNParameters for each call
'''),
'binding_argnames': ['level'],
'return_type':
'void',
},
}
if binding_name in dictdef:
docstr = dictdef[binding_name].get('docstr', '')
binding_argnames = dictdef[binding_name]['binding_argnames']
return_type = dictdef[binding_name]['return_type']
else:
raise NotImplementedError('Unknown binding name %r' %
(binding_name, ))
if c_source is None:
if c_source_part is not None:
try_ = ut.codeblock('''
try {{
''')
throw_ = '\n' + ut.indent(
ut.codeblock('''
if (index_ptr==NULL) {{
throw FLANNException("Invalid index");
}}
'''), ' ' * 4)
if 'index_ptr' not in binding_argnames:
throw_ = ''
if 'flann_params' in binding_argnames:
part1 = try_ + '\n' + ' init_flann_parameters(flann_params);' + throw_
else:
part1 = try_ + throw_
if return_type == 'int':
default_return = '-1'
else:
default_return = 'NULL'
part2 = ut.codeblock(r'''
}}
catch (std::runtime_error& e) {{
Logger::error("Caught exception: %s\n",e.what());
return ''' + default_return + ''';
}}
''')
c_source = part1 + '\n' + ut.indent(c_source_part,
' ' * 4) + '\n' + part2
else:
c_source = ut.codeblock('''
TODO: IMPLEMENT THIS FUNCTION WRAPPER
''')
try:
docstr_cpp = docstr[:]
if return_doc is not None:
param_docs = ut.dict_take(cpp_param_doc, binding_argnames)
cpp_param_docblock = '\n'.join([
'%s = %s' % (name, doc)
for name, doc in zip(binding_argnames, param_docs)
])
docstr_cpp += '\n\n' + 'Params:\n' + ut.indent(
cpp_param_docblock, ' ')
docstr_cpp += '\n\n' + 'Returns: ' + return_doc
if pydoc is None:
docstr_py = docstr[:]
else:
docstr_py = pydoc[:]
if py_args:
py_param_doc = cpp_param_doc.copy()
py_param_doc['pts'] = py_param_doc['dataset'].replace(
'pointer to ', '')
py_param_doc['qpts'] = (
py_param_doc['testset'].replace('pointer to ', '') +
' (may be a single point)')
py_param_doc['num_neighbors'] = py_param_doc['nn']
py_param_doc['**kwargs'] = py_param_doc['flann_params']
py_args_ = [a.split('=')[0] for a in py_args]
param_docs = ut.dict_take(py_param_doc, py_args_, '')
# py_types =
py_param_docblock = '\n'.join([
'%s: %s' % (name, doc)
for name, doc in zip(py_args_, param_docs)
])
docstr_py += '\n\n' + 'Params:\n' + ut.indent(
py_param_docblock, ' ')
except Exception as ex:
ut.printex(ex, keys=['binding_name'])
raise
pass
binding_def = {
'cpp_binding_name': cpp_binding_name,
'docstr_cpp': docstr_cpp,
'docstr_py': docstr_py,
'return_type': return_type,
'binding_argnames': binding_argnames,
'c_source': c_source,
'optional_args': optional_args,
'py_source': py_source,
'py_args': py_args,
'py_alias': py_alias,
}
return binding_def
def make_default_docstr(func,
with_args=True,
with_ret=True,
with_commandline=True,
with_example=True,
with_header=False,
with_debug=False,
):
r"""
Tries to make a sensible default docstr so the user
can fill things in without typing too much
# TODO: Interleave old documentation with new documentation
Args:
func (function): live python function
with_args (bool):
with_ret (bool): (default = True)
with_commandline (bool): (default = True)
with_example (bool): (default = True)
with_header (bool): (default = False)
with_debug (bool): (default = False)
Returns:
tuple: (argname, val)
Ignore:
pass
CommandLine:
python -m utool.util_autogen --exec-make_default_docstr --show
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_autogen import * # NOQA
>>> import utool as ut
>>> func = ut.make_default_docstr
>>> #func = ut.make_args_docstr
>>> func = PythonStatement
>>> default_docstr = make_default_docstr(func)
>>> result = str(default_docstr)
>>> print(result)
"""
import utool as ut
#from utool import util_inspect
funcinfo = ut.util_inspect.infer_function_info(func)
argname_list = funcinfo.argname_list
argtype_list = funcinfo.argtype_list
argdesc_list = funcinfo.argdesc_list
return_header = funcinfo.return_header
return_type = funcinfo.return_type
return_name = funcinfo.return_name
return_desc = funcinfo.return_desc
funcname = funcinfo.funcname
modname = funcinfo.modname
defaults = funcinfo.defaults
num_indent = funcinfo.num_indent
needs_surround = funcinfo.needs_surround
funcname = funcinfo.funcname
ismethod = funcinfo.ismethod
kwarg_keys = funcinfo.kwarg_keys
docstr_parts = []
# Header part
if with_header:
header_block = funcname
docstr_parts.append(header_block)
# Args part
if with_args and len(argname_list) > 0:
argheader = 'Args'
arg_docstr = make_args_docstr(argname_list, argtype_list, argdesc_list, ismethod)
argsblock = make_docstr_block(argheader, arg_docstr)
docstr_parts.append(argsblock)
with_kw = with_args
if with_kw and len(kwarg_keys) > 0:
#ut.embed()
import textwrap
kwargs_docstr = ', '.join(kwarg_keys)
kwargs_docstr = '\n'.join(textwrap.wrap(kwargs_docstr))
kwargsblock = make_docstr_block('Kwargs', kwargs_docstr)
docstr_parts.append(kwargsblock)
# Return / Yeild part
if with_ret and return_header is not None:
if return_header is not None:
return_doctr = make_returns_or_yeilds_docstr(return_type, return_name, return_desc)
returnblock = make_docstr_block(return_header, return_doctr)
docstr_parts.append(returnblock)
# Example part
# try to generate a simple and unit testable example
if with_commandline:
cmdlineheader = 'CommandLine'
cmdlinecode = make_cmdline_docstr(funcname, modname)
cmdlineblock = make_docstr_block(cmdlineheader, cmdlinecode)
docstr_parts.append(cmdlineblock)
if with_example:
exampleheader = 'Example'
examplecode = make_example_docstr(funcname, modname, argname_list,
defaults, return_type, return_name,
ismethod)
examplecode_ = ut.indent(examplecode, '>>> ')
exampleblock = make_docstr_block(exampleheader, examplecode_)
docstr_parts.append(exampleblock)
# DEBUG part (in case something goes wrong)
if with_debug:
debugheader = 'Debug'
debugblock = ut.codeblock(
'''
num_indent = {num_indent}
'''
).format(num_indent=num_indent)
debugblock = make_docstr_block(debugheader, debugblock)
docstr_parts.append(debugblock)
# Enclosure / Indentation Parts
if needs_surround:
docstr_parts = ['r"""'] + ['\n\n'.join(docstr_parts)] + ['"""']
default_docstr = '\n'.join(docstr_parts)
else:
default_docstr = '\n\n'.join(docstr_parts)
docstr_indent = ' ' * (num_indent + 4)
default_docstr = ut.indent(default_docstr, docstr_indent)
return default_docstr
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = [
'ibeis/tests/assert_modules.py'
]
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/utool'
exclude_list = []
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
implicit_build_modlist_str = ut.codeblock(
'''
import sys
exclude_doctests_fnames = set(['__init__.py'])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['utool']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
'''
)
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list])
modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(
r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
'''
)
pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
def def_test(header, pat=None, dpath=None, modname=None, default=False, testcmds=None):
""" interface to make test tuple """
return (header, default, modname, dpath, pat, testcmds)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST')
test_headers = [
# title, default, module, testpattern
def_test('DOC', testcmds=testcmds, default=True)
]
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list)
return shscript_text, pyscript_text
def get_dbinfo(
ibs,
verbose=True,
with_imgsize=False,
with_bytes=False,
with_contrib=False,
with_agesex=False,
with_header=True,
short=False,
tag='dbinfo',
aid_list=None,
aids=None,
):
"""
Returns dictionary of digestable database information
Infostr is a string summary of all the stats. Prints infostr in addition to
returning locals
Args:
ibs (IBEISController):
verbose (bool):
with_imgsize (bool):
with_bytes (bool):
Returns:
dict:
SeeAlso:
python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --use-hist=True --old=False --per_name_vpedge=False
python -m wbia.other.ibsfuncs --exec-get_annot_stats_dict --db PZ_PB_RF_TRAIN --all
CommandLine:
python -m wbia.other.dbinfo --exec-get_dbinfo:0
python -m wbia.other.dbinfo --test-get_dbinfo:1
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db NNP_Master3
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db PZ_Master1
python -m wbia.other.dbinfo --test-get_dbinfo:0 --db GZ_ALL
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db PZ_ViewPoints
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db GZ_Master1
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db LF_Bajo_bonito -a default
python -m wbia.other.dbinfo --exec-get_dbinfo:0 --db DETECT_SEATURTLES -a default --readonly
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a ctrl
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default:minqual=ok,require_timestamp=True --dbdir ~/lev/media/danger/LEWA --loadbackup=0
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA
python -m wbia.other.dbinfo --exec-get_dbinfo:0 -a default: --dbdir ~/lev/media/danger/LEWA --loadbackup=0
Example1:
>>> # SCRIPT
>>> from wbia.other.dbinfo import * # NOQA
>>> import wbia
>>> defaultdb = 'testdb1'
>>> ibs, aid_list = wbia.testdata_aids(defaultdb, a='default:minqual=ok,view=primary,view_ext1=1')
>>> kwargs = ut.get_kwdefaults(get_dbinfo)
>>> kwargs['verbose'] = False
>>> kwargs['aid_list'] = aid_list
>>> kwargs = ut.parse_dict_from_argv(kwargs)
>>> output = get_dbinfo(ibs, **kwargs)
>>> result = (output['info_str'])
>>> print(result)
>>> #ibs = wbia.opendb(defaultdb='testdb1')
>>> # <HACK FOR FILTERING>
>>> #from wbia.expt import cfghelpers
>>> #from wbia.expt import annotation_configs
>>> #from wbia.init import filter_annots
>>> #named_defaults_dict = ut.dict_take(annotation_configs.__dict__,
>>> # annotation_configs.TEST_NAMES)
>>> #named_qcfg_defaults = dict(zip(annotation_configs.TEST_NAMES,
>>> # ut.get_list_column(named_defaults_dict, 'qcfg')))
>>> #acfg = cfghelpers.parse_argv_cfg(('--annot-filter', '-a'), named_defaults_dict=named_qcfg_defaults, default=None)[0]
>>> #aid_list = ibs.get_valid_aids()
>>> # </HACK FOR FILTERING>
Example1:
>>> # ENABLE_DOCTEST
>>> from wbia.other.dbinfo import * # NOQA
>>> import wbia
>>> verbose = True
>>> short = True
>>> #ibs = wbia.opendb(db='GZ_ALL')
>>> #ibs = wbia.opendb(db='PZ_Master0')
>>> ibs = wbia.opendb('testdb1')
>>> assert ibs.get_dbname() == 'testdb1', 'DO NOT DELETE CONTRIBUTORS OF OTHER DBS'
>>> ibs.delete_contributors(ibs.get_valid_contributor_rowids())
>>> ibs.delete_empty_nids()
>>> #ibs = wbia.opendb(db='PZ_MTEST')
>>> output = get_dbinfo(ibs, with_contrib=False, verbose=False, short=True)
>>> result = (output['info_str'])
>>> print(result)
+============================
DB Info: testdb1
DB Notes: None
DB NumContrib: 0
----------
# Names = 7
# Names (unassociated) = 0
# Names (singleton) = 5
# Names (multiton) = 2
----------
# Annots = 13
# Annots (unknown) = 4
# Annots (singleton) = 5
# Annots (multiton) = 4
----------
# Img = 13
L============================
"""
# TODO Database size in bytes
# TODO: occurrence, contributors, etc...
if aids is not None:
aid_list = aids
# Basic variables
request_annot_subset = False
_input_aid_list = aid_list # NOQA
if aid_list is None:
valid_aids = ibs.get_valid_aids()
valid_nids = ibs.get_valid_nids()
valid_gids = ibs.get_valid_gids()
else:
if isinstance(aid_list, str):
# Hack to get experiment stats on aids
acfg_name_list = [aid_list]
logger.info('Specified custom aids via acfgname %s' % (acfg_name_list,))
from wbia.expt import experiment_helpers
acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
ibs, acfg_name_list
)
aid_list = sorted(list(set(ut.flatten(ut.flatten(expanded_aids_list)))))
# aid_list =
if verbose:
logger.info('Specified %d custom aids' % (len(aid_list,)))
request_annot_subset = True
valid_aids = aid_list
valid_nids = list(
set(ibs.get_annot_nids(aid_list, distinguish_unknowns=False))
- {const.UNKNOWN_NAME_ROWID}
)
valid_gids = list(set(ibs.get_annot_gids(aid_list)))
# associated_nids = ibs.get_valid_nids(filter_empty=True) # nids with at least one annotation
valid_images = ibs.images(valid_gids)
valid_annots = ibs.annots(valid_aids)
# Image info
if verbose:
logger.info('Checking Image Info')
gx2_aids = valid_images.aids
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
gx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in gx2_aids]
gx2_nAnnots = np.array(list(map(len, gx2_aids)))
image_without_annots = len(np.where(gx2_nAnnots == 0)[0])
gx2_nAnnots_stats = ut.repr4(
ut.get_stats(gx2_nAnnots, use_median=True), nl=0, precision=2, si=True
)
image_reviewed_list = ibs.get_image_reviewed(valid_gids)
# Name stats
if verbose:
logger.info('Checking Name Info')
nx2_aids = ibs.get_name_aids(valid_nids)
if request_annot_subset:
# remove annots not in this subset
valid_aids_set = set(valid_aids)
nx2_aids = [list(set(aids_).intersection(valid_aids_set)) for aids_ in nx2_aids]
associated_nids = ut.compress(valid_nids, list(map(len, nx2_aids)))
ibs.check_name_mapping_consistency(nx2_aids)
if False:
# Occurrence Info
def compute_annot_occurrence_ids(ibs, aid_list):
from wbia.algo.preproc import preproc_occurrence
gid_list = ibs.get_annot_gids(aid_list)
gid2_aids = ut.group_items(aid_list, gid_list)
config = {'seconds_thresh': 4 * 60 * 60}
flat_imgsetids, flat_gids = preproc_occurrence.wbia_compute_occurrences(
ibs, gid_list, config=config, verbose=False
)
occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
occurid2_aids = {
oid: ut.flatten(ut.take(gid2_aids, gids))
for oid, gids in occurid2_gids.items()
}
return occurid2_aids
import utool
with utool.embed_on_exception_context:
occurid2_aids = compute_annot_occurrence_ids(ibs, valid_aids)
occur_nids = ibs.unflat_map(ibs.get_annot_nids, occurid2_aids.values())
occur_unique_nids = [ut.unique(nids) for nids in occur_nids]
nid2_occurxs = ut.ddict(list)
for occurx, nids in enumerate(occur_unique_nids):
for nid in nids:
nid2_occurxs[nid].append(occurx)
nid2_occurx_single = {
nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) <= 1
}
nid2_occurx_resight = {
nid: occurxs for nid, occurxs in nid2_occurxs.items() if len(occurxs) > 1
}
singlesight_encounters = ibs.get_name_aids(nid2_occurx_single.keys())
singlesight_annot_stats = ut.get_stats(
list(map(len, singlesight_encounters)), use_median=True, use_sum=True
)
resight_name_stats = ut.get_stats(
list(map(len, nid2_occurx_resight.values())), use_median=True, use_sum=True
)
# Encounter Info
def break_annots_into_encounters(aids):
from wbia.algo.preproc import occurrence_blackbox
import datetime
thresh_sec = datetime.timedelta(minutes=30).seconds
posixtimes = np.array(ibs.get_annot_image_unixtimes_asfloat(aids))
# latlons = ibs.get_annot_image_gps(aids)
labels = occurrence_blackbox.cluster_timespace2(
posixtimes, None, thresh_sec=thresh_sec
)
return labels
# ave_enc_time = [np.mean(times) for lbl, times in ut.group_items(posixtimes, labels).items()]
# ut.square_pdist(ave_enc_time)
try:
am_rowids = ibs.get_annotmatch_rowids_between_groups([valid_aids], [valid_aids])[
0
]
aid_pairs = ibs.filter_aidpairs_by_tags(min_num=0, am_rowids=am_rowids)
undirected_tags = ibs.get_aidpair_tags(
aid_pairs.T[0], aid_pairs.T[1], directed=False
)
tagged_pairs = list(zip(aid_pairs.tolist(), undirected_tags))
tag_dict = ut.groupby_tags(tagged_pairs, undirected_tags)
pair_tag_info = ut.map_dict_vals(len, tag_dict)
except Exception:
pair_tag_info = {}
# logger.info(ut.repr2(pair_tag_info))
# Annot Stats
# TODO: number of images where chips cover entire image
# TODO: total image coverage of annotation
# TODO: total annotation overlap
"""
ax2_unknown = ibs.is_aid_unknown(valid_aids)
ax2_nid = ibs.get_annot_name_rowids(valid_aids)
assert all([nid < 0 if unknown else nid > 0 for nid, unknown in
zip(ax2_nid, ax2_unknown)]), 'bad annot nid'
"""
#
if verbose:
logger.info('Checking Annot Species')
unknown_annots = valid_annots.compress(ibs.is_aid_unknown(valid_annots))
species_list = valid_annots.species_texts
species2_annots = valid_annots.group_items(valid_annots.species_texts)
species2_nAids = {key: len(val) for key, val in species2_annots.items()}
if verbose:
logger.info('Checking Multiton/Singleton Species')
nx2_nAnnots = np.array(list(map(len, nx2_aids)))
# Seperate singleton / multitons
multiton_nxs = np.where(nx2_nAnnots > 1)[0]
singleton_nxs = np.where(nx2_nAnnots == 1)[0]
unassociated_nxs = np.where(nx2_nAnnots == 0)[0]
assert len(np.intersect1d(singleton_nxs, multiton_nxs)) == 0, 'intersecting names'
valid_nxs = np.hstack([multiton_nxs, singleton_nxs])
num_names_with_gt = len(multiton_nxs)
# Annot Info
if verbose:
logger.info('Checking Annot Info')
multiton_aids_list = ut.take(nx2_aids, multiton_nxs)
assert len(set(multiton_nxs)) == len(multiton_nxs)
if len(multiton_aids_list) == 0:
multiton_aids = np.array([], dtype=np.int)
else:
multiton_aids = np.hstack(multiton_aids_list)
assert len(set(multiton_aids)) == len(multiton_aids), 'duplicate annot'
singleton_aids = ut.take(nx2_aids, singleton_nxs)
multiton_nid2_nannots = list(map(len, multiton_aids_list))
# Image size stats
if with_imgsize:
if verbose:
logger.info('Checking ImageSize Info')
gpath_list = ibs.get_image_paths(valid_gids)
def wh_print_stats(wh_list):
if len(wh_list) == 0:
return '{empty}'
wh_list = np.asarray(wh_list)
stat_dict = collections.OrderedDict(
[
('max', wh_list.max(0)),
('min', wh_list.min(0)),
('mean', wh_list.mean(0)),
('std', wh_list.std(0)),
]
)
def arr2str(var):
return '[' + (', '.join(list(map(lambda x: '%.1f' % x, var)))) + ']'
ret = ',\n '.join(
['%s:%s' % (key, arr2str(val)) for key, val in stat_dict.items()]
)
return '{\n ' + ret + '\n}'
logger.info('reading image sizes')
# Image size stats
img_size_list = ibs.get_image_sizes(valid_gids)
img_size_stats = wh_print_stats(img_size_list)
# Chip size stats
annotation_bbox_list = ibs.get_annot_bboxes(valid_aids)
annotation_bbox_arr = np.array(annotation_bbox_list)
if len(annotation_bbox_arr) == 0:
annotation_size_list = []
else:
annotation_size_list = annotation_bbox_arr[:, 2:4]
chip_size_stats = wh_print_stats(annotation_size_list)
imgsize_stat_lines = [
(' # Img in dir = %d' % len(gpath_list)),
(' Image Size Stats = %s' % (img_size_stats,)),
(' * Chip Size Stats = %s' % (chip_size_stats,)),
]
else:
imgsize_stat_lines = []
if verbose:
logger.info('Building Stats String')
multiton_stats = ut.repr3(
ut.get_stats(multiton_nid2_nannots, use_median=True), nl=0, precision=2, si=True
)
# Time stats
unixtime_list = valid_images.unixtime2
# valid_unixtime_list = [time for time in unixtime_list if time != -1]
# unixtime_statstr = ibs.get_image_time_statstr(valid_gids)
if ut.get_argflag('--hackshow-unixtime'):
show_time_distributions(ibs, unixtime_list)
ut.show_if_requested()
unixtime_statstr = ut.repr3(ut.get_timestats_dict(unixtime_list, full=True), si=True)
# GPS stats
gps_list_ = ibs.get_image_gps(valid_gids)
gpsvalid_list = [gps != (-1, -1) for gps in gps_list_]
gps_list = ut.compress(gps_list_, gpsvalid_list)
def get_annot_age_stats(aid_list):
annot_age_months_est_min = ibs.get_annot_age_months_est_min(aid_list)
annot_age_months_est_max = ibs.get_annot_age_months_est_max(aid_list)
age_dict = ut.ddict((lambda: 0))
for min_age, max_age in zip(annot_age_months_est_min, annot_age_months_est_max):
if max_age is None:
max_age = min_age
if min_age is None:
min_age = max_age
if max_age is None and min_age is None:
logger.info('Found UNKNOWN Age: %r, %r' % (min_age, max_age,))
age_dict['UNKNOWN'] += 1
elif (min_age is None or min_age < 12) and max_age < 12:
age_dict['Infant'] += 1
elif 12 <= min_age and min_age < 36 and 12 <= max_age and max_age < 36:
age_dict['Juvenile'] += 1
elif 36 <= min_age and (max_age is None or 36 <= max_age):
age_dict['Adult'] += 1
return age_dict
def get_annot_sex_stats(aid_list):
annot_sextext_list = ibs.get_annot_sex_texts(aid_list)
sextext2_aids = ut.group_items(aid_list, annot_sextext_list)
sex_keys = list(ibs.const.SEX_TEXT_TO_INT.keys())
assert set(sex_keys) >= set(annot_sextext_list), 'bad keys: ' + str(
set(annot_sextext_list) - set(sex_keys)
)
sextext2_nAnnots = ut.odict(
[(key, len(sextext2_aids.get(key, []))) for key in sex_keys]
)
# Filter 0's
sextext2_nAnnots = {
key: val for key, val in six.iteritems(sextext2_nAnnots) if val != 0
}
return sextext2_nAnnots
def get_annot_qual_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
qualtext2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.quality_texts)),
list(ibs.const.QUALITY_TEXT_TO_INT.keys()),
)
return qualtext2_nAnnots
def get_annot_viewpoint_stats(ibs, aid_list):
annots = ibs.annots(aid_list)
viewcode2_nAnnots = ut.order_dict_by(
ut.map_vals(len, annots.group_items(annots.viewpoint_code)),
list(ibs.const.VIEW.CODE_TO_INT.keys()) + [None],
)
return viewcode2_nAnnots
if verbose:
logger.info('Checking Other Annot Stats')
qualtext2_nAnnots = get_annot_qual_stats(ibs, valid_aids)
viewcode2_nAnnots = get_annot_viewpoint_stats(ibs, valid_aids)
agetext2_nAnnots = get_annot_age_stats(valid_aids)
sextext2_nAnnots = get_annot_sex_stats(valid_aids)
if verbose:
logger.info('Checking Contrib Stats')
# Contributor Statistics
# hack remove colon for image alignment
def fix_tag_list(tag_list):
return [None if tag is None else tag.replace(':', ';') for tag in tag_list]
image_contributor_tags = fix_tag_list(ibs.get_image_contributor_tag(valid_gids))
annot_contributor_tags = fix_tag_list(ibs.get_annot_image_contributor_tag(valid_aids))
contributor_tag_to_gids = ut.group_items(valid_gids, image_contributor_tags)
contributor_tag_to_aids = ut.group_items(valid_aids, annot_contributor_tags)
contributor_tag_to_qualstats = {
key: get_annot_qual_stats(ibs, aids)
for key, aids in six.iteritems(contributor_tag_to_aids)
}
contributor_tag_to_viewstats = {
key: get_annot_viewpoint_stats(ibs, aids)
for key, aids in six.iteritems(contributor_tag_to_aids)
}
contributor_tag_to_nImages = {
key: len(val) for key, val in six.iteritems(contributor_tag_to_gids)
}
contributor_tag_to_nAnnots = {
key: len(val) for key, val in six.iteritems(contributor_tag_to_aids)
}
if verbose:
logger.info('Summarizing')
# Summarize stats
num_names = len(valid_nids)
num_names_unassociated = len(valid_nids) - len(associated_nids)
num_names_singleton = len(singleton_nxs)
num_names_multiton = len(multiton_nxs)
num_singleton_annots = len(singleton_aids)
num_multiton_annots = len(multiton_aids)
num_unknown_annots = len(unknown_annots)
num_annots = len(valid_aids)
if with_bytes:
if verbose:
logger.info('Checking Disk Space')
ibsdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_ibsdir()))
dbdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_dbdir()))
imgdir_space = ut.byte_str2(ut.get_disk_space(ibs.get_imgdir()))
cachedir_space = ut.byte_str2(ut.get_disk_space(ibs.get_cachedir()))
if True:
if verbose:
logger.info('Check asserts')
try:
bad_aids = np.intersect1d(multiton_aids, unknown_annots)
_num_names_total_check = (
num_names_singleton + num_names_unassociated + num_names_multiton
)
_num_annots_total_check = (
num_unknown_annots + num_singleton_annots + num_multiton_annots
)
assert len(bad_aids) == 0, 'intersecting multiton aids and unknown aids'
assert _num_names_total_check == num_names, 'inconsistent num names'
# if not request_annot_subset:
# dont check this if you have an annot subset
assert _num_annots_total_check == num_annots, 'inconsistent num annots'
except Exception as ex:
ut.printex(
ex,
keys=[
'_num_names_total_check',
'num_names',
'_num_annots_total_check',
'num_annots',
'num_names_singleton',
'num_names_multiton',
'num_unknown_annots',
'num_multiton_annots',
'num_singleton_annots',
],
)
raise
# Get contributor statistics
contributor_rowids = ibs.get_valid_contributor_rowids()
num_contributors = len(contributor_rowids)
# print
num_tabs = 5
def align2(str_):
return ut.align(str_, ':', ' :')
def align_dict2(dict_):
str_ = ut.repr2(dict_, si=True)
return align2(str_)
header_block_lines = [('+============================')] + (
[
('+ singleton := single sighting'),
('+ multiton := multiple sightings'),
('--' * num_tabs),
]
if not short and with_header
else []
)
source_block_lines = [
('DB Info: ' + ibs.get_dbname()),
('DB Notes: ' + ibs.get_dbnotes()),
('DB NumContrib: %d' % num_contributors),
]
bytes_block_lines = (
[
('--' * num_tabs),
('DB Bytes: '),
(' +- dbdir nBytes: ' + dbdir_space),
(' | +- _ibsdb nBytes: ' + ibsdir_space),
(' | | +-imgdir nBytes: ' + imgdir_space),
(' | | +-cachedir nBytes: ' + cachedir_space),
]
if with_bytes
else []
)
name_block_lines = [
('--' * num_tabs),
('# Names = %d' % num_names),
('# Names (unassociated) = %d' % num_names_unassociated),
('# Names (singleton) = %d' % num_names_singleton),
('# Names (multiton) = %d' % num_names_multiton),
]
subset_str = ' ' if not request_annot_subset else '(SUBSET)'
annot_block_lines = [
('--' * num_tabs),
('# Annots %s = %d' % (subset_str, num_annots,)),
('# Annots (unknown) = %d' % num_unknown_annots),
('# Annots (singleton) = %d' % num_singleton_annots),
('# Annots (multiton) = %d' % num_multiton_annots),
]
annot_per_basic_block_lines = (
[
('--' * num_tabs),
('# Annots per Name (multiton) = %s' % (align2(multiton_stats),)),
('# Annots per Image = %s' % (align2(gx2_nAnnots_stats),)),
('# Annots per Species = %s' % (align_dict2(species2_nAids),)),
]
if not short
else []
)
occurrence_block_lines = (
[
('--' * num_tabs),
# ('# Occurrence Per Name (Resights) = %s' % (align_dict2(resight_name_stats),)),
# ('# Annots per Encounter (Singlesights) = %s' % (align_dict2(singlesight_annot_stats),)),
('# Pair Tag Info (annots) = %s' % (align_dict2(pair_tag_info),)),
]
if not short
else []
)
annot_per_qualview_block_lines = [
None if short else '# Annots per Viewpoint = %s' % align_dict2(viewcode2_nAnnots),
None if short else '# Annots per Quality = %s' % align_dict2(qualtext2_nAnnots),
]
annot_per_agesex_block_lines = (
[
'# Annots per Age = %s' % align_dict2(agetext2_nAnnots),
'# Annots per Sex = %s' % align_dict2(sextext2_nAnnots),
]
if not short and with_agesex
else []
)
contributor_block_lines = (
[
'# Images per contributor = ' + align_dict2(contributor_tag_to_nImages),
'# Annots per contributor = ' + align_dict2(contributor_tag_to_nAnnots),
'# Quality per contributor = '
+ ut.repr2(contributor_tag_to_qualstats, sorted_=True),
'# Viewpoint per contributor = '
+ ut.repr2(contributor_tag_to_viewstats, sorted_=True),
]
if with_contrib
else []
)
img_block_lines = [
('--' * num_tabs),
('# Img = %d' % len(valid_gids)),
None
if short
else ('# Img reviewed = %d' % sum(image_reviewed_list)),
None if short else ('# Img with gps = %d' % len(gps_list)),
# ('# Img with timestamp = %d' % len(valid_unixtime_list)),
None
if short
else ('Img Time Stats = %s' % (align2(unixtime_statstr),)),
]
info_str_lines = (
header_block_lines
+ bytes_block_lines
+ source_block_lines
+ name_block_lines
+ annot_block_lines
+ annot_per_basic_block_lines
+ occurrence_block_lines
+ annot_per_qualview_block_lines
+ annot_per_agesex_block_lines
+ img_block_lines
+ contributor_block_lines
+ imgsize_stat_lines
+ [('L============================')]
)
info_str = '\n'.join(ut.filter_Nones(info_str_lines))
info_str2 = ut.indent(info_str, '[{tag}]'.format(tag=tag))
if verbose:
logger.info(info_str2)
locals_ = locals()
return locals_
def autogen_parts(binding_name=None):
r"""
CommandLine:
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=set_dataset
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=add_points
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_point --py
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=used_memory --py
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=remove_points --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=veclen --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=size --py --c
python ~/local/build_scripts/flannscripts/autogen_bindings.py --exec-autogen_parts --bindingname=clean_removed_points --py --c
Ignore:
# Logic goes here
~/code/flann/src/cpp/flann/algorithms/kdtree_index.h
~/code/flann/src/cpp/flann/util/serialization.h
~/code/flann/src/cpp/flann/util/dynamic_bitset.h
# Bindings go here
~/code/flann/src/cpp/flann/flann.cpp
~/code/flann/src/cpp/flann/flann.h
# Contains stuff for the flann namespace like flann::log_level
# Also has Index with
# Matrix<ElementType> features; SEEMS USEFUL
~/code/flann/src/cpp/flann/flann.hpp
# Wrappers go here
~/code/flann/src/python/pyflann/flann_ctypes.py
~/code/flann/src/python/pyflann/index.py
~/local/build_scripts/flannscripts/autogen_bindings.py
Example:
>>> # ENABLE_DOCTEST
>>> from autogen_bindings import * # NOQA
>>> result = autogen_parts()
>>> print(result)
"""
#ut.get_dynlib_exports(pyflann.flannlib._name)
#flannlib
if binding_name is None:
binding_name = ut.get_argval('--bindingname',
type_=str,
default='add_points')
# Variable names used in flann cpp source
simple_c_to_ctypes = {
'void':
None,
'char*':
'c_char_p',
'unsigned int':
'c_uint',
'int':
'c_int',
'float':
'c_float',
'float*':
'POINTER(c_float)',
'flann_index_t':
'FLANN_INDEX',
'FLANNParameters*':
'POINTER(FLANNParameters)',
'Distance::ResultType*':
"ndpointer(%(restype)s, flags='aligned, c_contiguous, writeable')",
'Distance::ElementType*':
"ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')",
'typename Distance::ElementType*':
"ndpointer(%(numpy)s, ndim=2, flags='aligned, c_contiguous')",
'typename Distance::ResultType*':
"ndpointer(%(restype), ndim=2, flags='aligned, c_contiguous, writeable')",
}
templated_ctype_map = {
'filename': 'char*',
'level': 'int',
'rows': 'int',
'cols': 'int',
'point_id': 'unsigned int',
'num': 'int',
'max_nn': 'int',
'tcount': 'int',
'nn': 'int',
'radius': 'float',
'clusters': 'int',
'rebuild_threshold': 'float',
'index_ptr': 'flann_index_t',
'flann_params': 'FLANNParameters*',
'speedup': 'float*',
'id_list': 'int*',
#'indices' : 'int*',
'dataset': 'typename Distance::ElementType*',
'points': 'typename Distance::ElementType*',
'query': 'typename Distance::ElementType*',
'query1d': 'typename Distance::ElementType*',
'testset': 'typename Distance::ElementType*',
'dists': 'typename Distance::ResultType*',
'dists1d': 'typename Distance::ResultType*',
'result_centers': 'typename Distance::ResultType*',
'result_ids': 'int*',
}
# Python ctype bindings
python_ctype_map = {
'flann_params':
'POINTER(FLANNParameters)',
'id_list':
"ndpointer(int32, ndim=1, flags='aligned, c_contiguous')",
#'indices' : "ndpointer(int32, ndim=1, flags='aligned, c_contiguous, writeable')",
'query1d':
"ndpointer(%(numpy)s, ndim=1, flags='aligned, c_contiguous')",
'dists1d':
"ndpointer(%(restype), ndim=1, flags='aligned, c_contiguous, writeable')",
'result_ids':
"ndpointer(int32, ndim=2, flags='aligned, c_contiguous, writeable')",
#'query' : "ndpointer(float64, ndim=1, flags='aligned, c_contiguous')",
}
for key, val in templated_ctype_map.items():
if key not in python_ctype_map:
python_ctype_map[key] = simple_c_to_ctypes[val]
binding_def = define_flann_bindings(binding_name)
docstr_cpp = binding_def['docstr_cpp']
docstr_py = binding_def['docstr_py']
cpp_binding_name = binding_def['cpp_binding_name']
return_type = binding_def['return_type']
binding_argnames = binding_def['binding_argnames']
c_source = binding_def['c_source']
py_source = binding_def['py_source']
optional_args = binding_def['optional_args']
py_alias = binding_def['py_alias']
py_args = binding_def['py_args']
binding_args = [
python_ctype_map[name] + ', # ' + name for name in binding_argnames
]
binding_args_str = ' ' + '\n '.join(binding_args)
callargs = ', '.join(binding_argnames)
pycallargs = ', '.join(
[name for name in ['self'] + binding_argnames if name != 'index_ptr'])
if py_args is None:
py_args = binding_argnames
pyinputargs = pycallargs # FIXME
else:
pyinputargs = ', '.join(['self'] + py_args)
pyrestype = simple_c_to_ctypes[return_type]
if py_alias is None:
py_binding_name = binding_name
else:
py_binding_name = py_alias
binding_def['py_binding_name'] = py_binding_name
#### flann_ctypes.py
flann_ctypes_codeblock = ut.codeblock('''
flann.{binding_name} = {{}}
define_functions(r"""
flannlib.flann_{binding_name}_%(C)s.restype = {pyrestype}
flannlib.flann_{binding_name}_%(C)s.argtypes = [
{binding_args_str}
]
flann.{binding_name}[%(numpy)s] = flannlib.flann_{binding_name}_%(C)s
""")
''').format(binding_name=binding_name,
binding_args_str=binding_args_str,
pyrestype=pyrestype)
#### index.py
default_py_source_parts = []
if 'pts' in py_args:
default_py_source_parts.append(
ut.codeblock('''
if pts.dtype.type not in allowed_types:
raise FLANNException('Cannot handle type: %s' % pts.dtype)
pts = ensure_2d_array(pts, default_flags)
'''))
default_py_source_parts.append(
ut.codeblock('''
rows = pts.shape[0]
raise NotImplementedError('requires custom implementation')
flann.{binding_name}[self.__curindex_type](self.__curindex, {pycallargs})
'''))
if py_source is None:
py_source = '\n'.join(default_py_source_parts)
flann_index_codeblock = ut.codeblock(r'''
def {py_binding_name}({pyinputargs}):
"""''' + ut.indent('\n' + docstr_py, ' ') + '''
"""''' + '\n' + ut.indent(py_source, ' ') + '''
''').format(binding_name=binding_name,
pycallargs=pycallargs,
py_binding_name=py_binding_name,
pyinputargs=pyinputargs,
py_source=py_source)
#### flann.cpp
#// {binding_name} BEGIN CPP BINDING
#template <typename Distance>
#{return_type} __flann_{binding_name}({templated_args})
flann_cpp_code_fmtstr_ = ut.codeblock(r'''
{{''' + '\n' + ut.indent(c_source, ' ' * (4 * 3)) + r'''
}}
''')
#implicit_type_bindings_fmtstr = ut.codeblock( # NOQA
# '''
# DISTANCE_TYPE_BINDINGS({return_type}, {binding_name},
# SINGLE_ARG({T_typed_sigargs_cpp}),
# SINGLE_ARG({callargs}))
# '''
#)
#type_bindings_fmtstr = implicit_type_bindings_fmtstr
explicit_type_bindings_part3_fmtstr = ut.codeblock(r'''
{{
if (flann_distance_type==FLANN_DIST_EUCLIDEAN) {{
return __flann_{binding_name}<L2<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_MANHATTAN) {{
return __flann_{binding_name}<L1<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_MINKOWSKI) {{
return __flann_{binding_name}<MinkowskiDistance<T> >({callargs}{minkowski_option});
}}
else if (flann_distance_type==FLANN_DIST_HIST_INTERSECT) {{
return __flann_{binding_name}<HistIntersectionDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_HELLINGER) {{
return __flann_{binding_name}<HellingerDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_CHI_SQUARE) {{
return __flann_{binding_name}<ChiSquareDistance<T> >({callargs});
}}
else if (flann_distance_type==FLANN_DIST_KULLBACK_LEIBLER) {{
return __flann_{binding_name}<KL_Divergence<T> >({callargs});
}}
else {{
Logger::error( "Distance type unsupported in the C bindings, use the C++ bindings instead\n");
{errorhandle}
}}
}}
''')
# c binding body
c_bodyblock_fmtstr = ut.codeblock('''
{return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2})
{{
{ifreturns}_flann_{binding_name}{iftemplate}({callargs});
}}
''')
#### flann.h
# c binding header
#c_headersig_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}({T_typed_sigargs2});'
c_headersig_part1_fmtstr = 'FLANN_EXPORT {return_type} flann_{binding_name}{signame_type}('
#### format cpp parts
binding_argtypes = ut.dict_take(templated_ctype_map, binding_argnames)
_fix_T = {
'typename Distance::ElementType*': 'T*',
'typename Distance::ResultType*': 'R*',
}
templated_ctype_map_cpp = {
name: _fix_T.get(type_, type_)
for name, type_ in templated_ctype_map.items()
}
binding_argtypes_cpp = ut.dict_take(templated_ctype_map_cpp,
binding_argnames)
binding_sigargs_cpp = [
type_ + ' ' + name
for type_, name in zip(binding_argtypes_cpp, binding_argnames)
]
templated_bindings = [
templated_ctype_map[name] + ' ' + name for name in binding_argnames
]
if optional_args is not None:
templated_bindings += optional_args
minkowski_option = ', MinkowskiDistance<T>(flann_distance_order)'
else:
minkowski_option = ''
templated_args = ', '.join(templated_bindings)
if binding_name == 'remove_point':
# HACK
templated_args += '_uint'
cpp_sig_part1 = '{return_type} __flann_{binding_name}('.format(
return_type=return_type, binding_name=binding_name)
maxlen = 100
cpp_sig_ = ut.packstr(cpp_sig_part1 + templated_args,
textwidth=maxlen,
breakchars=', ',
wordsep=', ',
break_words=False,
newline_prefix=' ' * len(cpp_sig_part1))
cpp_sig = cpp_sig_[:-2] + ')'
flann_cpp_code_fmtstr = 'template <typename Distance>\n' + cpp_sig + '\n' + flann_cpp_code_fmtstr_
#print(cpp_sig)
T_typed_sigargs_cpp = ', '.join(binding_sigargs_cpp)
used_template_list = []
used_template_list.append('typename T')
if 'typename Distance::ResultType*' in binding_argtypes:
used_template_list.append('typename R')
used_templates = ', '.join(used_template_list)
type_binding_part1 = 'template <{used_templates}>'.format(
used_templates=used_templates)
type_binding_part2_ = '{return_type} _flann_{binding_name}('.format(
return_type=return_type, binding_name=binding_name)
maxlen = 100
cpp_type_sig_ = ut.packstr(type_binding_part2_ + T_typed_sigargs_cpp,
textwidth=maxlen,
breakchars=', ',
wordsep=', ',
break_words=False,
newline_prefix=' ' * len(type_binding_part2_))
cpp_type_sig = cpp_type_sig_[:-2] + ')'
type_binding_part12 = type_binding_part1 + '\n' + cpp_type_sig
explicit_type_bindings_fmtstr = type_binding_part12 + '\n' + explicit_type_bindings_part3_fmtstr
flann_cpp_codeblock_fmtstr = flann_cpp_code_fmtstr + '\n\n\n' + explicit_type_bindings_fmtstr + '\n'
if return_type == 'int':
errorhandle = 'return -1;'
elif return_type == 'flann_index_t':
errorhandle = 'return NULL;'
else:
errorhandle = 'throw 0;'
# print('------')
# print('flann_cpp_codeblock_fmtstr.format = %s' % (flann_cpp_codeblock_fmtstr,))
try:
flann_cpp_codeblock = flann_cpp_codeblock_fmtstr.format(
cpp_binding_name=cpp_binding_name,
minkowski_option=minkowski_option,
binding_name=binding_name,
#templated_args=templated_args,
callargs=callargs,
#T_typed_sigargs_cpp=T_typed_sigargs_cpp,
errorhandle=errorhandle,
used_templates=used_templates,
return_type=return_type)
except KeyError as ex:
ut.printex(ex, keys=['binding_name'])
raise
dataset_types = [
'',
'float',
'double',
'byte',
'int',
]
#### format c parts
c_header_sigs = []
c_body_blocks = []
templated_ctype_map_c = templated_ctype_map.copy()
#templated_ctype_map_c['dataset'] = 'float'
_fix_explicit_ctype = {
'': 'float',
'byte': 'unsigned char',
}
# For each explicit c type
for dataset_type in dataset_types:
T_type = _fix_explicit_ctype.get(dataset_type, dataset_type)
if dataset_type != '':
signame_type = '_' + dataset_type
else:
signame_type = dataset_type
R_type = 'float' if T_type != 'double' else 'double'
dstype = T_type + '*'
rstype = R_type + '*'
# Overwrite template types with explicit c types
needstemplate = True
for type_, name in zip(binding_argtypes, binding_argnames):
if type_ == 'typename Distance::ElementType*':
templated_ctype_map_c[name] = dstype
needstemplate = False
if type_ == 'typename Distance::ResultType*':
templated_ctype_map_c[name] = rstype
needstemplate = False
if type_ == 'Distance::ResultType*':
templated_ctype_map_c[name] = rstype
needstemplate = False
#if type_ == 'struct FLANNParameters*':
# # hack
# templated_ctype_map_c[name] = 'FLANNParameters*'
# HACK
if binding_name == 'load_index' or binding_name == 'add_points':
needstemplate = True
if binding_name == 'build_index':
needstemplate = True
binding_argtypes2 = ut.dict_take(templated_ctype_map_c,
binding_argnames)
binding_sigargs2 = [
type_ + ' ' + name
for type_, name in zip(binding_argtypes2, binding_argnames)
]
T_typed_sigargs2 = ', '.join(binding_sigargs2)
T_typed_sigargs2_nl = ',\n'.join(binding_sigargs2)
if needstemplate:
iftemplate = '<{T_type}>'.format(T_type=T_type)
else:
iftemplate = ''
if return_type != 'void':
ifreturns = 'return '
else:
ifreturns = ''
bodyblock = c_bodyblock_fmtstr.format(
signame_type=signame_type,
T_typed_sigargs2=T_typed_sigargs2,
binding_name=binding_name,
callargs=callargs,
iftemplate=iftemplate,
ifreturns=ifreturns,
return_type=return_type)
header_line_part1 = c_headersig_part1_fmtstr.format(
signame_type=signame_type,
binding_name=binding_name,
return_type=return_type)
header_line = header_line_part1 + ut.indent(
T_typed_sigargs2_nl,
' ' * len(header_line_part1)).lstrip(' ') + ');'
# Hack for header
header_line = header_line.replace(
'FLANNParameters* flann_params',
'struct FLANNParameters* flann_params')
#header_line = c_headersig_fmtstr.format(signame_type=signame_type,
# T_typed_sigargs2=T_typed_sigargs2,
# binding_name=binding_name,
# return_type=return_type)
c_header_sigs.append(header_line)
c_body_blocks.append(bodyblock)
flann_cpp_codeblock += '\n' + '\n'.join(c_body_blocks)
#flann_cpp_codeblock += '\n' + '// {binding_name} END'.format(binding_name=binding_name)
#BEGIN {binding_name}
flann_h_codeblock = ut.codeblock(r'''
/**
{docstr_cpp}
*/
''').format(docstr_cpp=docstr_cpp, binding_name=binding_name)
flann_h_codeblock += '\n\n' + '\n\n'.join(c_header_sigs)
blocks_dict = {}
import re
flann_index_codeblock = ut.indent(flann_index_codeblock, ' ')
blocks_dict['flann_ctypes.py'] = flann_ctypes_codeblock
blocks_dict['index.py'] = flann_index_codeblock
blocks_dict['flann.h'] = flann_h_codeblock
blocks_dict['flann.cpp'] = flann_cpp_codeblock
for key in blocks_dict.keys():
blocks_dict[key] = re.sub('\n\s+\n', '\n\n', blocks_dict[key])
# , flags=re.MULTILINE)
blocks_dict[key] = re.sub('\s\s+\n', '\n', blocks_dict[key])
pass
if ut.get_argflag('--py'):
print('\n\n# ---------------\n\n')
print('GOES IN flann_ctypes.py')
print('\n\n# ---------------\n\n')
print(flann_ctypes_codeblock)
print('\n\n# ---------------\n\n')
print('GOES IN index.py')
print('\n\n# ---------------\n\n')
print(flann_index_codeblock)
if ut.get_argflag('--c'):
print('\n\n# ---------------\n\n')
print('GOES IN flann.h')
print('\n\n# ---------------\n\n')
print(flann_h_codeblock)
print('\n\n# ---------------\n\n')
print('GOES IN flann.cpp')
print('\n\n# ---------------\n\n')
print(flann_cpp_codeblock)
return blocks_dict, binding_def
def autogen_explicit_injectable_metaclass(classname, regen_command=None,
conditional_imports=None):
r"""
Args:
classname (?):
Returns:
?:
CommandLine:
python -m utool.util_class --exec-autogen_explicit_injectable_metaclass
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_class import * # NOQA
>>> from utool.util_class import __CLASSTYPE_ATTRIBUTES__ # NOQA
>>> import ibeis
>>> import ibeis.control.IBEISControl
>>> classname = ibeis.control.controller_inject.CONTROLLER_CLASSNAME
>>> result = autogen_explicit_injectable_metaclass(classname)
>>> print(result)
"""
import utool as ut
vals_list = []
def make_redirect(func):
# PRESERVES ALL SIGNATURES WITH EXECS
src_fmt = r'''
def {funcname}{defsig}:
""" {orig_docstr}"""
return {orig_funcname}{callsig}
'''
from utool._internal import meta_util_six
orig_docstr = meta_util_six.get_funcdoc(func)
funcname = meta_util_six.get_funcname(func)
orig_funcname = modname.split('.')[-1] + '.' + funcname
orig_docstr = '' if orig_docstr is None else orig_docstr
import textwrap
# Put wrapped function into a scope
import inspect
argspec = inspect.getargspec(func)
(args, varargs, varkw, defaults) = argspec
defsig = inspect.formatargspec(*argspec)
callsig = inspect.formatargspec(*argspec[0:3])
src_fmtdict = dict(funcname=funcname, orig_funcname=orig_funcname,
defsig=defsig, callsig=callsig,
orig_docstr=orig_docstr)
src = textwrap.dedent(src_fmt).format(**src_fmtdict)
return src
src_list = []
for classkey, vals in __CLASSTYPE_ATTRIBUTES__.items():
modname = classkey[1]
if classkey[0] == classname:
vals_list.append(vals)
for func in vals:
src = make_redirect(func)
src = ut.indent(src)
src = '\n'.join([_.rstrip() for _ in src.split('\n')])
src_list.append(src)
if regen_command is None:
regen_command = 'FIXME None given'
module_header = ut.codeblock(
"""
# -*- coding: utf-8 -*-
""" + ut.TRIPLE_DOUBLE_QUOTE + """
Static file containing autogenerated functions for {classname}
Autogenerated on {autogen_time}
RegenCommand:
{regen_command}
""" + ut.TRIPLE_DOUBLE_QUOTE + """
from __future__ import absolute_import, division, print_function
import utool as ut
""").format(
autogen_time=ut.get_timestamp(),
regen_command=regen_command,
classname=classname)
depends_module_block = autogen_import_list(classname, conditional_imports)
inject_statement_fmt = ("print, rrr, profile = "
"ut.inject2(__name__, '[autogen_explicit_inject_{classname}]')")
inject_statement = inject_statement_fmt.format(classname=classname)
source_block_lines = [
module_header,
depends_module_block,
inject_statement,
'\n',
'class ExplicitInject' + classname + '(object):',
] + src_list
source_block = '\n'.join(source_block_lines)
source_block = ut.autoformat_pep8(source_block, aggressive=2)
return source_block
def make_docstr_block(header, block):
import utool as ut
indented_block = '\n' + ut.indent(block)
docstr_block = ''.join([header, ':', indented_block])
return docstr_block
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = [
'ibeis/tests/assert_modules.py'
]
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/ibeis'
testdir = 'ibeis/tests'
exclude_list = []
# Hacky, but not too bad way of getting in doctests
# Test to see if doctest_funcs appears after main
# Do not doctest these modules
#exclude_doctests_fnames = set(['template_definitions.py',
# 'autogen_test_script.py'])
#exclude_dirs = [
# '_broken',
# 'old',
# 'tests',
# 'timeits',
# '_scripts',
# '_timeits',
# '_doc',
# 'notebook',
#]
#dpath_list = ['ibeis']
#doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
# The implict list is exactly the code we will use to make the implicit list
#module_list = None
#doctest_modname_list = None
#'export_subset.py',
#'_autogen_ibeiscontrol_funcs.py',
#'randomforest.py',
implicit_build_modlist_str = ut.codeblock(
'''
import sys
exclude_doctests_fnames = set([
'template_definitions.py',
'autogen_test_script.py',
])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['ibeis']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
'''
)
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
#module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list]
#for modname in doctest_modname_list:
# exec('import ' + modname, globals(), locals())
#module_list = [sys.modules[name] for name in doctest_modname_list]
#print('\n'.join(testcmds))
#print('\n'.join(['python -m ' + modname for modname in doctest_modname_list]))
import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list])
modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(
r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
'''
)
pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds_ = ut.get_module_testlines(module_list, remove_pyc=True,
verbose=False, pythoncmd='RUN_TEST',
testslow=True)
testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_]
test_headers = [
# title, default, module, testpattern
ut.def_test('VTOOL', dpath='vtool/tests', pat=['test*.py'], modname='vtool'),
ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']),
ut.def_test('IBEIS', dpath=testdir, pat=['test_ibs*.py', 'test_delete*.py'], default=False),
ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']),
ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']),
ut.def_test('MISC', dpath=testdir, pat=misc_pats),
ut.def_test('OTHER', dpath=testdir, pat='OTHER'),
ut.def_test('HESAFF', dpath='pyhesaff/tests', pat=['test_*.py'], modname='pyhesaff'),
ut.def_test('DOC', testcmds=testcmds, default=True)
]
# Referencs: https://docs.python.org/2/library/runpy.html
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list)
#print(pyscript_text)
return shscript_text, pyscript_text
def make_run_tests_script_text(test_headers,
test_argvs,
quick_tests=None,
repodir=None,
exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath,
specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar,
fpath=fpath,
specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "', 'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")'
]).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(
testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(
testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(
header_lower=header_lower,
testflag=testflag,
)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(
set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten(
[ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any(
[x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [
util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list
]
testline_list = [
format_testline(fpath, dirvar) for fpath in testfpath_list
]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(
ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,
)
header_test_block = header_test_block_fmstr.format(
**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs,
dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,
)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
