def test():
print('enter test')
log_fpath1 = utool.get_app_resource_dir('utool', 'test_logfile1.txt')
log_fpath2 = utool.get_app_resource_dir('utool', 'test_logfile2.txt')
utool.start_logging(log_fpath1, 'w')
func1()
func2()
utool.stop_logging()
print('\n\n')
print('This line is NOT logged')
print('\n\n')
utool.start_logging(log_fpath2, 'w')
print('This line is logged')
utool.stop_logging()
log1 = utool.read_from(log_fpath1, verbose=False)
log2 = utool.read_from(log_fpath2, verbose=False)
target1 = utool.unindent('''
<__LOG_START__>
logging to log_fpath=%r
[test][func1]enter func1
[test][func1]exit func1
[test][func2]enter func2
[test][func2][func1]enter func1
[test][func2][func1]exit func1
[test][func2]exit func2
<__LOG_STOP__>''' % log_fpath1).strip()
target2 = utool.unindent('''
<__LOG_START__>
logging to log_fpath=%r
[test]This line is logged
<__LOG_STOP__>''' % log_fpath2).strip()
output1 = remove_timestamp(log1).strip()
output2 = remove_timestamp(log2).strip()
try:
assert target1 == output1, 'target1 failed'
assert target2 == output2, 'target2 failed'
builtins.print('TEST PASSED')
except AssertionError:
builtins.print('\n<!!! TEST FAILED !!!>')
builtins.print('\ntarget1:')
builtins.print(target1)
builtins.print('\noutput1:')
builtins.print(output1)
builtins.print('\ntarget2:')
builtins.print(target2)
builtins.print('\noutput2:')
builtins.print(output2)
builtins.print('</!!! TEST FAILED !!!>\n')
raise
def test():
print('enter test')
log_fpath1 = utool.get_app_resource_dir('utool', 'test_logfile1.txt')
log_fpath2 = utool.get_app_resource_dir('utool', 'test_logfile2.txt')
utool.start_logging(log_fpath1, 'w')
func1()
func2()
utool.stop_logging()
print('\n\n')
print('This line is NOT logged')
print('\n\n')
utool.start_logging(log_fpath2, 'w')
print('This line is logged')
utool.stop_logging()
log1 = utool.read_from(log_fpath1, verbose=False)
log2 = utool.read_from(log_fpath2, verbose=False)
target1 = utool.unindent('''
<__LOG_START__>
logging to log_fpath=%r
[test][func1]enter func1
[test][func1]exit func1
[test][func2]enter func2
[test][func2][func1]enter func1
[test][func2][func1]exit func1
[test][func2]exit func2
<__LOG_STOP__>''' % log_fpath1).strip()
target2 = utool.unindent('''
<__LOG_START__>
logging to log_fpath=%r
[test]This line is logged
<__LOG_STOP__>''' % log_fpath2).strip()
output1 = remove_timestamp(log1).strip()
output2 = remove_timestamp(log2).strip()
try:
assert target1 == output1, 'target1 failed'
assert target2 == output2, 'target2 failed'
builtins.print('TEST PASSED')
except AssertionError:
builtins.print('\n<!!! TEST FAILED !!!>')
builtins.print('\ntarget1:')
builtins.print(target1)
builtins.print('\noutput1:')
builtins.print(output1)
builtins.print('\ntarget2:')
builtins.print(target2)
builtins.print('\noutput2:')
builtins.print(output2)
builtins.print('</!!! TEST FAILED !!!>\n')
raise
def dictinfo(dict_):
if not isinstance(dict_, dict):
return 'expected dict got %r' % type(dict_)
keys = list(dict_.keys())
vals = list(dict_.values())
num_keys = len(keys)
key_types = list(set(map(type, keys)))
val_types = list(set(map(type, vals)))
fmtstr_ = '\n' + ut.unindent('''
* num_keys = {num_keys}
* key_types = {key_types}
* val_types = {val_types}
'''.strip('\n'))
if len(val_types) == 1:
if val_types[0] == np.ndarray:
# each key holds an ndarray
val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0)
val_shape_stats_str = ut.dict_str(val_shape_stats,
strvals=True,
newlines=False)
val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* val_shape_stats = {val_shape_stats_str}
* val_dtypes = {val_dtypes}
'''.strip('\n'))
elif val_types[0] == list:
# each key holds a list
val_len_stats = ut.get_stats(set(map(len, vals)))
val_len_stats_str = ut.dict_str(val_len_stats,
strvals=True,
newlines=False)
depth = ut.list_depth(vals)
deep_val_types = list(set(ut.list_deep_types(vals)))
fmtstr_ += ut.unindent('''
* list_depth = {depth}
* val_len_stats = {val_len_stats_str}
* deep_types = {deep_val_types}
'''.strip('\n'))
if len(deep_val_types) == 1:
if deep_val_types[0] == np.ndarray:
deep_val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* deep_val_dtypes = {deep_val_dtypes}
''').strip('\n')
elif val_types[0] in [
np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32,
np.float64
]:
# each key holds a scalar
val_stats = ut.get_stats(vals)
fmtstr_ += ut.unindent('''
* val_stats = {val_stats}
''').strip('\n')
fmtstr = fmtstr_.format(**locals())
return ut.indent(fmtstr)
def timeit2(stmt, setup='', number=1000):
import timeit
stmt_ = utool.unindent(stmt)
setup_ = utool.unindent(setup)
print('----------')
print('TIMEIT: \n' + stmt_)
try:
total_time = timeit.timeit(stmt_, setup_, number=number)
except Exception as ex:
utool.printex(ex, iswarning=False)
raise
print(' * timed: %r seconds' % (total_time))
def timeit2(stmt, setup='', number=1000):
import timeit
stmt_ = utool.unindent(stmt)
setup_ = utool.unindent(setup)
print('----------')
print('TIMEIT: \n' + stmt_)
try:
total_time = timeit.timeit(stmt_, setup_, number=number)
except Exception as ex:
utool.printex(ex, iswarning=False)
raise
print(' * timed: %r seconds' % (total_time))
def dictinfo(dict_):
if not isinstance(dict_, dict):
return 'expected dict got %r' % type(dict_)
keys = list(dict_.keys())
vals = list(dict_.values())
num_keys = len(keys)
key_types = list(set(map(type, keys)))
val_types = list(set(map(type, vals)))
fmtstr_ = '\n' + ut.unindent('''
* num_keys = {num_keys}
* key_types = {key_types}
* val_types = {val_types}
'''.strip('\n'))
if len(val_types) == 1:
if val_types[0] == np.ndarray:
# each key holds an ndarray
val_shape_stats = ut.get_stats(set(map(np.shape, vals)), axis=0)
val_shape_stats_str = ut.dict_str(val_shape_stats, strvals=True, newlines=False)
val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* val_shape_stats = {val_shape_stats_str}
* val_dtypes = {val_dtypes}
'''.strip('\n'))
elif val_types[0] == list:
# each key holds a list
val_len_stats = ut.get_stats(set(map(len, vals)))
val_len_stats_str = ut.dict_str(val_len_stats, strvals=True, newlines=False)
depth = ut.list_depth(vals)
deep_val_types = list(set(ut.list_deep_types(vals)))
fmtstr_ += ut.unindent('''
* list_depth = {depth}
* val_len_stats = {val_len_stats_str}
* deep_types = {deep_val_types}
'''.strip('\n'))
if len(deep_val_types) == 1:
if deep_val_types[0] == np.ndarray:
deep_val_dtypes = list(set([val.dtype for val in vals]))
fmtstr_ += ut.unindent('''
* deep_val_dtypes = {deep_val_dtypes}
''').strip('\n')
elif val_types[0] in [np.uint8, np.int8, np.int32, np.int64, np.float16, np.float32, np.float64]:
# each key holds a scalar
val_stats = ut.get_stats(vals)
fmtstr_ += ut.unindent('''
* val_stats = {val_stats}
''').strip('\n')
fmtstr = fmtstr_.format(**locals())
return ut.indent(fmtstr)
def make_benchmarks(funcname, docstring, py_modname):
r"""
>>> from cyth.cyth_script import * # NOQA
>>> funcname = 'replace_funcalls'
>>> docstring = replace_funcalls.func_doc
>>> py_modname = 'cyth.cyth_script'
>>> benchmark_list = list(make_benchmarks(funcname, docstring, py_modname))
>>> print(benchmark_list)
[[("replace_funcalls('foo(5)', 'foo', 'bar')", "_replace_funcalls_cyth('foo(5)', 'foo', 'bar')"), ("replace_funcalls('foo(5)', 'bar', 'baz')", "_replace_funcalls_cyth('foo(5)', 'bar', 'baz')")], "from cyth.cyth_script import _replace_funcalls_cyth\nfrom cyth_script import * # NOQA\nreplace_funcalls('foo(5)', 'foo', 'bar')\nreplace_funcalls('foo(5)', 'bar', 'baz')\n"]
#>>> output = [((x.source, x.want), y.source, y.want) for x, y in benchmark_list]
#>>> print(utool.hashstr(repr(output)))
"""
doctest_examples = parse_doctest_examples(docstring)
test_lines = []
cyth_lines = []
setup_lines = []
cyth_funcname = cyth_helpers.get_cyth_name(funcname)
for example in doctest_examples:
benchline = get_benchline(example.source, funcname)
if benchline is not None:
test_lines.append(benchline)
cyth_lines.append(
replace_funcalls(benchline, funcname, cyth_funcname))
setup_lines.append(example.source)
else:
setup_lines.append(example.source)
test_tuples = list(zip(test_lines, cyth_lines))
setup_script = utool.unindent(''.join(setup_lines))
#modname = 'vtool.keypoint'
setup_script = 'from %s import %s\n' % (
py_modname,
cyth_funcname,
) + setup_script
return test_tuples, setup_script
def build_runbench_pyth_text(cy_bench_list):
# write script to run all cyth benchmarks
runbench_pytext_fmt_ = r'''
#!/usr/bin/env python
" Autogenerated by cyth on {timestamp} "
from __future__ import absolute_import, division, print_function
import utool
{bench_import_text}
SORTBY = utool.get_argval('--sortby', str, 'python')
if __name__ == '__main__':
all_results = []
iterations = utool.get_argval(('--iterations', '-n'), type_=int, default=100)
# Run the benchmarks
{bench_runline_text}
# Sort by chosen field
sortable_fields = ['python', 'cython']
sortx = sortable_fields.index(SORTBY)
sorted_allresults = sorted(all_results, key=lambda tup: tup[sortx])
sorted_lines = [tup[2] for tup in sorted_allresults]
# Report sorted results
print('\n\n')
print('==================================')
print('Aggregating all benchmarks results')
print('==================================')
print('\n')
print('sorting by %s' % sortable_fields[sortx])
print('\n'.join(utool.flatten(sorted_lines)))
'''
runbench_pytext_fmt = utool.unindent(runbench_pytext_fmt_).strip('\n')
from os.path import relpath, splitext
import os
def bench_fpath_to_modname(bench):
bench_upath = utool.unixpath(bench)
bench_relpath = relpath(bench_upath, os.getcwd())
bench_relname, _ = splitext(bench_relpath)
bench_modname = bench_relname.replace('\\', '/').replace('/', '.')
return bench_modname
bench_modnames = list(map(bench_fpath_to_modname, cy_bench_list))
bench_imports = [
'import ' + bench_modname for bench_modname in bench_modnames
]
runline_fmt = 'all_results.extend({bench_modname}.run_all_benchmarks(iterations))'
bench_runlines = [
runline_fmt.format(bench_modname=bench_modname)
for bench_modname in bench_modnames
]
bench_import_text = '\n'.join(bench_imports)
bench_runline_text = '\n '.join(bench_runlines)
timestamp = utool.get_timestamp() # NOQA
runbench_pytext = runbench_pytext_fmt.format(
timestamp=timestamp,
bench_runline_text=bench_runline_text,
bench_import_text=bench_import_text)
return runbench_pytext
def make_benchmarks(funcname, docstring, py_modname):
r"""
>>> from cyth.cyth_script import * # NOQA
>>> funcname = 'replace_funcalls'
>>> docstring = replace_funcalls.func_doc
>>> py_modname = 'cyth.cyth_script'
>>> benchmark_list = list(make_benchmarks(funcname, docstring, py_modname))
>>> print(benchmark_list)
[[("replace_funcalls('foo(5)', 'foo', 'bar')", "_replace_funcalls_cyth('foo(5)', 'foo', 'bar')"), ("replace_funcalls('foo(5)', 'bar', 'baz')", "_replace_funcalls_cyth('foo(5)', 'bar', 'baz')")], "from cyth.cyth_script import _replace_funcalls_cyth\nfrom cyth_script import * # NOQA\nreplace_funcalls('foo(5)', 'foo', 'bar')\nreplace_funcalls('foo(5)', 'bar', 'baz')\n"]
#>>> output = [((x.source, x.want), y.source, y.want) for x, y in benchmark_list]
#>>> print(utool.hashstr(repr(output)))
"""
doctest_examples = parse_doctest_examples(docstring)
test_lines = []
cyth_lines = []
setup_lines = []
cyth_funcname = cyth_helpers.get_cyth_name(funcname)
for example in doctest_examples:
benchline = get_benchline(example.source, funcname)
if benchline is not None:
test_lines.append(benchline)
cyth_lines.append(replace_funcalls(benchline, funcname, cyth_funcname))
setup_lines.append(example.source)
else:
setup_lines.append(example.source)
test_tuples = list(zip(test_lines, cyth_lines))
setup_script = utool.unindent(''.join(setup_lines))
#modname = 'vtool.keypoint'
setup_script = 'from %s import %s\n' % (py_modname, cyth_funcname,) + setup_script
return test_tuples, setup_script
def get_inspect_str(qres):
assert_qres(qres)
nFeatMatch_list = get_num_feats_in_matches(qres)
nFeatMatch_stats = utool.mystats(nFeatMatch_list)
top_lbl = utool.unindent('''
top aids
scores
ranks''').strip()
top_aids = qres.get_top_aids(num=5)
top_scores = qres.get_aid_scores(top_aids)
top_ranks = qres.get_aid_ranks(top_aids)
top_stack = np.vstack((top_aids, top_scores, top_ranks))
top_stack = np.array(top_stack, dtype=np.int32)
top_str = str(top_stack)
inspect_str = '\n'.join([
'QueryResult',
'qaid=%r ' % qres.qaid,
utool.horiz_string(top_lbl, ' ', top_str),
'num Feat Matches stats:',
utool.indent(utool.dict_str(nFeatMatch_stats)),
])
inspect_str = utool.indent(inspect_str, '[INSPECT] ')
return inspect_str
def parse_benchmarks(funcname, docstring, py_modname):
test_tuples_, setup_script_ = make_benchmarks(funcname, docstring,
py_modname)
if len(test_tuples_) == 0:
test_tuples = '[]'
else:
test_tuples = '[\n' + (' ' * 8) + '\n '.join(
list(map(str, test_tuples_))) + '\n ]' # NOQA
setup_script = utool.indent(setup_script_).strip() # NOQA
benchmark_name = utool.quasiquote('run_benchmark_{funcname}')
#test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''')
# http://en.wikipedia.org/wiki/Relative_change_and_difference
bench_code_fmt_ = r'''
def {benchmark_name}(iterations):
test_tuples = {test_tuples}
setup_script = textwrap.dedent("""
{setup_script}
""")
time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations)
time_pair = lambda (x, y): (time_line(x), time_line(y))
def print_timing_info(tup):
from math import log
test_lines = []
def test_print(str):
if not utool.QUIET:
print(str)
test_lines.append(str)
test_print('\n---------------')
test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations))
test_print('[bench] tests:')
test_print(' ' + str(tup))
(pyth_time, cyth_time) = time_pair(tup)
test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time))
test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time))
time_delta = cyth_time - pyth_time
#pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100
#pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100
pyth_per_cyth = (pyth_time / cyth_time) * 100
inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100
nepers = log(cyth_time / pyth_time)
if time_delta < 0:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,))
#test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,))
test_print('[bench.result] cython was faster by %f seconds' % -time_delta)
else:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,))
#test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers slower' % (nepers,))
test_print('[bench.result] python was faster by %f seconds' % time_delta)
pyth_call, cyth_call = tup
run_doctest(pyth_call, cyth_call, setup_script)
return (pyth_time, cyth_time, test_lines)
test_results = list(map(print_timing_info, test_tuples))
# results are lists of (time1, time2, strlist)
return test_results'''
bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n')
bench_code = utool.quasiquote(bench_code_fmt)
return (benchmark_name, bench_code)
def get_table_csv_header(db, tablename):
column_nametypes = db.table_columns[tablename]
header_constraints = '# CONSTRAINTS: %r' % db.table_constraints[tablename]
header_name = '# TABLENAME: %r' % tablename
header_types = utool.indentjoin(column_nametypes, '\n# ')
header_doc = utool.indentjoin(utool.unindent(db.table_docstr[tablename]).split('\n'), '\n# ')
header = header_doc + '\n' + header_name + header_types + '\n' + header_constraints
return header
def parse_benchmarks(funcname, docstring, py_modname):
test_tuples_, setup_script_ = make_benchmarks(funcname, docstring, py_modname)
if len(test_tuples_) == 0:
test_tuples = '[]'
else:
test_tuples = '[\n' + (' ' * 8) + '\n '.join(list(map(str, test_tuples_))) + '\n ]' # NOQA
setup_script = utool.indent(setup_script_).strip() # NOQA
benchmark_name = utool.quasiquote('run_benchmark_{funcname}')
#test_tuples, setup_script = make_benchmarks('''{funcname}''', '''{docstring}''')
# http://en.wikipedia.org/wiki/Relative_change_and_difference
bench_code_fmt_ = r'''
def {benchmark_name}(iterations):
test_tuples = {test_tuples}
setup_script = textwrap.dedent("""
{setup_script}
""")
time_line = lambda line: timeit.timeit(stmt=line, setup=setup_script, number=iterations)
time_pair = lambda (x, y): (time_line(x), time_line(y))
def print_timing_info(tup):
from math import log
test_lines = []
def test_print(str):
if not utool.QUIET:
print(str)
test_lines.append(str)
test_print('\n---------------')
test_print('[bench] timing {benchmark_name} for %d iterations' % (iterations))
test_print('[bench] tests:')
test_print(' ' + str(tup))
(pyth_time, cyth_time) = time_pair(tup)
test_print("[bench.python] {funcname} time=%f seconds" % (pyth_time))
test_print("[bench.cython] {funcname} time=%f seconds" % (cyth_time))
time_delta = cyth_time - pyth_time
#pcnt_change_wrt_pyth = (time_delta / pyth_time) * 100
#pcnt_change_wrt_cyth = (time_delta / cyth_time) * 100
pyth_per_cyth = (pyth_time / cyth_time) * 100
inv_cyth_per_pyth = 1 / (cyth_time / pyth_time) * 100
nepers = log(cyth_time / pyth_time)
if time_delta < 0:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (inv_cyth_per_pyth,))
#test_print('[bench.result] cython was %.1fx faster' % (-pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers faster' % (-nepers,))
test_print('[bench.result] cython was faster by %f seconds' % -time_delta)
else:
test_print('[bench.result] cython was %.1f%% of the speed of python' % (pyth_per_cyth,))
#test_print('[bench.result] cython was %.1fx slower' % (pcnt_change_wrt_pyth,))
test_print('[bench.result] cython was %.1f nepers slower' % (nepers,))
test_print('[bench.result] python was faster by %f seconds' % time_delta)
pyth_call, cyth_call = tup
run_doctest(pyth_call, cyth_call, setup_script)
return (pyth_time, cyth_time, test_lines)
test_results = list(map(print_timing_info, test_tuples))
# results are lists of (time1, time2, strlist)
return test_results'''
bench_code_fmt = utool.unindent(bench_code_fmt_).strip('\n')
bench_code = utool.quasiquote(bench_code_fmt)
return (benchmark_name, bench_code)
def parse_cyth_markup(self, docstr, toplevel=False, funcdef_node=None):
"""
Args:
docstr (str): block of text possibly with cyth markups
toplevel (bool): True if docstr is in the global scope
funcdef_node: parent funcdef node for return type inference
"""
funcname = None if funcdef_node is None else funcdef_node.name
comment_str = docstr.strip()
has_markup = comment_str.find('<CYTH') != -1
# type returned_action = [`defines of string * (string, string) Hashtbl.t | `replace of string] option
def make_defines(cyth_def, return_type=None):
typedict = self.parse_cythdef(cyth_def)
if not return_type:
return_type = infer_return_type(funcdef_node, typedict)
actiontup = ('defines', cyth_def, typedict, return_type)
return actiontup
# TODO: these actions are old-style cyth commands. Replace them.
tags_to_actions = [
('<CYTH>', lambda cyth_def: make_defines(cyth_def.group(1))),
('<CYTH returns="(.*?)">', lambda cyth_def: make_defines(
cyth_def.group(2), cyth_def.group(1))),
('<CYTH:REPLACE>', lambda cyth_def:
('replace', utool.unindent(cyth_def.group(1)))),
]
end_tag = '</CYTH>'
regex_flags = re.DOTALL | re.MULTILINE
regex_to_actions = [(re.compile(tag + '(.*?)' + end_tag,
regex_flags), act)
for tag, act in tags_to_actions]
if has_markup:
# if CYTH markup was found...
self.register_benchmark(funcname, docstr, self.py_modname)
if toplevel:
# hack global cyth-docstrs to always CYTH:REPLACE
comment_str = re.sub('<CYTH>', '<CYTH:REPLACE>', comment_str)
for (regex, action) in regex_to_actions:
match = regex.search(comment_str)
if match:
#cyth_def = match.group(1)
actiontup = action(match)
return actiontup
print(comment_str)
utool.printex(
NotImplementedError('no known cyth tag in docstring'),
iswarning=True,
key_list=[
'funcname', 'toplevel', 'regex', 'action', 'match',
'comment_str'
])
else:
# Do nothing if cyth markup was not found
pass
return None
def benchmark_L2_dist():
setup = utool.unindent(
"""
hist1 = np.random.rand(100, 128).astype(np.float64)
hist2 = np.random.rand(100, 128).astype(np.float64)
"""
)
func_list = ["vtool.linalg.L2_sqrd", "vtool.linalg.L2_sqrd_cython"]
argstr = "(hist1, hist2)"
return _run_benchmark(setup, func_list, argstr)
def benchmark_det_dist():
setup = utool.unindent(
"""
det1 = np.random.rand(100).astype(np.float64)
det2 = np.random.rand(100).astype(np.float64)
"""
)
func_list = ["vtool.linalg.det_distance", "vtool.linalg.det_distance_cython"]
argstr = "(det1, det2)"
return _run_benchmark(setup, func_list, argstr)
def benchmark_det_dist():
setup = utool.unindent('''
det1 = np.random.rand(100).astype(np.float64)
det2 = np.random.rand(100).astype(np.float64)
''')
func_list = [
'vtool.linalg.det_distance',
'vtool.linalg.det_distance_cython',
]
argstr = '(det1, det2)'
return _run_benchmark(setup, func_list, argstr)
def benchmark_L2_dist():
setup = utool.unindent('''
hist1 = np.random.rand(100, 128).astype(np.float64)
hist2 = np.random.rand(100, 128).astype(np.float64)
''')
func_list = [
'vtool.linalg.L2_sqrd',
'vtool.linalg.L2_sqrd_cython',
]
argstr = '(hist1, hist2)'
return _run_benchmark(setup, func_list, argstr)
def build_runbench_pyth_text(cy_bench_list):
# write script to run all cyth benchmarks
runbench_pytext_fmt_ = r'''
#!/usr/bin/env python
" Autogenerated by cyth on {timestamp} "
from __future__ import absolute_import, division, print_function
import utool
{bench_import_text}
SORTBY = utool.get_argval('--sortby', str, 'python')
if __name__ == '__main__':
all_results = []
iterations = utool.get_argval(('--iterations', '-n'), type_=int, default=100)
# Run the benchmarks
{bench_runline_text}
# Sort by chosen field
sortable_fields = ['python', 'cython']
sortx = sortable_fields.index(SORTBY)
sorted_allresults = sorted(all_results, key=lambda tup: tup[sortx])
sorted_lines = [tup[2] for tup in sorted_allresults]
# Report sorted results
print('\n\n')
print('==================================')
print('Aggregating all benchmarks results')
print('==================================')
print('\n')
print('sorting by %s' % sortable_fields[sortx])
print('\n'.join(utool.flatten(sorted_lines)))
'''
runbench_pytext_fmt = utool.unindent(runbench_pytext_fmt_).strip('\n')
from os.path import relpath, splitext
import os
def bench_fpath_to_modname(bench):
bench_upath = utool.unixpath(bench)
bench_relpath = relpath(bench_upath, os.getcwd())
bench_relname, _ = splitext(bench_relpath)
bench_modname = bench_relname.replace('\\', '/').replace('/', '.')
return bench_modname
bench_modnames = list(map(bench_fpath_to_modname, cy_bench_list))
bench_imports = ['import ' + bench_modname for bench_modname in bench_modnames]
runline_fmt = 'all_results.extend({bench_modname}.run_all_benchmarks(iterations))'
bench_runlines = [runline_fmt.format(bench_modname=bench_modname)
for bench_modname in bench_modnames]
bench_import_text = '\n'.join(bench_imports)
bench_runline_text = '\n '.join(bench_runlines)
timestamp = utool.get_timestamp() # NOQA
runbench_pytext = runbench_pytext_fmt.format(timestamp=timestamp,
bench_runline_text=bench_runline_text,
bench_import_text=bench_import_text)
return runbench_pytext
def expand_closure_source(funcname, func):
source = ut.get_func_sourcecode(func)
closure_vars = [(k, v.cell_contents) for k, v in
zip(func.func_code.co_freevars, func.func_closure)]
source = ut.unindent(source)
import re
for k, v in closure_vars:
source = re.sub('\\b' + k + '\\b', ut.repr2(v), source)
source = re.sub(r'def .*\(self', 'def ' + funcname + '(self', source)
source = ut.indent(source.strip(), ' ') + '\n'
return source
def benchmark_invVR_sqrd_scale():
setup = utool.unindent(
"""
import numpy as np
import numpy.linalg as npl
import vtool
invVRs = np.random.rand(100, 3, 3).astype(np.float64)
"""
)
func_list = ["vtool.keypoint.get_invVR_mats_sqrd_scale", "vtool.keypoint.get_invVR_mats_sqrd_scale_cython"]
argstr = "(invVRs)"
return _run_benchmark(setup, func_list, argstr)
def benchmark_invVR_sqrd_scale():
setup = utool.unindent('''
import numpy as np
import numpy.linalg as npl
import vtool
invVRs = np.random.rand(100, 3, 3).astype(np.float64)
''')
func_list = [
'vtool.keypoint.get_invVR_mats_sqrd_scale',
'vtool.keypoint.get_invVR_mats_sqrd_scale_cython',
]
argstr = '(invVRs)'
return _run_benchmark(setup, func_list, argstr)
def TIME_GEN_PREPROC_IMG(ibs):
from ibeis.algo.preproc.preproc_image import add_images_params_gen
print('[TIME_GEN_PREPROC_IMG]')
gid_list = ibs.get_valid_gids()
gpath_list = ibs.get_image_paths(gid_list)
# STABILITY
if not utool.get_argflag('--nostable'):
# TEST 1
with utool.Timer('parallel chunksize=1'):
output1 = list(add_images_params_gen(gpath_list, chunksize=1))
print(utool.truncate_str(str(output1), 80))
assert len(output1) == len(gpath_list), 'chuncksize changes output'
# TEST 2
with utool.Timer('parallel chunksize=2'):
output2 = list(add_images_params_gen(gpath_list, chunksize=2))
print(utool.truncate_str(str(output2), 80))
assert output1 == output2, 'chuncksize changes output'
# TEST N
with utool.Timer('parallel chunksize=None'):
outputN = list(add_images_params_gen(gpath_list, chunksize=None))
print(utool.truncate_str(str(output2), 80))
assert outputN == output2, 'chuncksize changes output'
# BENCHMARK
setup = utool.unindent(
'''
from ibeis.algo.preproc.preproc_image import add_images_params_gen
genkw = dict(prog=False, verbose=True)
gpath_list = %r
''' % (gpath_list,))
print(utool.truncate_str(str(gpath_list), 80))
print('Processing %d images' % (len(gpath_list),))
timeit3 = partial(timeit2, setup=setup, number=3)
timeit3('list(add_images_params_gen(gpath_list, chunksize=None, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=None, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=1, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=2, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=4, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=8, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=16, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=32, **genkw))')
print('[/TIME_GEN_PREPROC_IMG]')
return locals()
def parse_cyth_markup(self, docstr, toplevel=False, funcdef_node=None):
"""
Args:
docstr (str): block of text possibly with cyth markups
toplevel (bool): True if docstr is in the global scope
funcdef_node: parent funcdef node for return type inference
"""
funcname = None if funcdef_node is None else funcdef_node.name
comment_str = docstr.strip()
has_markup = comment_str.find('<CYTH') != -1
# type returned_action = [`defines of string * (string, string) Hashtbl.t | `replace of string] option
def make_defines(cyth_def, return_type=None):
typedict = self.parse_cythdef(cyth_def)
if not return_type:
return_type = infer_return_type(funcdef_node, typedict)
actiontup = ('defines', cyth_def, typedict, return_type)
return actiontup
# TODO: these actions are old-style cyth commands. Replace them.
tags_to_actions = [
('<CYTH>', lambda cyth_def: make_defines(cyth_def.group(1))),
('<CYTH returns="(.*?)">', lambda cyth_def: make_defines(cyth_def.group(2), cyth_def.group(1))),
('<CYTH:REPLACE>', lambda cyth_def: ('replace', utool.unindent(cyth_def.group(1)))),
]
end_tag = '</CYTH>'
regex_flags = re.DOTALL | re.MULTILINE
regex_to_actions = [(re.compile(tag + '(.*?)' + end_tag, regex_flags), act)
for tag, act in tags_to_actions]
if has_markup:
# if CYTH markup was found...
self.register_benchmark(funcname, docstr, self.py_modname)
if toplevel:
# hack global cyth-docstrs to always CYTH:REPLACE
comment_str = re.sub('<CYTH>', '<CYTH:REPLACE>', comment_str)
for (regex, action) in regex_to_actions:
match = regex.search(comment_str)
if match:
#cyth_def = match.group(1)
actiontup = action(match)
return actiontup
print(comment_str)
utool.printex(NotImplementedError('no known cyth tag in docstring'),
iswarning=True,
key_list=['funcname', 'toplevel', 'regex', 'action',
'match', 'comment_str'])
else:
# Do nothing if cyth markup was not found
pass
return None
def TIME_GEN_PREPROC_IMG(ibs):
from ibeis.algo.preproc.preproc_image import add_images_params_gen
print('[TIME_GEN_PREPROC_IMG]')
gid_list = ibs.get_valid_gids()
gpath_list = ibs.get_image_paths(gid_list)
# STABILITY
if not utool.get_argflag('--nostable'):
# TEST 1
with utool.Timer('parallel chunksize=1'):
output1 = list(add_images_params_gen(gpath_list, chunksize=1))
print(utool.truncate_str(str(output1), 80))
assert len(output1) == len(gpath_list), 'chuncksize changes output'
# TEST 2
with utool.Timer('parallel chunksize=2'):
output2 = list(add_images_params_gen(gpath_list, chunksize=2))
print(utool.truncate_str(str(output2), 80))
assert output1 == output2, 'chuncksize changes output'
# TEST N
with utool.Timer('parallel chunksize=None'):
outputN = list(add_images_params_gen(gpath_list, chunksize=None))
print(utool.truncate_str(str(output2), 80))
assert outputN == output2, 'chuncksize changes output'
# BENCHMARK
setup = utool.unindent('''
from ibeis.algo.preproc.preproc_image import add_images_params_gen
genkw = dict(prog=False, verbose=True)
gpath_list = %r
''' % (gpath_list, ))
print(utool.truncate_str(str(gpath_list), 80))
print('Processing %d images' % (len(gpath_list), ))
timeit3 = partial(timeit2, setup=setup, number=3)
timeit3('list(add_images_params_gen(gpath_list, chunksize=None, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=None, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=1, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=2, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=4, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=8, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=16, **genkw))')
timeit3('list(add_images_params_gen(gpath_list, chunksize=32, **genkw))')
print('[/TIME_GEN_PREPROC_IMG]')
return locals()
def import_cyth_execstr(pyth_modname):
"""
>>> from cyth.cyth_importer import * # NOQA
>>> from vtool import trig # NOQA
>>> pyth_modname = 'vtool.trig'
"""
dummy_cythonized_funcs = import_cyth_default(pyth_modname)
pyth_list = []
for funcname, func in dummy_cythonized_funcs.items():
pyth_list.append(funcname + ' = ' + get_funcname(func))
pyth_list2 = utool.align_lines(sorted(pyth_list), '=')
try:
cyth_list = []
pkgname, fromlist, cyth_modname = pkg_submodule_split(pyth_modname)
cythonized_funcs = get_cythonized_funcs(pyth_modname)
for funcname, func in cythonized_funcs.items():
cyth_list.append(funcname + ' = ' + cyth_modname + '.' + func.__name__)
cyth_list2 = ['import ' + cyth_modname] + utool.align_lines(sorted(cyth_list), '=')
except ImportError:
cyth_list2 = ['raise ImportError("no cyth")']
except Exception as ex:
cyth_list2 = ['raise ImportError("cyth import error: %s")' % str(ex)]
cyth_block = utool.indentjoin(cyth_list2).strip()
pyth_block = utool.indentjoin(pyth_list2).strip()
execstr = utool.unindent(
'''
try:
if not cyth.WITH_CYTH:
raise ImportError('no cyth')
{cyth_block}
CYTHONIZED = True
# print('cyth is on in %s' % (__name__,))
except ImportError:
{pyth_block}
# print('cyth is off in %s' % (__name__,))
CYTHONIZED = False''').format(**locals()).strip('\n')
#print(execstr)
if cyth_args.CYTH_WRITE:
write_explicit(pyth_modname, execstr)
return execstr
def make_setup(repodir):
setup_text_fmt_ = '''
# autogenerated setup.py on {timestamp} for {repodir}
from __future__ import absolute_import, division, print_function
import setuptools
#from utool import util_setup
INSTALL_REQUIRES = [
]
if __name__ == '__main__':
setuptools.setup(
name='{pkgname}',
packages={packages},
#packages=util_setup.find_packages(),
#version='0.0.0.autogen',
#description='short description',
#url='',
#ext_modules=util_setup.find_ext_modules(),
#cmdclass=util_setup.get_cmdclass(),
#author=''
#author_email='',
#keywords='',
#install_requires=INSTALL_REQUIRES,
package_data={{}},
scripts=[
],
classifiers=[],
)
'''
setup_text_fmt = utool.unindent(setup_text_fmt_)
timestamp = utool.get_timestamp()
pkgname = basename(repodir)
packages = utool.ls_moduledirs(repodir, full=False)
print(pkgname)
setup_text = setup_text_fmt.format(
packages=packages,
repodir=repodir,
timestamp=timestamp,
pkgname=pkgname,
)
return setup_text
def make_setup(repodir):
setup_text_fmt_ = '''
# autogenerated setup.py on {timestamp} for {repodir}
from __future__ import absolute_import, division, print_function
import setuptools
#from utool import util_setup
INSTALL_REQUIRES = [
]
if __name__ == '__main__':
setuptools.setup(
name='{pkgname}',
packages={packages},
#packages=util_setup.find_packages(),
#version='0.0.0.autogen',
#description='short description',
#url='',
#ext_modules=util_setup.find_ext_modules(),
#cmdclass=util_setup.get_cmdclass(),
#author=''
#author_email='',
#keywords='',
#install_requires=INSTALL_REQUIRES,
package_data={{}},
scripts=[
],
classifiers=[],
)
'''
setup_text_fmt = utool.unindent(setup_text_fmt_)
timestamp = utool.get_timestamp()
pkgname = basename(repodir)
packages = utool.ls_moduledirs(repodir, full=False)
print(pkgname)
setup_text = setup_text_fmt.format(
packages=packages,
repodir=repodir,
timestamp=timestamp,
pkgname=pkgname,
)
return setup_text
def injest_main():
r"""
CommandLine:
python -m ibeis.dbio.ingest_database --test-injest_main
python -m ibeis.dbio.ingest_database --test-injest_main --db snow-leopards
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.dbio.ingest_database import * # NOQA
>>> injest_main()
"""
print('__main__ = ingest_database.py')
print(ut.unindent(
'''
usage:
python ibeis/ingest/ingest_database.py --db [dbname]
Valid dbnames:''') + ut.indentjoin(STANDARD_INGEST_FUNCS.keys(), '\n * '))
dbname = ut.get_argval('--db', str, None)
force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
ibs = ingest_standard_database(dbname, force_delete) # NOQA
print('finished db injest')
def visit_FunctionDef(self, node):
"""
Gets called when parser encounters a function definition
"""
#super(CythVisitor, self).visit_FunctionDef(node)
new_body = []
#actiontup = None
cyth_mode = False # used for #if/else/endif
param_typedict = {}
bodyvars_typedict = {}
return_type = None
has_markup = False
first_docstr = None
inline_flag_ptr = [False]
collect_macro_input = False
macro_input_buffer_ptr = [[]]
suspended_macro_context_ptr = [None]
for stmt in node.body:
if is_docstring(stmt):
docstr = stmt.value.s
if first_docstr is None:
first_docstr = docstr
has_markup = has_markup or docstr.find('#if CYTH') != -1
#actiontup = self.parse_cyth_markup(docstr, funcdef_node=node)
(source_lines, cyth_mode, collect_macro_input,
new_param_typedict, new_bodyvars_typedict,
new_return_type) = self.parse_cyth_preproc_markup(
docstr, cyth_mode, collect_macro_input,
macro_input_buffer_ptr, suspended_macro_context_ptr,
inline_flag_ptr)
#print('source_lines: %r' % (source_lines,))
new_body.extend(source_lines)
if new_return_type is not None and return_type is None:
return_type = new_return_type
param_typedict.update(new_param_typedict)
bodyvars_typedict.update(new_bodyvars_typedict)
else:
#print('cyth_mode: %r, stmt: %r' % (cyth_mode, ast.dump(stmt)))
if not (cyth_mode or collect_macro_input):
new_body.append(stmt)
if collect_macro_input:
macro_input_buffer_ptr[0].append(
cyth_helpers.ast_to_sourcecode(stmt))
if has_markup:
self.cythonized_funcs[node.name] = node
self.register_benchmark(node.name, first_docstr, self.py_modname)
union_typedict = {}
union_typedict.update(param_typedict)
union_typedict.update(bodyvars_typedict)
if return_type is None:
return_type = infer_return_type(node, union_typedict)
self.newline(extra=1)
cyth_funcname = cyth_helpers.get_cyth_name(node.name)
# TODO: should allow for user specification
func_prefix = utool.unindent('''
@cython.boundscheck(False)
@cython.wraparound(False)
''').strip()
# http://docs.cython.org/src/reference/compilation.html#compiler-directives
# [(name, defaultval), ...]
cython_compiler_directives = [ # NOQA
('boundscheck', True),
('wraparound', True),
('nonecheck', False),
('overflowcheck', False),
(
'overflowcheck.fold', True
), # may help or hurt depending on compiler, arch, and opt settings
('embedsignature', False),
('cdivision', False),
('cdivision_warnings', False),
('always_allow_keywords', None),
('profile', False),
('linetrace', False),
('invert_types', None),
('language_level', 2),
('c_string_type', bytes),
('c_string_encoding', 'ascii'),
('type_version_tag', True),
('unraisable_tracebacks', None),
]
return_string = (" %s " %
return_type) if return_type is not None else " "
self.statement(node, func_prefix + '\n')
# HACK: indexing is used to extract a portion of the generated stream, which wouldn't
# be needed if statement/write/etc all returned values rather than writing a stream
index_before = len(self.result)
inline_string = ' inline ' if inline_flag_ptr[0] else ''
self.write('cpdef%s%s%s(' % (
inline_string,
return_string,
cyth_funcname,
))
nonsig_typedict = self.signature(node.args,
typedict=union_typedict)
cyth_def_body = self.typedict_to_cythdef(nonsig_typedict)
self.write(')')
function_signature = ''.join(self.result[index_before:])
self.interface_lines.append(function_signature)
self.write(':')
# TODO FIXME: the typedict parser is a giant hack right now.
# Find a good cython parser
self.indentation += 1
#cyth_def_body = self.typedict_to_cythdef(bodyvars_typedict)
for s in cyth_def_body:
self.write('\n', s)
self.write('\n')
self.indentation -= 1
self.body(new_body)
else:
self.plain_funcs[node.name] = node
def write_explicit(pyth_modname, execstr):
import multiprocessing
is_main_proc = multiprocessing.current_process().name == 'MainProcess'
if is_main_proc:
from utool import util_str
from os.path import exists
new_else = util_str.indent(execstr)
#print(new_else)
# Get path to init file so we can overwrite it
pyth_fpath = pyth_modname
module = sys.modules[pyth_modname] # module currently being imported
modpath, ext = splitext(module.__file__)
assert ext == '.pyc' or ext == '.py'
pyth_fpath = modpath + '.py'
#import IPython
#IPython.embed()
print("attempting to update: %r" % pyth_fpath)
assert exists(pyth_fpath)
new_lines = []
broken = False
closed = False
start_sentinal = ' # <AUTOGEN_CYTH>'
end_sentinal = ' # </AUTOGEN_CYTH>'
with open(pyth_fpath, 'r') as file_:
lines = file_.readlines()
for line in lines:
if not closed and not broken:
# Append lines until you see start_sentinal
new_lines.append(line)
if line.startswith(start_sentinal):
indent = ' '
help_line = '# Regen command: python -c "import %s" --cyth-write\n' % pyth_modname
new_lines.append(indent + help_line)
new_lines.append(new_else + '\n')
broken = True
elif not closed and broken:
# Skip lines between sentinals
if line.startswith(end_sentinal):
new_lines.append(end_sentinal + '\n')
closed = True
elif closed and broken:
# Append lines after sentinals
new_lines.append(line)
else:
raise AssertionError('closed before opening cyth tags')
if broken and closed:
print("writing updated file: %r" % pyth_fpath)
new_text = ''.join(new_lines)
#print(new_text)
with open(pyth_fpath, 'w') as file_:
file_.write(new_text)
else:
default_cyth_block = utool.unindent('''
import cyth
if cyth.DYNAMIC:
exec(cyth.import_cyth_execstr(__name__))
else:
# <AUTOGEN_CYTH>
# </AUTOGEN_CYTH>
pass
''')
default_cyth_block # NOQA
print("no write hook for file: %r" % pyth_fpath)
def get_bench_text_fmt():
bench_text_fmt_ = r'''
" Autogenerated by cyth on {timestamp} "
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import timeit
import textwrap
import warnings
import utool
import six
warnings.simplefilter('ignore', SyntaxWarning)
warnings.simplefilter('ignore', RuntimeWarning)
print, print_, printDBG, rrr, profile = utool.inject(__name__, '[{py_modname}.bench]')
def run_doctest(pyth_call, cyth_call, setup_script):
setup_globals_py, setup_locals_py = {{}}, {{}}
setup_globals_cy, setup_locals_cy = {{}}, {{}}
six.exec_(setup_script, setup_globals_py, setup_locals_py)
six.exec_(setup_script, setup_globals_cy, setup_locals_cy)
pyth_result = eval(cyth_call, setup_globals_py, setup_locals_py)
cyth_result = eval(cyth_call, setup_globals_cy, setup_locals_cy)
if repr(pyth_result) == repr(cyth_result):
#print('%r and %r have the same result.' % (pyth_call, cyth_call))
print('PASS: output is equivalent')
#%r and %r have the same result.' % (pyth_call, cyth_call))
else:
print('<FAILED>')
print('INCONSISTENCY: %r has different output than %r' % (pyth_call, cyth_call))
print('___________')
print('pyth_result')
print('-----------')
print(repr(pyth_result))
print('=========== (end pyth_result)')
print('___________')
print('cyth_result')
print('-----------')
print(repr(cyth_result))
print('=========== (end cyth_result)')
print('</FAILED>')
{codes}
def run_all_benchmarks(iterations):
print('\n\n')
print('=======================================')
print('[cyth] Run benchmarks for: {py_modname}')
with utool.Indenter(' * '):
results = []
{all_benchmarks}
#sorted_results = sorted(results)
#sorted_lines = [tup[2] for tup in sorted_results]
#print('\n'.join(utool.flatten(sorted_lines)))
return results
if __name__ == '__main__':
iterations = utool.get_argval(('--iterations', '-n'), type_=int, default=100)
run_all_benchmarks(iterations)
'''
bench_text_fmt = utool.unindent(bench_text_fmt_).strip('\n')
return bench_text_fmt
def visit_FunctionDef(self, node):
"""
Gets called when parser encounters a function definition
"""
#super(CythVisitor, self).visit_FunctionDef(node)
new_body = []
#actiontup = None
cyth_mode = False # used for #if/else/endif
param_typedict = {}
bodyvars_typedict = {}
return_type = None
has_markup = False
first_docstr = None
inline_flag_ptr = [False]
collect_macro_input = False
macro_input_buffer_ptr = [[]]
suspended_macro_context_ptr = [None]
for stmt in node.body:
if is_docstring(stmt):
docstr = stmt.value.s
if first_docstr is None:
first_docstr = docstr
has_markup = has_markup or docstr.find('#if CYTH') != -1
#actiontup = self.parse_cyth_markup(docstr, funcdef_node=node)
(source_lines, cyth_mode, collect_macro_input,
new_param_typedict, new_bodyvars_typedict,
new_return_type) = self.parse_cyth_preproc_markup(
docstr, cyth_mode, collect_macro_input,
macro_input_buffer_ptr, suspended_macro_context_ptr,
inline_flag_ptr)
#print('source_lines: %r' % (source_lines,))
new_body.extend(source_lines)
if new_return_type is not None and return_type is None:
return_type = new_return_type
param_typedict.update(new_param_typedict)
bodyvars_typedict.update(new_bodyvars_typedict)
else:
#print('cyth_mode: %r, stmt: %r' % (cyth_mode, ast.dump(stmt)))
if not (cyth_mode or collect_macro_input):
new_body.append(stmt)
if collect_macro_input:
macro_input_buffer_ptr[0].append(cyth_helpers.ast_to_sourcecode(stmt))
if has_markup:
self.cythonized_funcs[node.name] = node
self.register_benchmark(node.name, first_docstr, self.py_modname)
union_typedict = {}
union_typedict.update(param_typedict)
union_typedict.update(bodyvars_typedict)
if return_type is None:
return_type = infer_return_type(node, union_typedict)
self.newline(extra=1)
cyth_funcname = cyth_helpers.get_cyth_name(node.name)
# TODO: should allow for user specification
func_prefix = utool.unindent('''
@cython.boundscheck(False)
@cython.wraparound(False)
''').strip()
# http://docs.cython.org/src/reference/compilation.html#compiler-directives
# [(name, defaultval), ...]
cython_compiler_directives = [ # NOQA
('boundscheck', True),
('wraparound', True),
('nonecheck', False),
('overflowcheck', False),
('overflowcheck.fold', True), # may help or hurt depending on compiler, arch, and opt settings
('embedsignature', False),
('cdivision', False),
('cdivision_warnings', False),
('always_allow_keywords', None),
('profile', False),
('linetrace', False),
('invert_types', None),
('language_level', 2),
('c_string_type', bytes),
('c_string_encoding', 'ascii'),
('type_version_tag', True),
('unraisable_tracebacks', None),
]
return_string = (" %s " % return_type) if return_type is not None else " "
self.statement(node, func_prefix + '\n')
# HACK: indexing is used to extract a portion of the generated stream, which wouldn't
# be needed if statement/write/etc all returned values rather than writing a stream
index_before = len(self.result)
inline_string = ' inline ' if inline_flag_ptr[0] else ''
self.write('cpdef%s%s%s(' % (inline_string, return_string, cyth_funcname,))
nonsig_typedict = self.signature(node.args, typedict=union_typedict)
cyth_def_body = self.typedict_to_cythdef(nonsig_typedict)
self.write(')')
function_signature = ''.join(self.result[index_before:])
self.interface_lines.append(function_signature)
self.write(':')
# TODO FIXME: the typedict parser is a giant hack right now.
# Find a good cython parser
self.indentation += 1
#cyth_def_body = self.typedict_to_cythdef(bodyvars_typedict)
for s in cyth_def_body:
self.write('\n', s)
self.write('\n')
self.indentation -= 1
self.body(new_body)
else:
self.plain_funcs[node.name] = node
def TIME_GEN_PREPROC_FEAT(ibs):
print('[TIME_GEN_PREPROC_FEAT]')
from ibeis.algo.preproc.preproc_feat import extract_hesaff_sift_feats
from six.moves import zip
import numpy as np
def _listeq(x1, x2):
if isinstance(x1, np.ndarray):
return np.all(x2 == x2)
return x1 == x2
aid_list = ibs.get_valid_aids()
cid_list = ibs.get_annot_chip_rowids(aid_list)
cfpath_list = ibs.get_chip_fpath(cid_list)
# STABILITY
if not utool.get_argflag('--nostable'):
# TEST 1
with utool.Timer('parallel chunksize=1'):
output1 = list(extract_hesaff_sift_feats(cfpath_list, chunksize=1))
print(utool.truncate_str(str(output1), 80))
# TEST 2
with utool.Timer('parallel chunksize=2'):
output2 = list(extract_hesaff_sift_feats(cfpath_list, chunksize=2))
print(utool.truncate_str(str(output2), 80))
assert all([
_listeq(*xtup) for tup in zip(output1, output2)
for xtup in zip(*tup)
]), 'chuncksize changes output'
# TEST N
with utool.Timer('parallel chunksize=None'):
outputN = list(
extract_hesaff_sift_feats(cfpath_list, chunksize=None))
print(utool.truncate_str(str(output2), 80))
assert all([
_listeq(*xtup) for tup in zip(outputN, output2)
for xtup in zip(*tup)
]), 'chuncksize changes output'
# BENCHMARK
setup = utool.unindent('''
from ibeis.algo.preproc.preproc_feat import extract_hesaff_sift_feats
genkw = dict(prog=False, verbose=True)
cfpath_list = %r
''' % (cfpath_list, ))
print(utool.truncate_str(str(cid_list), 80))
print('Processing %d chips' % (len(cid_list), ))
timeit3 = partial(timeit2, setup=setup, number=1)
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))'
)
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))'
)
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=1, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=2, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=4, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=8, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=16, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=32, **genkw))')
timeit3(
'list(extract_hesaff_sift_feats(cfpath_list, chunksize=64, **genkw))')
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))
#[parallel] initializing pool with 7 processes
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=21
# * timed: 125.17100650510471 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=21
# * timed: 97.37531812573734 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=1, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=1
# * timed: 89.11060989484363 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=2, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=2
# * timed: 89.3294122591355 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=4, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=4
# * timed: 114.7752637914524 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=8, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=8
# * timed: 123.35112345890252 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=16, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=16
# * timed: 124.47361485097099 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=32, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=32
# * timed: 126.47238857719219 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=64, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=64
# * timed: 137.3404114996564 seconds
print('[/TIME_GEN_PREPROC_FEAT]')
return locals()
def get_bench_text_fmt():
bench_text_fmt_ = r'''
" Autogenerated by cyth on {timestamp} "
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import timeit
import textwrap
import warnings
import utool
import six
warnings.simplefilter('ignore', SyntaxWarning)
warnings.simplefilter('ignore', RuntimeWarning)
print, print_, printDBG, rrr, profile = utool.inject(__name__, '[{py_modname}.bench]')
def run_doctest(pyth_call, cyth_call, setup_script):
setup_globals_py, setup_locals_py = {{}}, {{}}
setup_globals_cy, setup_locals_cy = {{}}, {{}}
six.exec_(setup_script, setup_globals_py, setup_locals_py)
six.exec_(setup_script, setup_globals_cy, setup_locals_cy)
pyth_result = eval(cyth_call, setup_globals_py, setup_locals_py)
cyth_result = eval(cyth_call, setup_globals_cy, setup_locals_cy)
if repr(pyth_result) == repr(cyth_result):
#print('%r and %r have the same result.' % (pyth_call, cyth_call))
print('PASS: output is equivalent')
#%r and %r have the same result.' % (pyth_call, cyth_call))
else:
print('<FAILED>')
print('INCONSISTENCY: %r has different output than %r' % (pyth_call, cyth_call))
print('___________')
print('pyth_result')
print('-----------')
print(repr(pyth_result))
print('=========== (end pyth_result)')
print('___________')
print('cyth_result')
print('-----------')
print(repr(cyth_result))
print('=========== (end cyth_result)')
print('</FAILED>')
{codes}
def run_all_benchmarks(iterations):
print('\n\n')
print('=======================================')
print('[cyth] Run benchmarks for: {py_modname}')
with utool.Indenter(' * '):
results = []
{all_benchmarks}
#sorted_results = sorted(results)
#sorted_lines = [tup[2] for tup in sorted_results]
#print('\n'.join(utool.flatten(sorted_lines)))
return results
if __name__ == '__main__':
iterations = utool.get_argval(('--iterations', '-n'), type_=int, default=100)
run_all_benchmarks(iterations)
'''
bench_text_fmt = utool.unindent(bench_text_fmt_).strip('\n')
return bench_text_fmt
def TIME_GEN_PREPROC_FEAT(ibs):
print('[TIME_GEN_PREPROC_FEAT]')
from ibeis.algo.preproc.preproc_feat import extract_hesaff_sift_feats
from six.moves import zip
import numpy as np
def _listeq(x1, x2):
if isinstance(x1, np.ndarray):
return np.all(x2 == x2)
return x1 == x2
aid_list = ibs.get_valid_aids()
cid_list = ibs.get_annot_chip_rowids(aid_list)
cfpath_list = ibs.get_chip_fpath(cid_list)
# STABILITY
if not utool.get_argflag('--nostable'):
# TEST 1
with utool.Timer('parallel chunksize=1'):
output1 = list(extract_hesaff_sift_feats(cfpath_list, chunksize=1))
print(utool.truncate_str(str(output1), 80))
# TEST 2
with utool.Timer('parallel chunksize=2'):
output2 = list(extract_hesaff_sift_feats(cfpath_list, chunksize=2))
print(utool.truncate_str(str(output2), 80))
assert all([_listeq(*xtup) for tup in zip(output1, output2)
for xtup in zip(*tup)]), 'chuncksize changes output'
# TEST N
with utool.Timer('parallel chunksize=None'):
outputN = list(extract_hesaff_sift_feats(cfpath_list, chunksize=None))
print(utool.truncate_str(str(output2), 80))
assert all([_listeq(*xtup) for tup in zip(outputN, output2)
for xtup in zip(*tup)]), 'chuncksize changes output'
# BENCHMARK
setup = utool.unindent(
'''
from ibeis.algo.preproc.preproc_feat import extract_hesaff_sift_feats
genkw = dict(prog=False, verbose=True)
cfpath_list = %r
''' % (cfpath_list,))
print(utool.truncate_str(str(cid_list), 80))
print('Processing %d chips' % (len(cid_list),))
timeit3 = partial(timeit2, setup=setup, number=1)
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=1, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=2, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=4, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=8, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=16, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=32, **genkw))')
timeit3('list(extract_hesaff_sift_feats(cfpath_list, chunksize=64, **genkw))')
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))
#[parallel] initializing pool with 7 processes
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=21
# * timed: 125.17100650510471 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=None, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=21
# * timed: 97.37531812573734 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=1, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=1
# * timed: 89.11060989484363 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=2, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=2
# * timed: 89.3294122591355 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=4, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=4
# * timed: 114.7752637914524 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=8, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=8
# * timed: 123.35112345890252 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=16, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=16
# * timed: 124.47361485097099 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=32, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=32
# * timed: 126.47238857719219 seconds
#----------
#list(extract_hesaff_sift_feats(cfpath_list, chunksize=64, **genkw))
#[parallel] executing 1049 gen_feat_worker tasks using 7 processes with chunksize=64
# * timed: 137.3404114996564 seconds
print('[/TIME_GEN_PREPROC_FEAT]')
return locals()
return aid_list
def list_ingestable_images(img_dir, fullpath=True, recursive=True):
ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
gpath_list = utool.list_images(img_dir,
fullpath=fullpath,
recursive=recursive,
ignore_list=ignore_list)
# Ensure in unix format
gpath_list = map(utool.unixpath, gpath_list)
return gpath_list
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support() # win32
print('__main__ = ingest_database.py')
print(utool.unindent(
'''
usage:
./ibeis/ingest/ingest_database.py --db [dbname]
Valid dbnames:''') + utool.indentjoin(STANDARD_INGEST_FUNCS.keys(), '\n * '))
db = utool.get_arg('--db', str, None)
ibs = ingest_standard_database(db)
#img_dir = join(ibeis.sysres.get_workdir(), 'polar_bears')
#main_locals = ibeis.main(dbdir=img_dir, gui=False)
#ibs = main_locals['ibs']
#ingest_rawdata(ibs, img_dir)