def get_table_csv_header(db, tablename):
column_nametypes = db.table_columns[tablename]
header_constraints = '# CONSTRAINTS: %r' % db.table_constraints[tablename]
header_name = '# TABLENAME: %r' % tablename
header_types = utool.indentjoin(column_nametypes, '\n# ')
header_doc = utool.indentjoin(utool.unindent(db.table_docstr[tablename]).split('\n'), '\n# ')
header = header_doc + '\n' + header_name + header_types + '\n' + header_constraints
return header
def find_requested_hrefs(all_href_list, py_version, pkg_list):
"""
Filters out everything but the requested urls
Returns the urls to download the requested installers
"""
print('Filtering to only requested HREFS')
href_list1, missing1 = filter_href_list(all_href_list, pkg_list, OS_VERSION, py_version)
#print('missing1 = %r' % (missing1,))
href_list2, missing2 = filter_href_list(all_href_list, missing1, OS_VERSION, py_version)
#print('missing2 = %r' % (missing2,))
#print(href_list2)
href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64', py_version.replace('p', 'P'))
#print('missing3 = %r' % (missing3,))
href_list4, missing4 = filter_href_list(all_href_list, missing3, 'any', py_version.replace('cp', 'py')[0:3])
if len(missing4) > 0:
print('Could not find a match for missing4=%r' % (missing4,))
#import Levenshtein
for pkg in missing4:
#dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list]
dist_list = [0 if (href.find(pkg) > -1) else 100 for href in all_href_list]
closest_matche_xs = ut.list_argsort(dist_list)[::1]
print('Perhaps pkg=%r could match one of these?' % (pkg,))
closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3])
print(ut.indentjoin(closest_hrefs, '\n '))
href_list = href_list1 + href_list2 + href_list3 + href_list4
return href_list
def find_requested_hrefs(all_href_list, py_version, pkg_list):
"""
Filters out everything but the requested urls
Returns the urls to download the requested installers
"""
print('Filtering to only requested HREFS')
href_list1, missing1 = filter_href_list(all_href_list, pkg_list,
OS_VERSION, py_version)
#print('missing1 = %r' % (missing1,))
href_list2, missing2 = filter_href_list(all_href_list, missing1,
OS_VERSION, py_version)
#print('missing2 = %r' % (missing2,))
#print(href_list2)
href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64',
py_version.replace('p', 'P'))
#print('missing3 = %r' % (missing3,))
href_list4, missing4 = filter_href_list(
all_href_list, missing3, 'any',
py_version.replace('cp', 'py')[0:3])
if len(missing4) > 0:
print('Could not find a match for missing4=%r' % (missing4, ))
#import Levenshtein
for pkg in missing4:
#dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list]
dist_list = [
0 if (href.find(pkg) > -1) else 100 for href in all_href_list
]
closest_matche_xs = ut.list_argsort(dist_list)[::1]
print('Perhaps pkg=%r could match one of these?' % (pkg, ))
closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3])
print(ut.indentjoin(closest_hrefs, '\n '))
href_list = href_list1 + href_list2 + href_list3 + href_list4
return href_list
def db_to_dbdir(db, allow_newdir=False, extra_workdirs=[], use_sync=False):
""" Implicitly gets dbdir. Searches for db inside of workdir """
work_dir = get_workdir()
dbalias_dict = get_dbalias_dict()
workdir_list = []
for extra_dir in extra_workdirs:
if exists(extra_dir):
workdir_list.append(extra_dir)
if use_sync:
sync_dir = join(work_dir, '../sync')
if exists(sync_dir):
workdir_list.append(sync_dir)
workdir_list.append(work_dir) # TODO: Allow multiple workdirs
# Check all of your work directories for the database
for _dir in workdir_list:
dbdir = realpath(join(_dir, db))
# Use db aliases
if not exists(dbdir) and db.upper() in dbalias_dict:
dbdir = join(_dir, dbalias_dict[db.upper()])
if exists(dbdir):
break
# Create the database if newdbs are allowed in the workdir
#print('allow_newdir=%r' % allow_newdir)
if allow_newdir:
utool.ensuredir(dbdir, verbose=True)
# Complain if the implicit dbdir does not exist
if not exists(dbdir):
print('!!!')
print('[sysres] WARNING: db=%r not found in work_dir=%r' %
(db, work_dir))
fname_list = os.listdir(work_dir)
lower_list = [fname.lower() for fname in fname_list]
index = util_list.listfind(lower_list, db.lower())
if index is not None:
print('[sysres] WARNING: db capitalization seems to be off')
if not utool.STRICT:
print('[sysres] attempting to fix it')
db = fname_list[index]
dbdir = join(work_dir, db)
print('[sysres] dbdir=%r' % dbdir)
print('[sysres] db=%r' % db)
if not exists(dbdir):
msg = '[sysres!] ERROR: Database does not exist'
print('<!!!>')
print(msg)
print('[sysres!] Here is a list of valid dbs: ' +
utool.indentjoin(fname_list, '\n * '))
print('[sysres!] dbdir=%r' % dbdir)
print('[sysres!] db=%r' % db)
print('[sysres!] work_dir=%r' % work_dir)
print('</!!!>')
raise AssertionError(msg)
print('!!!')
return dbdir
def import_cyth_execstr(pyth_modname):
"""
>>> from cyth.cyth_importer import * # NOQA
>>> from vtool import trig # NOQA
>>> pyth_modname = 'vtool.trig'
"""
dummy_cythonized_funcs = import_cyth_default(pyth_modname)
pyth_list = []
for funcname, func in dummy_cythonized_funcs.items():
pyth_list.append(funcname + ' = ' + get_funcname(func))
pyth_list2 = utool.align_lines(sorted(pyth_list), '=')
try:
cyth_list = []
pkgname, fromlist, cyth_modname = pkg_submodule_split(pyth_modname)
cythonized_funcs = get_cythonized_funcs(pyth_modname)
for funcname, func in cythonized_funcs.items():
cyth_list.append(funcname + ' = ' + cyth_modname + '.' + func.__name__)
cyth_list2 = ['import ' + cyth_modname] + utool.align_lines(sorted(cyth_list), '=')
except ImportError:
cyth_list2 = ['raise ImportError("no cyth")']
except Exception as ex:
cyth_list2 = ['raise ImportError("cyth import error: %s")' % str(ex)]
cyth_block = utool.indentjoin(cyth_list2).strip()
pyth_block = utool.indentjoin(pyth_list2).strip()
execstr = utool.unindent(
'''
try:
if not cyth.WITH_CYTH:
raise ImportError('no cyth')
{cyth_block}
CYTHONIZED = True
# print('cyth is on in %s' % (__name__,))
except ImportError:
{pyth_block}
# print('cyth is off in %s' % (__name__,))
CYTHONIZED = False''').format(**locals()).strip('\n')
#print(execstr)
if cyth_args.CYTH_WRITE:
write_explicit(pyth_modname, execstr)
return execstr
def boxjoin(list_, header=None):
topline = '+----------'
botline = 'L__________'
boxlines = []
boxlines.append(topline + '\n')
if header is not None:
boxlines.append(header + '\n')
boxlines.append(topline)
body = utool.indentjoin(list_, '\n | ')
boxlines.append(body + '\n ')
boxlines.append(botline + '\n')
return ''.join(boxlines)
def boxjoin(list_, header=None):
topline = '+----------'
botline = 'L__________'
boxlines = []
boxlines.append(topline + '\n')
if header is not None:
boxlines.append(header + '\n')
boxlines.append(topline)
body = utool.indentjoin(list_, '\n | ')
boxlines.append(body + '\n ')
boxlines.append(botline + '\n')
return ''.join(boxlines)
def get_cachestats_str(ibs):
"""
Returns info about the underlying SQL cache memory
"""
total_size_str = ut.get_object_size_str(ibs.table_cache,
lbl='size(table_cache): ')
total_size_str = '\nlen(table_cache) = %r' % (len(ibs.table_cache))
table_size_str_list = [
ut.get_object_size_str(val, lbl='size(table_cache[%s]): ' % (key,))
for key, val in six.iteritems(ibs.table_cache)]
cachestats_str = (
total_size_str + ut.indentjoin(table_size_str_list, '\n * '))
return cachestats_str
def get_cachestats_str(ibs):
"""
Returns info about the underlying SQL cache memory
"""
total_size_str = ut.get_object_size_str(ibs.table_cache,
lbl='size(table_cache): ')
total_size_str = '\nlen(table_cache) = %r' % (len(ibs.table_cache))
table_size_str_list = [
ut.get_object_size_str(val, lbl='size(table_cache[%s]): ' % (key,))
for key, val in six.iteritems(ibs.table_cache)]
cachestats_str = (
total_size_str + ut.indentjoin(table_size_str_list, '\n * '))
return cachestats_str
def delete_all_learned_normalizers():
r"""
DELETES ALL CACHED NORMALIZERS IN ALL DATABASES
CommandLine:
python -m ibeis.algo.hots.score_normalization --test-delete_all_learned_normalizers
#-y
Example:
>>> # DOCTEST_DISABLE
>>> from ibeis.algo.hots import score_normalization
>>> score_normalization.delete_all_learned_normalizers()
"""
from ibeis.algo.hots import score_normalization
import utool as ut
print('DELETE_ALL_LEARNED_NORMALIZERS')
normalizer_fpath_list = score_normalization.list_available_score_normalizers()
print('The following normalizers will be deleted: ' + ut.indentjoin(normalizer_fpath_list, '\n '))
if ut.are_you_sure('Deleting all learned normalizers'):
ut.remove_fpaths(normalizer_fpath_list, verbose=True)
def injest_main():
r"""
CommandLine:
python -m ibeis.dbio.ingest_database --test-injest_main
python -m ibeis.dbio.ingest_database --test-injest_main --db snow-leopards
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.dbio.ingest_database import * # NOQA
>>> injest_main()
"""
print('__main__ = ingest_database.py')
print(ut.unindent(
'''
usage:
python ibeis/ingest/ingest_database.py --db [dbname]
Valid dbnames:''') + ut.indentjoin(STANDARD_INGEST_FUNCS.keys(), '\n * '))
dbname = ut.get_argval('--db', str, None)
force_delete = ut.get_argflag(('--force_delete', '--force-delete'))
ibs = ingest_standard_database(dbname, force_delete) # NOQA
print('finished db injest')
def db_to_dbdir(db, allow_newdir=False, extra_workdirs=[], use_sync=False):
""" Implicitly gets dbdir. Searches for db inside of workdir """
if ut.VERBOSE:
print('[sysres] db_to_dbdir: db=%r, allow_newdir=%r' %
(db, allow_newdir))
work_dir = get_workdir()
dbalias_dict = get_dbalias_dict()
workdir_list = []
for extra_dir in extra_workdirs:
if exists(extra_dir):
workdir_list.append(extra_dir)
if use_sync:
sync_dir = join(work_dir, '../sync')
if exists(sync_dir):
workdir_list.append(sync_dir)
workdir_list.append(work_dir) # TODO: Allow multiple workdirs
# Check all of your work directories for the database
for _dir in workdir_list:
dbdir = realpath(join(_dir, db))
# Use db aliases
if not exists(dbdir) and db.upper() in dbalias_dict:
dbdir = join(_dir, dbalias_dict[db.upper()])
if exists(dbdir):
break
# Create the database if newdbs are allowed in the workdir
#print('allow_newdir=%r' % allow_newdir)
if allow_newdir:
ut.ensuredir(dbdir, verbose=True)
# Complain if the implicit dbdir does not exist
if not exists(dbdir):
print('!!!')
print('[sysres] WARNING: db=%r not found in work_dir=%r' %
(db, work_dir))
fname_list = os.listdir(work_dir)
lower_list = [fname.lower() for fname in fname_list]
index = util_list.listfind(lower_list, db.lower())
if index is not None:
print('[sysres] WARNING: db capitalization seems to be off')
if not ut.STRICT:
print('[sysres] attempting to fix it')
db = fname_list[index]
dbdir = join(work_dir, db)
print('[sysres] dbdir=%r' % dbdir)
print('[sysres] db=%r' % db)
if not exists(dbdir):
msg = '[sysres!] ERROR: Database does not exist and allow_newdir=False'
print('<!!!>')
print(msg)
print('[sysres!] Here is a list of valid dbs: ' +
ut.indentjoin(sorted(fname_list), '\n * '))
print('[sysres!] dbdir=%r' % dbdir)
print('[sysres!] db=%r' % db)
print('[sysres!] work_dir=%r' % work_dir)
print('</!!!>')
raise AssertionError(msg)
print('!!!')
return dbdir
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = ['ibeis/tests/assert_modules.py']
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/ibeis'
testdir = 'ibeis/tests'
exclude_list = []
# Hacky, but not too bad way of getting in doctests
# Test to see if doctest_funcs appears after main
# Do not doctest these modules
#exclude_doctests_fnames = set(['template_definitions.py',
# 'autogen_test_script.py'])
#exclude_dirs = [
# '_broken',
# 'old',
# 'tests',
# 'timeits',
# '_scripts',
# '_timeits',
# '_doc',
# 'notebook',
#]
#dpath_list = ['ibeis']
#doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
# The implict list is exactly the code we will use to make the implicit list
#module_list = None
#doctest_modname_list = None
#'export_subset.py',
#'_autogen_ibeiscontrol_funcs.py',
#'randomforest.py',
implicit_build_modlist_str = ut.codeblock('''
import sys
exclude_doctests_fnames = set([
'template_definitions.py',
'autogen_test_script.py',
])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['ibeis']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
''')
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
#module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list]
#for modname in doctest_modname_list:
# exec('import ' + modname, globals(), locals())
#module_list = [sys.modules[name] for name in doctest_modname_list]
#print('\n'.join(testcmds))
#print('\n'.join(['python -m ' + modname for modname in doctest_modname_list]))
import_str = '\n'.join(
['import ' + modname for modname in doctest_modname_list])
modlist_str = (
'module_list = [%s\n]' %
ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
''')
pyscript_text = pyscript_fmtstr.format(
build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds_ = ut.get_module_testlines(module_list,
remove_pyc=True,
verbose=False,
pythoncmd='RUN_TEST',
testslow=True)
testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_]
test_headers = [
# title, default, module, testpattern
ut.def_test('VTOOL',
dpath='vtool/tests',
pat=['test*.py'],
modname='vtool'),
ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']),
ut.def_test('IBEIS',
dpath=testdir,
pat=['test_ibs*.py', 'test_delete*.py'],
default=False),
ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']),
ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']),
ut.def_test('MISC', dpath=testdir, pat=misc_pats),
ut.def_test('OTHER', dpath=testdir, pat='OTHER'),
ut.def_test('HESAFF',
dpath='pyhesaff/tests',
pat=['test_*.py'],
modname='pyhesaff'),
ut.def_test('DOC', testcmds=testcmds, default=True)
]
# Referencs: https://docs.python.org/2/library/runpy.html
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs,
quick_tests, repodir,
exclude_list)
#print(pyscript_text)
return shscript_text, pyscript_text
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = [
'ibeis/tests/assert_modules.py'
]
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/utool'
exclude_list = []
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
implicit_build_modlist_str = ut.codeblock(
'''
import sys
exclude_doctests_fnames = set(['__init__.py'])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['utool']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
'''
)
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list])
modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(
r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
'''
)
pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
def def_test(header, pat=None, dpath=None, modname=None, default=False, testcmds=None):
""" interface to make test tuple """
return (header, default, modname, dpath, pat, testcmds)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds = ut.get_module_testlines(module_list, remove_pyc=True, verbose=False, pythoncmd='RUN_TEST')
test_headers = [
# title, default, module, testpattern
def_test('DOC', testcmds=testcmds, default=True)
]
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list)
return shscript_text, pyscript_text
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None,
repodir=None, exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(
r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(
r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(
r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "',
'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")']).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(header_lower=header_lower,
testflag=testflag,)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten([ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any([x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list]
testline_list = [format_testline(fpath, dirvar) for fpath in testfpath_list]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,)
header_test_block = header_test_block_fmstr.format(**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs, dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
def _pack(bits, errs, thresh):
return utool.indentjoin(['%5s %f < %f' % (bit, err, thresh) for
(bit, err) in zip(bits, errs)])
if __name__ == '__main__':
CODE_DIR = get_codedir()
rman = ut.RepoManager(repo_urls=[
'https://github.com/Erotemic/utool.git',
'https://github.com/Erotemic/guitool.git',
'https://github.com/Erotemic/plottool.git',
'https://github.com/Erotemic/vtool.git',
'https://github.com/bluemellophone/detecttools.git',
'https://github.com/Erotemic/hesaff.git',
'https://github.com/bluemellophone/pyrf.git',
'https://github.com/Erotemic/ibeis.git',
'https://github.com/aweinstock314/cyth.git',
#'https://github.com/hjweide/pygist',
], code_dir=CODE_DIR)
# (IBEIS_REPO_URLS, IBEIS_REPO_DIRS) = ut.repo_list(, forcessh=False)
# ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS)
dpath_list = rman.repo_dirs
# IBEIS_REPO_DIRS
fname = ut.truepath(sys.argv[1])
#if len(sys.argv) >= 3:
# grep_dpath = ut.truepath(sys.argv[2])
print('Classfuncs of %r' % fname)
funcname_list = ut.list_class_funcnames(fname)
funcname_list = ut.list_global_funcnames(fname)
print(ut.indentjoin(funcname_list, '\n * '))
show_function_usage(fname, funcname_list, dpath_list)
def auto_docstr(**kwargs):
import imp
import utool as ut
ut.util_dbg.COLORED_EXCEPTIONS = False
ut.ENABLE_COLORS = False
ut.util_str.ENABLE_COLORS = False
try:
print("RELOADING UTOOL via imp")
imp.reload(ut)
imp.reload(ut._internal.meta_util_arg)
except Exception as ex:
print("... errored")
pass
print("RELOADING UTOOL via rrrr")
ut.rrrr(verbose=0)
imp.reload(ut)
import vim
modname = None
funcname = None
flag = False
dbgtext = ''
docstr = ''
dbgmsg = ''
try:
funcname, searchlines, pos, foundline = find_pyfunc_above_cursor()
modname, moddir = get_current_modulename()
modpath = vim.current.buffer.name
print('modpath = {!r}'.format(modpath))
if funcname is None:
funcname = '[vimerr] UNKNOWN_FUNC: funcname is None'
flag = True
else:
# Text to insert into the current buffer
verbose = True
autodockw = dict(verbose=verbose)
autodockw.update(kwargs)
docstr = ut.auto_docstr(modname, funcname, moddir=moddir,
modpath=modpath, **autodockw)
#if docstr.find('unexpected indent') > 0:
# docstr = funcname + ' ' + docstr
if docstr[:].strip() == 'error':
flag = True
except vim.error as ex:
dbgmsg = 'vim_error: ' + str(ex)
flag = False
except Exception as ex:
dbgmsg = 'exception(%r): %s' % (type(ex), str(ex))
ut.printex(ex, tb=True)
flag = False
if flag:
dbgtext += '\n+======================'
dbgtext += '\n| --- DEBUG OUTPUT --- '
if len(dbgmsg) > 0:
dbgtext += '\n| Message: '
dbgtext += dbgmsg
dbgtext += '\n+----------------------'
dbgtext += '\n| InsertDoctstr(modname=%r, funcname=%r' % (modname, funcname)
pycmd = ('import ut; print(ut.auto_docstr(%r, %r, %r)))' % (modname, funcname, modpath))
pycmd = pycmd.replace('\'', '\\"')
dbgtext += '\n| python -c "%s"' % (pycmd,)
dbgtext += '\n+----------------------'
dbgtext += '\n+searchlines = '
dbgtext += ut.indentjoin(searchlines, '\n| ')
dbgtext += '\nL----------------------'
elif len(dbgmsg) > 0:
dbgtext += '\n| Message: '
dbgtext += dbgmsg
text = '\n'.join([docstr + dbgtext])
if text == '':
print('No Text! For some reason flag=%r' % (flag,))
return text
def embed(parent_locals=None, parent_globals=None, exec_lines=None,
remove_pyqt_hook=True, N=0):
"""
Starts interactive session. Similar to keyboard command in matlab.
Wrapper around IPython.embed
Args:
parent_locals (None):
parent_globals (None):
exec_lines (None):
remove_pyqt_hook (bool):
N (int):
CommandLine:
python -m utool.util_dbg --test-embed
References:
http://stackoverflow.com/questions/27911570/can-you-specify-a-command-to-run-after-you-embed-into-ipython/27914204#27914204
http://stackoverflow.com/questions/15167200/how-do-i-embed-an-ipython-interpreter-into-an-application-running-in-an-ipython
TODO:
try:
get_ipython
except NameError:
banner=exit_msg=''
else:
banner = '*** Nested interpreter ***'
exit_msg = '*** Back in main IPython ***'
# First import the embed function
from IPython.frontend.terminal.embed import InteractiveShellEmbed
# Now create the IPython shell instance. Put ipshell() anywhere in your code
# where you want it to open.
ipshell = InteractiveShellEmbed(banner1=banner, exit_msg=exit_msg)
#Then use ipshell() whenever you want to be dropped into an IPython shell. This
#will allow you to embed (and even nest) IPython interpreters in your code and
#inspect objects or the state of the program.
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_dbg import * # NOQA
>>> # build test data
>>> parent_locals = None
>>> parent_globals = None
>>> exec_lines = None
>>> remove_pyqt_hook = True
>>> N = 0
>>> # execute function
>>> result = embed(parent_locals, parent_globals, exec_lines, remove_pyqt_hook, N)
>>> # verify results
>>> print(result)
"""
if parent_globals is None:
parent_globals = get_parent_globals(N=N)
#parent_globals1 = get_parent_globals(N=0)
#exec(execstr_dict(parent_globals1, 'parent_globals1'))
if parent_locals is None:
parent_locals = get_parent_locals(N=N)
stackdepth = N # NOQA
import utool as ut
from functools import partial
getframe = partial(ut.get_caller_stack_frame, N=N) # NOQA
exec(execstr_dict(parent_globals, 'parent_globals'))
exec(execstr_dict(parent_locals, 'parent_locals'))
print('')
print('================')
print(ut.bubbletext('EMBEDING'))
print('================')
print('[util] embedding')
import IPython
try:
if remove_pyqt_hook:
try:
import guitool
guitool.remove_pyqt_input_hook()
except (ImportError, ValueError) as ex:
#print(ex)
printex(ex, iswarning=True)
pass
# make qt not loop forever (I had qflag loop forever with this off)
except ImportError as ex:
print(ex)
NEW_METHOD = False
if NEW_METHOD:
user_ns = globals()
user_ns = globals().copy()
user_ns.update(locals())
if parent_globals is not None:
user_ns.update(parent_globals)
if parent_locals is not None:
user_ns.update(parent_locals)
orig_argv = sys.argv # NOQA
print('About to start_ipython')
config = IPython.Config()
exec_lines_ = [
'%pylab qt4',
'print("Entered IPYTHON via utool")',
'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=11).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=10).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=9).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=8).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=7).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=6).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=5).f_code.co_name,))',
#execstr_dict(parent_locals)
] + ut.ensure_str_list(exec_lines if exec_lines is not None else [])
config.InteractiveShellApp.exec_lines = exec_lines_
print('Exec Lines: ')
print(ut.indentjoin(exec_lines_, '\n >>> '))
IPython.start_ipython(config=config, argv=[], user_ns=user_ns)
# Exit python immediately if specifed
if user_ns.get('qqq', False) or vars.get('qqq', False) or user_ns.get('EXIT_NOW', False):
print('[utool.embed] EXIT_NOW or qqq specified')
sys.exit(1)
else:
#from IPython.config.loader import Config
# cfg = Config()
#config_dict = {}
#if exec_lines is not None:
# config_dict['exec_lines'] = exec_lines
#IPython.embed(**config_dict)
print('[util] Get stack location with: ')
print('[util] ut.get_caller_stack_frame(N=8).f_code.co_name')
print('[util] set EXIT_NOW or qqq to True(ish) to hard exit on unembed')
#print('set iup to True to draw plottool stuff')
print('[util] call %pylab qt4 to get plottool stuff working')
once = True
# Allow user to set iup and redo the loop
while once or vars().get('iup', False):
if not once:
# SUPER HACKY WAY OF GETTING FIGURES ON THE SCREEN BETWEEN UPDATES
#vars()['iup'] = False
# ALL YOU NEED TO DO IS %pylab qt4
print('re-emebeding')
#import plottool as pt
#pt.update()
#(pt.present())
for _ in range(100):
time.sleep(.01)
once = False
#vars().get('iup', False):
print('[util] calling IPython.embed()')
"""
Notes:
/usr/local/lib/python2.7/dist-packages/IPython/terminal/embed.py
IPython.terminal.embed.InteractiveShellEmbed
# instance comes from IPython.config.configurable.SingletonConfigurable.instance
"""
IPython.embed()
#config = IPython.terminal.ipapp.load_default_config()
#config.InteractiveShellEmbed = config.TerminalInteractiveShell
#module = sys.modules[parent_globals['__name__']]
#config['module'] = module
#config['module'] = module
#embed2(stack_depth=N + 2 + 1)
#IPython.embed(config=config)
#IPython.embed(config=config)
#IPython.embed(module=module)
# Exit python immediately if specifed
if vars().get('EXIT_NOW', False) or vars().get('qqq', False):
print('[utool.embed] EXIT_NOW specified')
sys.exit(1)
return aid_list
def list_ingestable_images(img_dir, fullpath=True, recursive=True):
ignore_list = ['_hsdb', '.hs_internals', '_ibeis_cache', '_ibsdb']
gpath_list = utool.list_images(img_dir,
fullpath=fullpath,
recursive=recursive,
ignore_list=ignore_list)
# Ensure in unix format
gpath_list = map(utool.unixpath, gpath_list)
return gpath_list
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support() # win32
print('__main__ = ingest_database.py')
print(utool.unindent(
'''
usage:
./ibeis/ingest/ingest_database.py --db [dbname]
Valid dbnames:''') + utool.indentjoin(STANDARD_INGEST_FUNCS.keys(), '\n * '))
db = utool.get_arg('--db', str, None)
ibs = ingest_standard_database(db)
#img_dir = join(ibeis.sysres.get_workdir(), 'polar_bears')
#main_locals = ibeis.main(dbdir=img_dir, gui=False)
#ibs = main_locals['ibs']
#ingest_rawdata(ibs, img_dir)
def wildbook_signal_annot_name_changes(ibs, aid_list=None, tomcat_dpath=None, wb_target=None, dryrun=False):
r"""
Args:
aid_list (int): list of annotation ids(default = None)
tomcat_dpath (None): (default = None)
wb_target (None): (default = None)
dryrun (bool): (default = False)
CommandLine:
python -m ibeis.control.manual_wildbook_funcs --test-wildbook_signal_annot_name_changes:0 --dryrun
python -m ibeis.control.manual_wildbook_funcs --test-wildbook_signal_annot_name_changes:1 --dryrun
python -m ibeis.control.manual_wildbook_funcs --test-wildbook_signal_annot_name_changes:1
python -m ibeis.control.manual_wildbook_funcs --test-wildbook_signal_annot_name_changes:2
python -m ibeis --tf wildbook_signal_annot_name_changes:0 --dryrun
python -m ibeis --tf wildbook_signal_annot_name_changes:1 --dryrun
python -m ibeis --tf wildbook_signal_annot_name_changes:1
python -m ibeis --tf wildbook_signal_annot_name_changes:2
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control.manual_wildbook_funcs import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> #gid_list = ibs.get_valid_gids()[0:10]
>>> gid_list = ibs.get_valid_gids()[3:5]
>>> aid_list = ut.flatten(ibs.get_image_aids(gid_list))
>>> # Test case where some names change, some do not. There are no new names.
>>> old_nid_list = ibs.get_annot_name_rowids(aid_list)
>>> new_nid_list = ut.list_roll(old_nid_list, 1)
>>> ibs.set_annot_name_rowids(aid_list, new_nid_list)
>>> dryrun = ut.get_argflag('--dryrun')
>>> wb_target, tomcat_dpath = testdata_wildbook_server()
>>> result = ibs.wildbook_signal_annot_name_changes(aid_list, tomcat_dpath, wb_target, dryrun)
>>> ibs.set_annot_name_rowids(aid_list, old_nid_list)
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control.manual_wildbook_funcs import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> #gid_list = ibs.get_valid_gids()[0:10]
>>> gid_list = ibs.get_valid_gids()[3:5]
>>> aid_list = ut.flatten(ibs.get_image_aids(gid_list))
>>> # Test case where all names change to one known name
>>> #old_nid_list = ibs.get_annot_name_rowids(aid_list)
>>> #new_nid_list = [old_nid_list[0]] * len(old_nid_list)
>>> old_nid_list = [1, 2]
>>> new_nid_list = [1, 1]
>>> print('old_nid_list = %r' % (old_nid_list,))
>>> print('new_nid_list = %r' % (new_nid_list,))
>>> ibs.set_annot_name_rowids(aid_list, new_nid_list)
>>> dryrun = ut.get_argflag('--dryrun')
>>> wb_target, tomcat_dpath = testdata_wildbook_server()
>>> result = ibs.wildbook_signal_annot_name_changes(aid_list, tomcat_dpath, wb_target, dryrun)
>>> # Undo changes here (not undone in wildbook)
>>> #ibs.set_annot_name_rowids(aid_list, old_nid_list)
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.control.manual_wildbook_funcs import * # NOQA
>>> import ibeis
>>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
>>> gid_list = ibs.get_valid_gids()[3:5]
>>> aid_list = ut.flatten(ibs.get_image_aids(gid_list))
>>> old_nid_list = [1, 2]
>>> ibs.set_annot_name_rowids(aid_list, old_nid_list)
>>> # Signal what currently exists (should put them back to normal)
>>> dryrun = ut.get_argflag('--dryrun')
>>> wb_target, tomcat_dpath = testdata_wildbook_server()
>>> result = ibs.wildbook_signal_annot_name_changes(aid_list, tomcat_dpath, wb_target, dryrun)
"""
print('[ibs.wildbook_signal_eid_list()] signaling any annotation name changes to wildbook')
wildbook_base_url, wildbook_tomcat_path = ibs.get_wildbook_info(tomcat_dpath, wb_target)
url_command = 'EncounterSetMarkedIndividual'
BASIC_AUTH = False
if BASIC_AUTH:
#url_command += '=authcBasicWildbook'
username = '******'
password = '******'
wildbook_base_url = ('http://' + username + ':' + password + '@' +
wildbook_base_url.replace('http://', ''))
url_args_fmtstr = '&'.join([
'encounterID={annot_uuid!s}',
'individualID={name_text!s}',
])
submit_namchange_url_fmtstr = (
wildbook_base_url + '/' + url_command + '?' + url_args_fmtstr)
if aid_list is None:
aid_list = ibs.get_valid_aids(is_known=True)
# Build URLs to submit
annot_uuid_list = ibs.get_annot_uuids(aid_list)
annot_name_text_list = ibs.get_annot_name_texts(aid_list)
submit_url_list = [
submit_namchange_url_fmtstr.format(
annot_uuid=str(annot_uuid), name_text=str(name_text))
for annot_uuid, name_text in zip(annot_uuid_list, annot_name_text_list)
]
payload = {}
# Submit each URL
status_list = []
print('Submitting URL list')
print(ut.indentjoin(submit_url_list))
message_list = []
for url in ut.ProgressIter(submit_url_list, lbl='submitting URL', freq=1):
print(url)
status, response = submit_wildbook_url(url, payload, dryrun=dryrun)
#print(ut.dict_str(response.__dict__, truncate=0))
status_list.append(status)
try:
response_json = response.json()
# encounter in this message is a wb-encounter not our ia-encounter
#if ut.VERBOSE:
print(response_json['message'])
message_list.append(str(response_json['message']))
except Exception as ex:
print(ut.indentjoin(message_list))
ut.printex(ex, ('Failed getting json from response. '
'Is there an authentication issue?'))
raise
assert response_json['success']
print(ut.indentjoin(message_list))
return status_list
def _wip_embed(parent_locals=None, parent_globals=None, exec_lines=None,
remove_pyqt_hook=True, N=0):
"""
Starts interactive session. Similar to keyboard command in matlab.
Wrapper around IPython.embed
Notes:
#https://github.com/ipython/ipython/wiki/Cookbook%3a-Updating-code-for-use-with-IPython-0.11-and-later
import IPython
x = 3
IPython.embed()
c = IPython.Config()
c.InteractiveShellApp.exec_lines = [
'%pylab qt4',
"print 'System Ready!'",
]
def foo():
return x + 3
a = 3
def bar():
return a + 3
bar()
#NameError: global name 'a' is not defined
from IPython.terminal.ipapp import TerminalIPythonApp
x = 3
app = TerminalIPythonApp.instance()
app.initialize(argv=[]) # argv=[] instructs IPython to ignore sys.argv
app.start()
Args:
parent_locals (None):
parent_globals (None):
exec_lines (None):
remove_pyqt_hook (bool):
N (int):
CommandLine:
python -m utool.util_dbg --test-embed
References:
http://stackoverflow.com/questions/27911570/can-you-specify-a-command-to-run-after-you-embed-into-ipython/27914204#27914204
http://stackoverflow.com/questions/15167200/how-do-i-embed-an-ipython-interpreter-into-an-application-running-in-an-ipython
Notes:
Use cases I want to achieve
1) Simply stop execution and embed in an IPython terminal session
2) Like case 1, but execute a specific set of command (eg '%gui qt')
AFTER IPython has started
3) Embed and pause GUI execution (this is just case 1)
3) Embed and let GUI execution continue while embeded. (basically just need case 2)
TODO:
try:
get_ipython
except NameError:
banner=exit_msg=''
else:
banner = '*** Nested interpreter ***'
exit_msg = '*** Back in main IPython ***'
# First import the embed function
from IPython.frontend.terminal.embed import InteractiveShellEmbed
# Now create the IPython shell instance. Put ipshell() anywhere in your code
# where you want it to open.
ipshell = InteractiveShellEmbed(banner1=banner, exit_msg=exit_msg)
#Then use ipshell() whenever you want to be dropped into an IPython shell. This
#will allow you to embed (and even nest) IPython interpreters in your code and
#inspect objects or the state of the program.
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_dbg import * # NOQA
>>> # build test data
>>> parent_locals = None
>>> parent_globals = None
>>> exec_lines = None
>>> remove_pyqt_hook = True
>>> N = 0
>>> # execute function
>>> result = embed(parent_locals, parent_globals, exec_lines, remove_pyqt_hook, N)
>>> # verify results
>>> print(result)
"""
import utool as ut
from functools import partial
import IPython
if parent_globals is None:
parent_globals = get_parent_frame(N=N).f_globals
if parent_locals is None:
parent_locals = get_parent_frame(N=N).f_locals
stackdepth = N # NOQA
getframe = partial(ut.get_parent_frame, N=N) # NOQA
exec(execstr_dict(parent_globals, 'parent_globals'))
exec(execstr_dict(parent_locals, 'parent_locals'))
print('')
print('================')
print(ut.bubbletext('EMBEDDING'))
print('================')
print('[util] embedding')
try:
if remove_pyqt_hook:
try:
import guitool
guitool.remove_pyqt_input_hook()
except (ImportError, ValueError, AttributeError) as ex:
#print(ex)
printex(ex, iswarning=True)
pass
# make qt not loop forever (I had qflag loop forever with this off)
except ImportError as ex:
print(ex)
user_ns = globals()
user_ns = globals().copy()
user_ns.update(locals())
if parent_globals is not None:
user_ns.update(parent_globals)
if parent_locals is not None:
user_ns.update(parent_locals)
orig_argv = sys.argv # NOQA
print('About to start_ipython')
config = IPython.Config()
exec_lines_ = [
'%pylab qt4',
'print("Entered IPYTHON via utool")',
'print("Entry Point: %r" % (ut.get_parent_frame(N=11).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=10).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=9).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=8).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=7).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=6).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_parent_frame(N=5).f_code.co_name,))',
#execstr_dict(parent_locals)
] + ut.ensure_str_list(exec_lines if exec_lines is not None else [])
config.InteractiveShellApp.exec_lines = exec_lines_
print('Exec Lines: ')
print(ut.indentjoin(exec_lines_, '\n >>> '))
IPython.start_ipython(config=config, argv=[], user_ns=user_ns)
# Exit python immediately if specifed
if user_ns.get('qqq', False) or vars.get('qqq', False) or user_ns.get('EXIT_NOW', False):
print('[utool.embed] EXIT_NOW or qqq specified')
sys.exit(1)
def TEST_GUI_ALL(ibs, back, gpath_list):
"""
Creates a new database
Adds test images
Creates dummy ANNOTATIONS
Selects things
"""
# DELETE OLD
print('[TEST] DELETE_OLD_DATABASE')
work_dir = sysres.get_workdir()
new_dbname = 'testdb_guiall'
new_dbdir = utool.truepath(utool.join(work_dir, new_dbname))
ibs_dbdir = utool.truepath(ibs.dbdir)
msg = 'must start in different dir new_dbdir=%r != ibs_dbdir=%r,' % (new_dbdir, ibs_dbdir)
assert new_dbdir != ibs_dbdir, msg
print('passed: ' + msg)
utool.delete(new_dbdir, ignore_errors=False)
#
#
# CREATE NEW
print('[TEST] CREATE_NEW_DATABASE')
back.new_database(new_dbdir)
ibs = back.ibs # The backend has a new ibeis do not use the old one
# Dont refresh for speed
_kwargs = {'refresh': False}
#
#
# IMPORT IMAGES
print('[TEST] IMPORT_TEST_GPATHS')
print('gpath_list = ' + utool.indentjoin(gpath_list))
gid_list = back.import_images(gpath_list=gpath_list, **_kwargs)
print('\n'.join(' * gid_list[%d] = %r' % (count, gid) for count, gid in enumerate(gid_list)))
assert len(gid_list) == len(gpath_list)
#
#
# ADD ANNOTATIONS
print('[TEST] ADD_ANNOTATIONS')
def add_annot(gid, bbox, theta=0.0):
aid = back.add_annot(gid=gid, bbox=bbox, theta=theta, **_kwargs)
return aid
preadd_aids = ibs.get_valid_aids() # this should be []
assert len(preadd_aids) == 0, 'there are already aids in the database!'
print('preadd_aids = %r' % preadd_aids)
aid1 = add_annot(gid_list[0], (50, 50, 100, 100), (np.tau / 8))
aid2 = add_annot(gid_list[1], (50, 50, 100, 100))
aid3 = add_annot(gid_list[2], (50, 50, 64, 64))
aid4 = add_annot(gid_list[2], (50, 50, 200, 200))
aid5 = add_annot(gid_list[1], (0, 0, 400, 400))
print('aid1 = %r' % aid1)
print('aid2 = %r' % aid2)
print('aid3 = %r' % aid3)
print('aid4 = %r' % aid4)
print('aid5 = %r' % aid5)
#
#
# SELECT ANNOTATIONS
print('[TEST] SELECT ANNOTATION / Add Chips')
# get_valid_aids seems to return aids in an arbitrary order, it's an SQL thing
aid_list = sorted(ibs.get_valid_aids())
print('\n'.join(' * aid_list[%d] = %r' % (count, aid) for count, aid in enumerate(aid_list)))
back.select_aid(aid_list[0], show_image=True, **_kwargs)
try:
bbox_list = ibs.get_annot_bboxes(aid_list)
assert bbox_list[0] == (50, 50, 100, 100)
except AssertionError as ex:
utool.printex(ex, key_list=['bbox_list', 'aid_list'])
raise
back.reselect_annotation(bbox=[51, 52, 103, 104])
assert ibs.get_annot_bboxes(aid_list[0]) == (51, 52, 103, 104)
back.compute_encounters()
unixtime_list = [100, 23, 24]
ibs.set_image_unixtime(gid_list, unixtime_list)
back.compute_encounters()
# Change some ANNOTATIONs
#add_annot(gid_list[2], None) # user selection
#add_annot(None, [42, 42, 8, 8]) # back selection
# I'm not sure how I want to integrate that IPython stuff
return locals()
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = ['ibeis/tests/assert_modules.py']
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/utool'
exclude_list = []
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
implicit_build_modlist_str = ut.codeblock('''
import sys
exclude_doctests_fnames = set(['__init__.py'])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['utool']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
''')
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
import_str = '\n'.join(
['import ' + modname for modname in doctest_modname_list])
modlist_str = (
'module_list = [%s\n]' %
ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
''')
pyscript_text = pyscript_fmtstr.format(
build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
def def_test(header,
pat=None,
dpath=None,
modname=None,
default=False,
testcmds=None):
""" interface to make test tuple """
return (header, default, modname, dpath, pat, testcmds)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds = ut.get_module_testlines(module_list,
remove_pyc=True,
verbose=False,
pythoncmd='RUN_TEST')
test_headers = [
# title, default, module, testpattern
def_test('DOC', testcmds=testcmds, default=True)
]
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs,
quick_tests, repodir,
exclude_list)
return shscript_text, pyscript_text
def print_unused():
print(ut.indentjoin(ut.sortedby(unused_keys, map(len, unused_keys))))
print('len(unused_keys) = %r' % (len(unused_keys), ))
def make_run_tests_script_text(test_headers,
test_argvs,
quick_tests=None,
repodir=None,
exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath,
specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar,
fpath=fpath,
specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "', 'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")'
]).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(
testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(
testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(
header_lower=header_lower,
testflag=testflag,
)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(
set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten(
[ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any(
[x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [
util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list
]
testline_list = [
format_testline(fpath, dirvar) for fpath in testfpath_list
]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(
ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,
)
header_test_block = header_test_block_fmstr.format(
**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs,
dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,
)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
def _pack(bits, errs, thresh):
return utool.indentjoin([
'%5s %f < %f' % (bit, err, thresh)
for (bit, err) in zip(bits, errs)
])
def autogen_ibeis_runtest():
""" special case to generate tests script for IBEIS
Example:
>>> from autogen_test_script import * # NOQA
>>> test_script = autogen_ibeis_runtest()
>>> print(test_script)
CommandLine:
python -c "import utool; utool.autogen_ibeis_runtest()"
python -c "import utool; print(utool.autogen_ibeis_runtest())"
python -c "import utool; print(utool.autogen_ibeis_runtest())" > run_tests.sh
chmod +x run_tests.sh
"""
quick_tests = [
'ibeis/tests/assert_modules.py'
]
#test_repos = [
# '~/code/ibeis'
# '~/code/vtool'
# '~/code/hesaff'
# '~/code/guitool'
#]
#test_pattern = [
# '~/code/ibeis/test_ibs*.py'
#]
test_argvs = '--quiet --noshow'
misc_pats = [
'test_utool_parallel.py',
'test_pil_hash.py',
]
repodir = '~/code/ibeis'
testdir = 'ibeis/tests'
exclude_list = []
# Hacky, but not too bad way of getting in doctests
# Test to see if doctest_funcs appears after main
# Do not doctest these modules
#exclude_doctests_fnames = set(['template_definitions.py',
# 'autogen_test_script.py'])
#exclude_dirs = [
# '_broken',
# 'old',
# 'tests',
# 'timeits',
# '_scripts',
# '_timeits',
# '_doc',
# 'notebook',
#]
#dpath_list = ['ibeis']
#doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
# Verbosity to show which modules at least have some tests
#untested_modnames = ut.find_untested_modpaths(dpath_list, exclude_doctests_fnames, exclude_dirs)
#print('\nUNTESTED MODULES:' + ut.indentjoin(untested_modnames))
#print('\nTESTED MODULES:' + ut.indentjoin(doctest_modname_list))
# The implict list is exactly the code we will use to make the implicit list
#module_list = None
#doctest_modname_list = None
#'export_subset.py',
#'_autogen_ibeiscontrol_funcs.py',
#'randomforest.py',
implicit_build_modlist_str = ut.codeblock(
'''
import sys
exclude_doctests_fnames = set([
'template_definitions.py',
'autogen_test_script.py',
])
exclude_dirs = [
'_broken',
'old',
'tests',
'timeits',
'_scripts',
'_timeits',
'_doc',
'notebook',
]
dpath_list = ['ibeis']
doctest_modname_list = ut.find_doctestable_modnames(dpath_list, exclude_doctests_fnames, exclude_dirs)
for modname in doctest_modname_list:
exec('import ' + modname, globals(), locals())
module_list = [sys.modules[name] for name in doctest_modname_list]
'''
)
globals_ = globals()
locals_ = locals()
exec(implicit_build_modlist_str, globals_, locals_)
module_list = locals_['module_list']
doctest_modname_list = locals_['doctest_modname_list']
#module_list = [__import__(name, globals(), locals(), fromlist=[], level=0) for name in modname_list]
#for modname in doctest_modname_list:
# exec('import ' + modname, globals(), locals())
#module_list = [sys.modules[name] for name in doctest_modname_list]
#print('\n'.join(testcmds))
#print('\n'.join(['python -m ' + modname for modname in doctest_modname_list]))
import_str = '\n'.join(['import ' + modname for modname in doctest_modname_list])
modlist_str = ('module_list = [%s\n]' % ut.indentjoin([modname + ',' for modname in doctest_modname_list]))
explicit_build_modlist_str = '\n\n'.join((import_str, modlist_str))
build_modlist_str = implicit_build_modlist_str
#build_modlist_str = explicit_build_modlist_str
pyscript_fmtstr = ut.codeblock(
r'''
#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import utool as ut
def run_tests():
# Build module list and run tests
{build_modlist_str}
ut.doctest_module_list(module_list)
if __name__ == '__main__':
import multiprocessing
multiprocessing.freeze_support()
run_tests()
'''
)
pyscript_text = pyscript_fmtstr.format(build_modlist_str=ut.indent(build_modlist_str).strip())
pyscript_text = ut.autofix_codeblock(pyscript_text)
# BUILD OLD SHELL RUN TESTS HARNESS
testcmds_ = ut.get_module_testlines(module_list, remove_pyc=True,
verbose=False, pythoncmd='RUN_TEST',
testslow=True)
testcmds = [cmd + ' --sysexitonfail' for cmd in testcmds_]
test_headers = [
# title, default, module, testpattern
ut.def_test('VTOOL', dpath='vtool/tests', pat=['test*.py'], modname='vtool'),
ut.def_test('GUI', dpath=testdir, pat=['test_gui*.py']),
ut.def_test('IBEIS', dpath=testdir, pat=['test_ibs*.py', 'test_delete*.py'], default=False),
ut.def_test('SQL', dpath=testdir, pat=['test_sql*.py']),
ut.def_test('VIEW', dpath=testdir, pat=['test_view*.py']),
ut.def_test('MISC', dpath=testdir, pat=misc_pats),
ut.def_test('OTHER', dpath=testdir, pat='OTHER'),
ut.def_test('HESAFF', dpath='pyhesaff/tests', pat=['test_*.py'], modname='pyhesaff'),
ut.def_test('DOC', testcmds=testcmds, default=True)
]
# Referencs: https://docs.python.org/2/library/runpy.html
shscript_text = ut.make_run_tests_script_text(test_headers, test_argvs, quick_tests, repodir, exclude_list)
#print(pyscript_text)
return shscript_text, pyscript_text
def auto_docstr(**kwargs):
import imp
import utool as ut
ut.util_dbg.COLORED_EXCEPTIONS = False
ut.ENABLE_COLORS = False
ut.util_str.ENABLE_COLORS = False
try:
print("RELOADING UTOOL via imp")
imp.reload(ut)
imp.reload(ut._internal.meta_util_arg)
except Exception as ex:
print("... errored")
pass
print("RELOADING UTOOL via rrrr")
ut.rrrr(verbose=0)
imp.reload(ut)
import vim
modname = None
funcname = None
flag = False
dbgtext = ''
docstr = ''
dbgmsg = ''
try:
funcname, searchlines = find_pyfunc_above_cursor()
modname, moddir = get_current_modulename()
if funcname is None:
funcname = '[vimerr] UNKNOWN_FUNC: funcname is None'
flag = True
else:
# Text to insert into the current buffer
verbose = True
autodockw = dict(verbose=verbose)
autodockw.update(kwargs)
docstr = ut.auto_docstr(modname, funcname, moddir=moddir, **autodockw)
#if docstr.find('unexpected indent') > 0:
# docstr = funcname + ' ' + docstr
if docstr[:].strip() == 'error':
flag = True
except vim.error as ex:
dbgmsg = 'vim_error: ' + str(ex)
flag = False
except Exception as ex:
dbgmsg = 'exception(%r): %s' % (type(ex), str(ex))
ut.printex(ex, tb=True)
flag = False
if flag:
dbgtext += '\n+======================'
dbgtext += '\n| --- DEBUG OUTPUT --- '
if len(dbgmsg) > 0:
dbgtext += '\n| Message: '
dbgtext += dbgmsg
dbgtext += '\n+----------------------'
dbgtext += '\n| InsertDoctstr(modname=%r, funcname=%r' % (modname, funcname)
pycmd = ('import ut; print(ut.auto_docstr(%r, %r)))' % (modname, funcname))
pycmd = pycmd.replace('\'', '\\"')
dbgtext += '\n| python -c "%s"' % (pycmd,)
dbgtext += '\n+----------------------'
dbgtext += '\n+searchlines = '
dbgtext += ut.indentjoin(searchlines, '\n| ')
dbgtext += '\nL----------------------'
elif len(dbgmsg) > 0:
dbgtext += '\n| Message: '
dbgtext += dbgmsg
text = '\n'.join([docstr + dbgtext])
if text == '':
print('No Text! For some reason flag=%r' % (flag,))
return text
def embed(parent_locals=None, parent_globals=None, exec_lines=None,
remove_pyqt_hook=True, N=0):
"""
Starts interactive session. Similar to keyboard command in matlab.
Wrapper around IPython.embed
Notes:
#https://github.com/ipython/ipython/wiki/Cookbook%3a-Updating-code-for-use-with-IPython-0.11-and-later
import IPython
x = 3
IPython.embed()
c = IPython.Config()
c.InteractiveShellApp.exec_lines = [
'%pylab qt4',
"print 'System Ready!'",
]
def foo():
return x + 3
a = 3
def bar():
return a + 3
bar()
#NameError: global name 'a' is not defined
from IPython.terminal.ipapp import TerminalIPythonApp
x = 3
app = TerminalIPythonApp.instance()
app.initialize(argv=[]) # argv=[] instructs IPython to ignore sys.argv
app.start()
Args:
parent_locals (None):
parent_globals (None):
exec_lines (None):
remove_pyqt_hook (bool):
N (int):
CommandLine:
python -m utool.util_dbg --test-embed
References:
http://stackoverflow.com/questions/27911570/can-you-specify-a-command-to-run-after-you-embed-into-ipython/27914204#27914204
http://stackoverflow.com/questions/15167200/how-do-i-embed-an-ipython-interpreter-into-an-application-running-in-an-ipython
TODO:
try:
get_ipython
except NameError:
banner=exit_msg=''
else:
banner = '*** Nested interpreter ***'
exit_msg = '*** Back in main IPython ***'
# First import the embed function
from IPython.frontend.terminal.embed import InteractiveShellEmbed
# Now create the IPython shell instance. Put ipshell() anywhere in your code
# where you want it to open.
ipshell = InteractiveShellEmbed(banner1=banner, exit_msg=exit_msg)
#Then use ipshell() whenever you want to be dropped into an IPython shell. This
#will allow you to embed (and even nest) IPython interpreters in your code and
#inspect objects or the state of the program.
Example:
>>> # DISABLE_DOCTEST
>>> from utool.util_dbg import * # NOQA
>>> # build test data
>>> parent_locals = None
>>> parent_globals = None
>>> exec_lines = None
>>> remove_pyqt_hook = True
>>> N = 0
>>> # execute function
>>> result = embed(parent_locals, parent_globals, exec_lines, remove_pyqt_hook, N)
>>> # verify results
>>> print(result)
"""
import utool as ut
from functools import partial
import IPython
if parent_globals is None:
parent_globals = get_parent_globals(N=N)
#parent_globals1 = get_parent_globals(N=0)
#exec(execstr_dict(parent_globals1, 'parent_globals1'))
if parent_locals is None:
parent_locals = get_parent_locals(N=N)
stackdepth = N # NOQA
getframe = partial(ut.get_caller_stack_frame, N=N) # NOQA
exec(execstr_dict(parent_globals, 'parent_globals'))
exec(execstr_dict(parent_locals, 'parent_locals'))
print('')
print('================')
print(ut.bubbletext('EMBEDDING'))
print('================')
print('[util] embedding')
try:
if remove_pyqt_hook:
try:
import guitool
guitool.remove_pyqt_input_hook()
except (ImportError, ValueError, AttributeError) as ex:
#print(ex)
printex(ex, iswarning=True)
pass
# make qt not loop forever (I had qflag loop forever with this off)
except ImportError as ex:
print(ex)
NEW_METHOD = False
if NEW_METHOD:
user_ns = globals()
user_ns = globals().copy()
user_ns.update(locals())
if parent_globals is not None:
user_ns.update(parent_globals)
if parent_locals is not None:
user_ns.update(parent_locals)
orig_argv = sys.argv # NOQA
print('About to start_ipython')
config = IPython.Config()
exec_lines_ = [
'%pylab qt4',
'print("Entered IPYTHON via utool")',
'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=11).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=10).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=9).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=8).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=7).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=6).f_code.co_name,))',
#'print("Entry Point: %r" % (ut.get_caller_stack_frame(N=5).f_code.co_name,))',
#execstr_dict(parent_locals)
] + ut.ensure_str_list(exec_lines if exec_lines is not None else [])
config.InteractiveShellApp.exec_lines = exec_lines_
print('Exec Lines: ')
print(ut.indentjoin(exec_lines_, '\n >>> '))
IPython.start_ipython(config=config, argv=[], user_ns=user_ns)
# Exit python immediately if specifed
if user_ns.get('qqq', False) or vars.get('qqq', False) or user_ns.get('EXIT_NOW', False):
print('[utool.embed] EXIT_NOW or qqq specified')
sys.exit(1)
else:
#from IPython.config.loader import Config
# cfg = Config()
#config_dict = {}
#if exec_lines is not None:
# config_dict['exec_lines'] = exec_lines
#IPython.embed(**config_dict)
print('[util] Get stack location with: ')
print('[util] ut.get_caller_stack_frame(N=8).f_code.co_name')
print('[util] set EXIT_NOW or qqq to True(ish) to hard exit on unembed')
#print('set iup to True to draw plottool stuff')
print('[util] call %pylab qt4 to get plottool stuff working')
once = True
# Allow user to set iup and redo the loop
while once or vars().get('iup', False):
if not once:
# SUPER HACKY WAY OF GETTING FIGURES ON THE SCREEN BETWEEN UPDATES
#vars()['iup'] = False
# ALL YOU NEED TO DO IS %pylab qt4
print('re-emebeding')
#import plottool as pt
#pt.update()
#(pt.present())
for _ in range(100):
time.sleep(.01)
once = False
#vars().get('iup', False):
print('[util] calling IPython.embed()')
"""
Notes:
/usr/local/lib/python2.7/dist-packages/IPython/terminal/embed.py
IPython.terminal.embed.InteractiveShellEmbed
# instance comes from IPython.config.configurable.SingletonConfigurable.instance
"""
#c = IPython.Config()
#c.InteractiveShellApp.exec_lines = [
# '%pylab qt4',
# "print 'System Ready!'",
#]
#IPython.embed(config=c)
try:
IPython.embed()
except RuntimeError as ex:
ut.printex(ex, 'Failed to open ipython')
#config = IPython.terminal.ipapp.load_default_config()
#config.InteractiveShellEmbed = config.TerminalInteractiveShell
#module = sys.modules[parent_globals['__name__']]
#config['module'] = module
#config['module'] = module
#embed2(stack_depth=N + 2 + 1)
#IPython.embed(config=config)
#IPython.embed(config=config)
#IPython.embed(module=module)
# Exit python immediately if specifed
if vars().get('EXIT_NOW', False) or vars().get('qqq', False):
print('[utool.embed] EXIT_NOW specified')
sys.exit(1)
