def get_dist_app_fpath():
app_fpath = ut.unixpath('dist/ibeis/IBEISApp')
if ut.DARWIN:
app_fpath = ut.unixpath('dist/IBEIS.app/Contents/MacOS/IBEISApp')
if ut.WIN32:
app_fpath += '.exe'
return app_fpath
def get_dist_app_fpath():
app_fpath = ut.unixpath('dist/ibeis/IBEISApp')
if ut.DARWIN:
app_fpath = ut.unixpath('dist/IBEIS.app/Contents/MacOS/IBEISApp')
if ut.WIN32:
app_fpath += '.exe'
return app_fpath
def get_cyth_pxd_path(py_fpath):
"""
>>> py_fpath = '/foo/vtool/vtool/keypoint.py'
>>> cy_fpath = get_cyth_pxd_path(py_fpath)
>>> print(cy_fpath)
/foo/vtool/vtool/_keypoint_cyth.pxd
"""
dpath, fname = split(py_fpath)
name, ext = splitext(fname)
assert ext == '.py', 'not a python file'
cy_fpath = utool.unixpath(join(dpath, get_cyth_name(name) + '.pxd'))
return cy_fpath
def translate_all():
""" Translates a all python paths in directory """
dpaths = utool.ls_moduledirs('.')
#print('[cyth] translate_all: %r' % (dpaths,))
globkw = {'recursive': True, 'with_dirs': False, 'with_files': True}
# Find all unique python files in directory
fpaths_iter = [
utool.glob(utool.unixpath(dpath), '*.py', **globkw) for dpath in dpaths
]
fpath_iter = utool.iflatten(fpaths_iter)
abspath_iter = map(utool.unixpath, fpath_iter)
fpath_list = list(set(list(abspath_iter)))
#print('[cyth] translate_all: %s' % ('\n'.join(fpath_list),))
# Try to translate each
translate(*fpath_list)
def translate_all():
""" Translates a all python paths in directory """
dpaths = utool.ls_moduledirs('.')
#print('[cyth] translate_all: %r' % (dpaths,))
globkw = {
'recursive': True,
'with_dirs': False,
'with_files': True
}
# Find all unique python files in directory
fpaths_iter = [utool.glob(utool.unixpath(dpath), '*.py', **globkw)
for dpath in dpaths]
fpath_iter = utool.iflatten(fpaths_iter)
abspath_iter = map(utool.unixpath, fpath_iter)
fpath_list = list(set(list(abspath_iter)))
#print('[cyth] translate_all: %s' % ('\n'.join(fpath_list),))
# Try to translate each
translate(*fpath_list)
def translate(*paths):
""" Translates a list of paths """
cy_bench_list = []
for fpath in paths:
if isfile(fpath):
abspath = utool.unixpath(fpath)
cy_bench = translate_fpath(abspath)
if cy_bench is not None:
cy_bench_list.append(cy_bench)
if len(cy_bench_list) > 0:
runbench_shtext = cyth_benchmarks.build_runbench_shell_text(cy_bench_list)
runbench_pytext = cyth_benchmarks.build_runbench_pyth_text(cy_bench_list)
utool.write_to('_old_run_cyth_benchmarks.sh', runbench_shtext)
utool.write_to('run_cyth_benchmarks.py', runbench_pytext)
#try:
os.chmod('_old_run_cyth_benchmarks.sh', 33277)
os.chmod('run_cyth_benchmarks.py', 33277)
def translate(*paths):
""" Translates a list of paths """
cy_bench_list = []
for fpath in paths:
if isfile(fpath):
abspath = utool.unixpath(fpath)
cy_bench = translate_fpath(abspath)
if cy_bench is not None:
cy_bench_list.append(cy_bench)
if len(cy_bench_list) > 0:
runbench_shtext = cyth_benchmarks.build_runbench_shell_text(
cy_bench_list)
runbench_pytext = cyth_benchmarks.build_runbench_pyth_text(
cy_bench_list)
utool.write_to('_old_run_cyth_benchmarks.sh', runbench_shtext)
utool.write_to('run_cyth_benchmarks.py', runbench_pytext)
#try:
os.chmod('_old_run_cyth_benchmarks.sh', 33277)
os.chmod('run_cyth_benchmarks.py', 33277)
pkgname = basename(repodir)
packages = utool.ls_moduledirs(repodir, full=False)
print(pkgname)
setup_text = setup_text_fmt.format(
packages=packages,
repodir=repodir,
timestamp=timestamp,
pkgname=pkgname,
)
return setup_text
if __name__ == '__main__':
writeflag = utool.get_argflag(('--write', '-w'))
overwriteflag = utool.get_argflag(('--yes', '-y'))
repodir = utool.unixpath(os.getcwd())
print('[utool] making setup.py for: %r' % repodir)
setup_text = make_setup(repodir)
if writeflag:
setup_fpath = utool.unixjoin(repodir, 'setup.py')
if utool.checkpath(setup_fpath):
confirm_flag = overwriteflag
else:
confirm_flag = True
if confirm_flag:
utool.write_to(setup_fpath, setup_text)
else:
print('setup.py file exists not writing')
else:
print(setup_text)
def bench_fpath_to_modname(bench):
bench_upath = utool.unixpath(bench)
bench_relpath = relpath(bench_upath, os.getcwd())
bench_relname, _ = splitext(bench_relpath)
bench_modname = bench_relname.replace('\\', '/').replace('/', '.')
return bench_modname
DCVS_DEFAULT = ut.ParamInfoList('distinctivness', [
ut.ParamInfo('dcvs_power', 1.0, 'p', varyvals=[.5, 1.0, 1.5, 2.0]),
ut.ParamInfo('dcvs_min_clip', .2, 'mn', varyvals=[.2, .02, .03][0:1]),
ut.ParamInfo('dcvs_max_clip', .5, 'mx', varyvals=[.05, .3, .4, .45, .5, 1.0][1:4]),
ut.ParamInfo('dcvs_K', 5, 'dcvsK', varyvals=[5, 7, 15][0:1]),
])
DISTINCTIVENESS_NORMALIZER_CACHE = {}
BASELINE_DISTINCTIVNESS_URLS = {
# TODO: Populate
'zebra_grevys': const.ZIPPED_URLS.GZ_DISTINCTIVE,
'zebra_plains': const.ZIPPED_URLS.PZ_DISTINCTIVE,
}
PUBLISH_DIR = ut.unixpath('~/Dropbox/IBEIS')
def testdata_distinctiveness():
"""
Example:
>>> # SLOW_DOCTEST
>>> from ibeis.algo.hots.distinctiveness_normalizer import * # NOQA
>>> dstcnvs_normer, qreq_ = testdata_distinctiveness()
"""
import ibeis
# build test data
db = ut.get_argval('--db', str, 'testdb1')
species = ut.get_argval('--species', str, None)
aid = ut.get_argval('--aid', int, None)
ibs = ibeis.opendb(db)
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None,
repodir=None, exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(
r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(
r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(
r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "',
'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")']).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(header_lower=header_lower,
testflag=testflag,)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten([ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any([x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list]
testline_list = [format_testline(fpath, dirvar) for fpath in testfpath_list]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,)
header_test_block = header_test_block_fmstr.format(**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs, dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
def bench_fpath_to_modname(bench):
bench_upath = utool.unixpath(bench)
bench_relpath = relpath(bench_upath, os.getcwd())
bench_relname, _ = splitext(bench_relpath)
bench_modname = bench_relname.replace('\\', '/').replace('/', '.')
return bench_modname
pkgname = basename(repodir)
packages = utool.ls_moduledirs(repodir, full=False)
print(pkgname)
setup_text = setup_text_fmt.format(
packages=packages,
repodir=repodir,
timestamp=timestamp,
pkgname=pkgname,
)
return setup_text
if __name__ == '__main__':
writeflag = utool.get_argflag(('--write', '-w'))
overwriteflag = utool.get_argflag(('--yes', '-y'))
repodir = utool.unixpath(os.getcwd())
print('[utool] making setup.py for: %r' % repodir)
setup_text = make_setup(repodir)
if writeflag:
setup_fpath = utool.unixjoin(repodir, 'setup.py')
if utool.checkpath(setup_fpath):
confirm_flag = overwriteflag
else:
confirm_flag = True
if confirm_flag:
utool.write_to(setup_fpath, setup_text)
else:
print('setup.py file exists not writing')
else:
print(setup_text)
module_name = ut.get_modname_from_modpath(module_path)
IMPORT_TUPLES = util_importer.make_import_tuples(module_path, exclude_modnames=exclude_modnames)
initstr = util_importer.make_initstr(module_name, IMPORT_TUPLES)
regen_command = 'cd %s\n' % (module_path)
regen_command += ' makeinit.py'
if len(exclude_modnames ) > 0:
regen_command += ' -x ' + ' '.join(exclude_modnames)
regen_block = (ut.codeblock('''
"""
Regen Command:
{regen_command}
"""
''').format(regen_command=regen_command))
print('### __init__.py ###')
print(initstr)
print('\nIMPORT_TUPLES = ' + ut.list_str(IMPORT_TUPLES))
print(regen_block)
if __name__ == '__main__':
exclude_modnames = ut.get_argval(('--exclude', '-x'), list, default=[])
modname = ut.get_argval('--modname', str, default=None)
if modname is None:
dpath = os.getcwd()
else:
dpath = ut.get_modpath_from_modname(modname)
module_path = ut.unixpath(dpath)
makeinit(module_path, exclude_modnames=exclude_modnames)
print('# autogenerated __init__.py for: %r' % module_path)
def write_default_ipython_profile():
"""
CommandLine:
python ~/local/init/init_ipython_config.py
python -c "import utool as ut; ut.vd(ut.unixpath('~/.ipython/profile_default'))"
python -c "import utool as ut; ut.editfile(ut.unixpath('~/.ipython/profile_default/ipython_config.py'))"
References:
http://2sn.org/python/ipython_config.py
"""
dpath = ut.unixpath('~/.ipython/profile_default')
ut.ensuredir(dpath, info=True, verbose=True)
ipy_config_fpath = ut.unixjoin(dpath, 'ipython_config.py')
ipy_config_text = ut.codeblock(
r'''
# STARTBLOCK
c = get_config() # NOQA
c.InteractiveShellApp.exec_lines = []
future_line = (
'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals')
c.InteractiveShellApp.exec_lines.append(future_line)
# Fix sip versions
try:
import sip
# http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string
sip.setapi('QVariant', 2)
sip.setapi('QString', 2)
sip.setapi('QTextStream', 2)
sip.setapi('QTime', 2)
sip.setapi('QUrl', 2)
sip.setapi('QDate', 2)
sip.setapi('QDateTime', 2)
if hasattr(sip, 'setdestroyonexit'):
sip.setdestroyonexit(False) # This prevents a crash on windows
except ImportError as ex:
pass
except ValueError as ex:
print('Warning: Value Error: %s' % str(ex))
pass
c.InteractiveShellApp.exec_lines.append('%load_ext autoreload')
c.InteractiveShellApp.exec_lines.append('%autoreload 2')
#c.InteractiveShellApp.exec_lines.append('%pylab qt4')
c.InteractiveShellApp.exec_lines.append('import numpy as np')
c.InteractiveShellApp.exec_lines.append('import utool as ut')
#c.InteractiveShellApp.exec_lines.append('import plottool as pt')
c.InteractiveShellApp.exec_lines.append('from os.path import *')
c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle')
#c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True')
import utool as ut
c.InteractiveShellApp.exec_lines.append(ut.codeblock(
"""
class classproperty(property):
def __get__(self, cls, owner):
return classmethod(self.fget).__get__(None, owner)()
class vim(object):
@classproperty
def focus(cls):
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),)
@classproperty
def copy(cls):
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),)
import utool as ut
import IPython
ipy = IPython.get_ipython()
lastline = ipy.history_manager.input_hist_parsed[-2]
ut.copy_text_to_clipboard(lastline)
# import utool as ut
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(
('focus', 'GVIM'),
('key', 'ctrl+v'),
('focus', 'x-terminal-emulator.X-terminal-emulator')
)
"""
))
#c.InteractiveShell.autoindent = True
#c.InteractiveShell.colors = 'LightBG'
#c.InteractiveShell.confirm_exit = False
#c.InteractiveShell.deep_reload = True
c.InteractiveShell.editor = 'gvim'
#c.InteractiveShell.xmode = 'Context'
# ENDBOCK
'''
)
ut.write_to(ipy_config_fpath, ipy_config_text)
'dcvs_min_clip', 0.2, 'mn', varyvals=[0.2, 0.02, 0.03][0:1]),
ut.ParamInfo('dcvs_max_clip',
0.5,
'mx',
varyvals=[0.05, 0.3, 0.4, 0.45, 0.5, 1.0][1:4]),
ut.ParamInfo('dcvs_K', 5, 'dcvsK', varyvals=[5, 7, 15][0:1]),
],
)
DISTINCTIVENESS_NORMALIZER_CACHE = {}
BASELINE_DISTINCTIVNESS_URLS = {
# TODO: Populate
'zebra_grevys': const.ZIPPED_URLS.GZ_DISTINCTIVE,
'zebra_plains': const.ZIPPED_URLS.PZ_DISTINCTIVE,
}
PUBLISH_DIR = ut.unixpath('~/Dropbox/IBEIS')
def testdata_distinctiveness():
"""
Example:
>>> # SLOW_DOCTEST
>>> from wbia.algo.hots.distinctiveness_normalizer import * # NOQA
>>> dstcnvs_normer, qreq_ = testdata_distinctiveness()
"""
import wbia
# build test data
db = ut.get_argval('--db', str, 'testdb1')
species = ut.get_argval('--species', str, None)
aid = ut.get_argval('--aid', int, None)
def make_run_tests_script_text(test_headers,
test_argvs,
quick_tests=None,
repodir=None,
exclude_list=[]):
"""
Autogeneration function
TODO move to util_autogen or just depricate
Examples:
>>> from utool.util_tests import * # NOQA
>>> import utool # NOQA
>>> testdirs = ['~/code/ibeis/test_ibs*.py']
"""
import utool as ut
from os.path import relpath, join, dirname # NOQA
exclude_list += ['__init__.py']
# General format of the testing script
script_fmtstr = ut.codeblock(r'''
#!/bin/bash
# Runs all tests
# Win32 path hacks
export CWD=$(pwd)
export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")"
# <CORRECT_PYTHON>
# GET CORRECT PYTHON ON ALL PLATFORMS
export SYSNAME="$(expr substr $(uname -s) 1 10)"
if [ "$SYSNAME" = "MINGW32_NT" ]; then
export PYEXE=python
else
if [ "$PYMAJOR" = "3" ]; then
# virtual env?
export PYEXE=python
else
export PYEXE=python2.7
fi
fi
# </CORRECT_PYTHON>
PRINT_DELIMETER()
{{
printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n"
}}
export TEST_ARGV="{test_argvs} $@"
{dirdef_block}
# Default tests to run
set_test_flags()
{{
export DEFAULT=$1
{testdefault_block}
}}
set_test_flags OFF
{testdefaulton_block}
# Parse for bash commandline args
for i in "$@"
do
case $i in --testall)
set_test_flags ON
;;
esac
{testcmdline_block}
done
BEGIN_TESTS()
{{
cat <<EOF
{runtests_bubbletext}
EOF
echo "BEGIN: TEST_ARGV=$TEST_ARGV"
PRINT_DELIMETER
num_passed=0
num_ran=0
export FAILED_TESTS=''
}}
RUN_TEST()
{{
echo "RUN_TEST: $@"
export TEST="$PYEXE $@ $TEST_ARGV"
$TEST
export RETURN_CODE=$?
echo "RETURN_CODE=$RETURN_CODE"
PRINT_DELIMETER
num_ran=$(($num_ran + 1))
if [ "$RETURN_CODE" == "0" ] ; then
num_passed=$(($num_passed + 1))
fi
if [ "$RETURN_CODE" != "0" ] ; then
export FAILED_TESTS="$FAILED_TESTS\n$TEST"
fi
}}
END_TESTS()
{{
echo "RUN_TESTS: DONE"
if [ "$FAILED_TESTS" != "" ] ; then
echo "-----"
printf "Failed Tests:"
printf "$FAILED_TESTS\n"
printf "$FAILED_TESTS\n" >> failed_shelltests.txt
echo "-----"
fi
echo "$num_passed / $num_ran tests passed"
}}
#---------------------------------------------
# START TESTS
BEGIN_TESTS
{quicktest_block}
{test_block}
#---------------------------------------------
# END TESTING
END_TESTS
''')
testcmdline_fmtstr = ut.codeblock(r'''
case $i in --notest{header_lower})
export {testflag}=OFF
;;
esac
case $i in --test{header_lower})
export {testflag}=ON
;;
esac
''')
header_test_block_fmstr = ut.codeblock(r'''
#---------------------------------------------
#{header_text}
if [ "${testflag}" = "ON" ] ; then
cat <<EOF
{header_bubble_text}
EOF
{testlines_block}
fi
''')
#specialargv = '--noshow'
specialargv = ''
testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}'
testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}'
def format_testline(fpath, dirvar):
if dirvar is None:
return testline_fmtstr2.format(fpath=fpath,
specialargv=specialargv)
else:
return testline_fmtstr.format(dirvar=dirvar,
fpath=fpath,
specialargv=specialargv)
default_flag_line_list = []
defaulton_flag_line_list = []
testcmdline_list = []
dirdef_list = []
header_test_block_list = []
known_tests = ut.ddict(list)
# Tests to always run
if quick_tests is not None:
quicktest_block = '\n'.join(
['# Quick Tests (always run)'] +
['RUN_TEST ' + testline for testline in quick_tests])
else:
quicktest_block = '# No quick tests'
# Loop over different test types
for testdef_tup in test_headers:
header, default, modname, dpath, pats, testcmds = testdef_tup
# Build individual test type information
header_upper = header.upper()
header_lower = header.lower()
testflag = header_upper + '_TEST'
if modname is not None:
dirvar = header_upper + '_DIR'
dirdef = ''.join([
'export {dirvar}=$($PYEXE -c "', 'import os, {modname};',
'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))',
'")'
]).format(dirvar=dirvar, modname=modname)
dirdef_list.append(dirdef)
else:
dirvar = None
# Build test dir
#dirvar = header_upper + '_DIR'
#dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname)
#dirdef_list.append(dirdef)
# Build command line flags
default_flag_line = 'export {testflag}=$DEFAULT'.format(
testflag=testflag)
if default:
defaulton_flag_line = 'export {testflag}=ON'.format(
testflag=testflag)
defaulton_flag_line_list.append(defaulton_flag_line)
testcmdline_fmtdict = dict(
header_lower=header_lower,
testflag=testflag,
)
testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict)
#ut.ls(dpath)
# VERY HACK BIT OF CODE
# Get list of tests from patterns
if testcmds is None:
if modname is not None:
module = __import__(modname)
repo_path = dirname(dirname(module.__file__))
else:
repo_path = repodir
dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath))
if header_upper == 'OTHER':
# Hacky way to grab any other tests not explicitly seen in this directory
_testfpath_list = list(
set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_]))
#_testfpath_list = ut.glob(dpath_, '*.py')
#set(known_tests[dpath_])
else:
_testfpath_list = ut.flatten(
[ut.glob(dpath_, pat) for pat in pats])
def not_excluded(x):
return not any(
[x.find(exclude) > -1 for exclude in exclude_list])
_testfpath_list = list(filter(not_excluded, _testfpath_list))
known_tests[dpath_].extend(_testfpath_list)
#print(_testfpath_list)
testfpath_list = [
util_path.unixjoin(dpath, relpath(fpath, dpath_))
for fpath in _testfpath_list
]
testline_list = [
format_testline(fpath, dirvar) for fpath in testfpath_list
]
else:
testline_list = testcmds
testlines_block = ut.indentjoin(testline_list).strip('\n')
# Construct test block for this type
header_text = header_upper + ' TESTS'
headerfont = 'cybermedium'
header_bubble_text = ut.indent(
ut.bubbletext(header_text, headerfont).strip())
header_test_block_dict = dict(
testflag=testflag,
header_text=header_text,
testlines_block=testlines_block,
header_bubble_text=header_bubble_text,
)
header_test_block = header_test_block_fmstr.format(
**header_test_block_dict)
# Append to script lists
header_test_block_list.append(header_test_block)
default_flag_line_list.append(default_flag_line)
testcmdline_list.append(testcmdline)
runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge')
test_block = '\n'.join(header_test_block_list)
dirdef_block = '\n'.join(dirdef_list)
testdefault_block = ut.indent('\n'.join(default_flag_line_list))
testdefaulton_block = '\n'.join(defaulton_flag_line_list)
testcmdline_block = '\n'.join(testcmdline_list)
script_fmtdict = dict(
quicktest_block=quicktest_block,
runtests_bubbletext=runtests_bubbletext,
test_argvs=test_argvs,
dirdef_block=dirdef_block,
testdefault_block=testdefault_block,
testdefaulton_block=testdefaulton_block,
testcmdline_block=testcmdline_block,
test_block=test_block,
)
script_text = script_fmtstr.format(**script_fmtdict)
return script_text
page = urllib2.urlopen(req)
page_str = page.read()
page_str = ut.ensure_unicode(page_str)
next = False
lines = page_str.split('\n')
for index, x in enumerate(lines):
if next:
print(x)
import parse
url_suffix = parse.parse('{foo}href="{href}"{other}', x)['href']
url = r'https://cmake.org' + url_suffix
break
if 'Linux x86_64' in x:
next = True
url = url.replace('.sh', '.tar.gz')
cmake_unzipped_fpath = ut.grab_zipped_url(url)
install_prefix = ut.unixpath('~')
for dname in ['bin', 'doc', 'man', 'share']:
install_dst = join(install_prefix, dname)
install_src = join(cmake_unzipped_fpath, dname)
# FIXME: this broke
#ut.util_path.copy(install_src, install_dst)
# HACK AROUND IT
from os.path import dirname
cmd = str('cp -r "' + install_src + '" "' + dirname(install_dst) + '"')
print(cmd)
ut.cmd(cmd)
#os.system(cmd)
print(cmake_unzipped_fpath)
def test_pyrf():
category = 'zebra_plains'
detect_config = {
'save_detection_images': True,
'save_scales': True,
'percentage_top': 0.40,
}
#=================================
# Train / Detect Initialization
#=================================
testdata_dir = utool.unixpath('~/code/pyrf/results')
# assert utool.checkpath(testdata_dir)
if utool.get_argflag('--vd'):
print(utool.ls(testdata_dir))
# Create detector
detector = Random_Forest_Detector()
test_path = utool.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool')
models_path = utool.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool')
trees_path = join(models_path, category)
results_path = join(utool.unixpath('~/code/pyrf/results'), category)
# detect_path = join(results_path, 'detect')
trees_path = join(results_path, 'trees')
detect_path = join(test_path, category, 'detect')
utool.ensuredir(detect_path)
utool.ensuredir(test_path)
utool.ensuredir(trees_path)
#=================================
# Detect using Random Forest
#=================================
# Get input images
from vtool import image
big_gpath_list = utool.list_images(test_path, fullpath=True, recursive=False)
print(big_gpath_list)
# Resize images to standard size
if utool.get_argflag('--small'):
big_gpath_list = big_gpath_list[0:8]
#big_gpath_list = big_gpath_list[0:8]
output_dir = join(test_path, 'resized')
std_gpath_list = image.resize_imagelist_to_sqrtarea(big_gpath_list,
sqrt_area=800,
output_dir=output_dir,
checkexists=True)
dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list]
#utool.view_directory(test_path)
#utool.view_directory('.')
print(std_gpath_list)
num_images = len(std_gpath_list)
#assert num_images == 16, 'the test has diverged!'
print('Testing on %r images' % num_images)
# Load forest, so we don't have to reload every time
forest = detector.load(trees_path, category + '-', num_trees=25)
detector.set_detect_params(**detect_config)
results_list1 = []
with utool.Timer('[test_pyrf] for loop detector.detect') as t1:
if not utool.get_argflag('--skip1'):
for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)):
#img_fname = split(img_fpath)[1]
#dst_fpath = join(detect_path, img_fname)
#print(' * img_fpath = %r' % img_fpath)
#print(' * dst_fpath = %r' % dst_fpath)
with utool.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)):
results = detector.detect(forest, img_fpath, dst_fpath)
results_list1.append(results)
print('num results = %r' % len(results))
else:
print('...skipped')
# with utool.Timer('[test_pyrf] detector.detect_many') as t2:
# results_list2 = detector.detect_many(forest, std_gpath_list,
# dst_gpath_list, use_openmp=True)
print('')
print('+ --------------')
print('| total time1: %r' % t1.ellapsed)
# print('| total time2: %r' % t2.ellapsed)
print('|')
print('| num results1 = %r' % (list(map(len, results_list1))))
# print('| num results2 = %r' % (list(map(len, results_list2))))
#assert results_list2 == results_list1
return locals()
def write_default_ipython_profile():
"""
CommandLine:
python ~/local/init/init_ipython_config.py
python -c "import utool as ut; ut.vd(ut.unixpath('~/.ipython/profile_default'))"
python -c "import utool as ut; ut.editfile(ut.unixpath('~/.ipython/profile_default/ipython_config.py'))"
References:
http://2sn.org/python/ipython_config.py
"""
dpath = ut.unixpath('~/.ipython/profile_default')
ut.ensuredir(dpath, info=True, verbose=True)
ipy_config_fpath = ut.unixjoin(dpath, 'ipython_config.py')
ipy_config_text = ut.codeblock(r'''
# STARTBLOCK
c = get_config() # NOQA
c.InteractiveShellApp.exec_lines = []
import six
if six.PY2:
future_line = (
'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals')
c.InteractiveShellApp.exec_lines.append(future_line)
# Fix sip versions
try:
import sip
# http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string
sip.setapi('QVariant', 2)
sip.setapi('QString', 2)
sip.setapi('QTextStream', 2)
sip.setapi('QTime', 2)
sip.setapi('QUrl', 2)
sip.setapi('QDate', 2)
sip.setapi('QDateTime', 2)
if hasattr(sip, 'setdestroyonexit'):
sip.setdestroyonexit(False) # This prevents a crash on windows
except ImportError as ex:
pass
except ValueError as ex:
print('Warning: Value Error: %s' % str(ex))
pass
c.InteractiveShellApp.exec_lines.append('%load_ext autoreload')
c.InteractiveShellApp.exec_lines.append('%autoreload 2')
#c.InteractiveShellApp.exec_lines.append('%pylab qt4')
c.InteractiveShellApp.exec_lines.append('import numpy as np')
c.InteractiveShellApp.exec_lines.append('import ubelt as ub')
c.InteractiveShellApp.exec_lines.append('import utool as ut')
c.InteractiveShellApp.exec_lines.append('import pandas as pd')
c.InteractiveShellApp.exec_lines.append('pd.options.display.max_columns = 40')
c.InteractiveShellApp.exec_lines.append('pd.options.display.width = 160')
c.InteractiveShellApp.exec_lines.append('pd.options.display.max_rows = 20')
c.InteractiveShellApp.exec_lines.append('pd.options.display.float_format = lambda x: \'%.4f\' % (x,)')
c.InteractiveShellApp.exec_lines.append('import networkx as nx')
#c.InteractiveShellApp.exec_lines.append('import plottool as pt')
c.InteractiveShellApp.exec_lines.append('from os.path import *')
c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle')
#c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True')
import utool as ut
c.InteractiveShellApp.exec_lines.append(ut.codeblock(
"""
class classproperty(property):
def __get__(self, cls, owner):
return classmethod(self.fget).__get__(None, owner)()
class vim(object):
@classproperty
def focus(cls):
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),)
@classproperty
def copy(cls):
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),)
import utool as ut
import IPython
ipy = IPython.get_ipython()
lastline = ipy.history_manager.input_hist_parsed[-2]
ut.copy_text_to_clipboard(lastline)
# import utool as ut
import utool.util_ubuntu
utool.util_ubuntu.xctrl.do(
('focus', 'GVIM'),
('key', 'ctrl+v'),
('focus', 'x-terminal-emulator.X-terminal-emulator')
)
"""
))
#c.InteractiveShell.autoindent = True
#c.InteractiveShell.colors = 'LightBG'
#c.InteractiveShell.confirm_exit = False
#c.InteractiveShell.deep_reload = True
c.InteractiveShell.editor = 'gvim'
#c.InteractiveShell.xmode = 'Context'
# ENDBOCK
''')
ut.write_to(ipy_config_fpath, ipy_config_text)
def train_pyrf():
# boosting = 3
num_trees = 5
category = 'zebra_grevys'
#=================================
# Train / Detect Configurations
#=================================
train_config = {
'object_min_width': 32,
'object_min_height': 32,
'mine_negatives': True,
'mine_max_keep': 1,
'mine_exclude_categories': [category],
'mine_width_min': 128,
'mine_width_max': 512,
'mine_height_min': 128,
'mine_height_max': 512,
'neg_exclude_categories': [category],
'max_rois_pos': 900,
'max_rois_neg': 550,
'num_trees': num_trees,
}
detect_config = {
'save_detection_images': True,
'percentage_top': 0.40,
}
#=================================
# Train / Detect Initialization
#=================================
# Create detector
detector = Random_Forest_Detector()
# Gather Dataset
dataset_path = utool.unixpath('~/code/IBEIS2014/')
dataset = IBEIS_Data(dataset_path, **train_config)
results_path = join(utool.unixpath('~/code/pyrf/results'), category)
# pos_path = join(results_path, 'train-positives')
# neg_path = join(results_path, 'train-negatives')
# val_path = join(results_path, 'val')
test_path = join(results_path, 'test')
# test_pos_path = join(results_path, 'test-positives')
# test_neg_path = join(results_path, 'test-negatives')
detect_path = join(results_path, 'detect')
trees_path = join(results_path, 'trees')
# # Ensure result path for the category
# # rmtreedir(results_path)
# ensuredir(results_path)
# for phase in range(1, boosting + 1):
# print("*********************")
# print("Phase: %s" % phase)
# print("*********************")
# # raw_input()
# # =================================
# # Train Random Forest
# #=================================
# detector.train(dataset, category, pos_path, neg_path, val_path,
# test_path, test_pos_path, test_neg_path,
# trees_path, reshuffle=(phase == 1), **train_config)
# if phase < boosting:
# #=================================
# # Detect using Random Forest
# #=================================
# # Load forest, so we don't have to reload every time
# forest = detector.load(trees_path, category + '-', num_trees=(phase * num_trees))
# detector.set_detect_params(**detect_config)
# # Ensure output detection paths
# rmtreedir(detect_path)
# ensuredir(detect_path)
# # Calculate error on test set
# direct = Directory(test_path, include_file_extensions=["jpg"])
# accuracy_list = []
# image_filepath_list = direct.files()
# dst_filepath_list = [ join(detect_path, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
# predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
# for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
# image_path, image_filename = split(image_filepath)
# image = dataset[image_filename]
# accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
# accuracy_list.append(accuracy)
# progress = "%0.2f" % (float(index) / len(image_filepath_list))
# print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
# sys.stdout.flush()
# # image.show(prediction_list=predictions, category=category)
# print(' ' * 100, end='\r')
# print("TEST ERROR: %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
# #=================================
# # Eval and prep boosting train set
# #=================================
# detector.boosting(phase, forest, dataset, category, pos_path, neg_path,
# test_pos_path, test_neg_path, detect_path)
####################################
# New FAST
####################################
detector = Random_Forest_Detector(
scales='6 1.0 0.75 0.55 0.40 0.30 0.20'
)
# Ensure output detection paths
detect_path_temp = detect_path + "_1"
rmtreedir(detect_path_temp)
ensuredir(detect_path_temp)
# Load forest, so we don't have to reload every time
forest = detector.load(trees_path, category + '-', num_trees=25)
detector.set_detect_params(**detect_config)
# Calculate error on test set
direct = Directory(test_path, include_file_extensions=["jpg"])
accuracy_list = []
true_pos_list = []
false_pos_list = []
false_neg_list = []
image_filepath_list = direct.files()
dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
image_path, image_filename = split(image_filepath)
image = dataset[image_filename]
accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
accuracy_list.append(accuracy)
true_pos_list.append(true_pos)
false_pos_list.append(false_pos)
false_neg_list.append(false_neg)
progress = "%0.2f" % (float(index) / len(image_filepath_list))
print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
sys.stdout.flush()
# image.show(prediction_list=predictions, category=category)
print(' ' * 100, end='\r')
print("1 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
print("1 TEST TRUE POS : %d" % (sum(true_pos_list)))
print("1 TEST FALSE POS : %d" % (sum(false_pos_list)))
print("1 TEST FALSE NEG : %d" % (sum(false_neg_list)))
####################################
# New SLOW
####################################
detector = Random_Forest_Detector(
scales='11 1.5 1.25 1.0 0.8 0.64 0.51 0.41 0.33 0.26 0.21 0.17'
)
# Ensure output detection paths
detect_path_temp = detect_path + "_2"
rmtreedir(detect_path_temp)
ensuredir(detect_path_temp)
# Load forest, so we don't have to reload every time
forest = detector.load(trees_path, category + '-', num_trees=25)
detector.set_detect_params(**detect_config)
# Calculate error on test set
direct = Directory(test_path, include_file_extensions=["jpg"])
accuracy_list = []
true_pos_list = []
false_pos_list = []
false_neg_list = []
image_filepath_list = direct.files()
dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ]
predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True)
for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)):
image_path, image_filename = split(image_filepath)
image = dataset[image_filename]
accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category)
accuracy_list.append(accuracy)
true_pos_list.append(true_pos)
false_pos_list.append(false_pos)
false_neg_list.append(false_neg)
progress = "%0.2f" % (float(index) / len(image_filepath_list))
print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r')
sys.stdout.flush()
# image.show(prediction_list=predictions, category=category)
print(' ' * 100, end='\r')
print("2 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list))))
print("2 TEST TRUE POS : %d" % (sum(true_pos_list)))
print("2 TEST FALSE POS : %d" % (sum(false_pos_list)))
print("2 TEST FALSE NEG : %d" % (sum(false_neg_list)))
def test_pyrf():
r"""
CommandLine:
python run_tests.py --test-test_pyrf
Example:
>>> # ENABLE_DOCTEST
>>> from run_tests import * # NOQA
>>> result = test_pyrf()
>>> print(result)
"""
#=================================
# Initialization
#=================================
category = 'zebra_plains'
#detect_config = {
# 'save_detection_images': True,
# 'percentage_top': 0.40,
#}
testdata_dir = ut.unixpath('~/code/pyrf/results')
# assert ut.checkpath(testdata_dir)
if ut.get_argflag('--vd'):
print(ut.ls(testdata_dir))
# Create detector
detector = Random_Forest_Detector()
test_path = ut.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool')
models_path = ut.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool')
trees_path = join(models_path, category)
detect_path = join(test_path, category, 'detect')
ut.ensuredir(detect_path)
ut.ensuredir(test_path)
ut.ensuredir(trees_path)
#=================================
# Load Input Images
#=================================
# Get input images
big_gpath_list = ut.list_images(test_path, fullpath=True, recursive=False)
print(big_gpath_list)
# Resize images to standard size
if ut.get_argflag('--small'):
big_gpath_list = big_gpath_list[0:8]
#big_gpath_list = big_gpath_list[0:8]
output_dir = join(test_path, 'resized')
std_gpath_list = resize_imagelist_to_sqrtarea(big_gpath_list,
sqrt_area=800,
output_dir=output_dir,
checkexists=True)
dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list]
#ut.view_directory(test_path)
#ut.view_directory('.')
print(std_gpath_list)
num_images = len(std_gpath_list)
#assert num_images == 16, 'the test has diverged!'
print('Testing on %r images' % num_images)
#=================================
# Load Pretrained Forests
#=================================
# Load forest, so we don't have to reload every time
trees_fpath_list = ut.ls(trees_path, '*.txt')
#forest = detector.load(trees_path, category + '-')
forest = detector.forest(trees_fpath_list)
#detector.set_detect_params(**detect_config)
results_list1 = []
#=================================
# Detect using Random Forest
#=================================
with ut.Timer('[test_pyrf] for loop detector.detect') as t1:
if not ut.get_argflag('--skip1'):
results_list1 = detector.detect(forest, std_gpath_list, output_gpath_list=dst_gpath_list)
#for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)):
# #img_fname = split(img_fpath)[1]
# #dst_fpath = join(detect_path, img_fname)
# #print(' * img_fpath = %r' % img_fpath)
# #print(' * dst_fpath = %r' % dst_fpath)
# with ut.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)):
# results = detector.detect(forest, img_fpath, dst_fpath)
# results_list1.append(results)
# print('num results = %r' % len(results))
#else:
# print('...skipped')
#with ut.Timer('[test_pyrf] detector.detect_many') as t2:
# results_list2 = detector.detect_many(forest, std_gpath_list,
# dst_gpath_list, use_openmp=True)
detector.free_forest(forest)
print('')
print('+ --------------')
print('| total time1: %r' % t1.ellapsed)
#print('| total time2: %r' % t2.ellapsed)
print('|')
print('| num results1 = %r' % (list(map(len, results_list1))))
#print('| num results2 = %r' % (list(map(len, results_list2))))
#assert results_list2 == results_list1
return locals()
