def get_dist_app_fpath(): app_fpath = ut.unixpath('dist/ibeis/IBEISApp') if ut.DARWIN: app_fpath = ut.unixpath('dist/IBEIS.app/Contents/MacOS/IBEISApp') if ut.WIN32: app_fpath += '.exe' return app_fpath
def get_cyth_pxd_path(py_fpath): """ >>> py_fpath = '/foo/vtool/vtool/keypoint.py' >>> cy_fpath = get_cyth_pxd_path(py_fpath) >>> print(cy_fpath) /foo/vtool/vtool/_keypoint_cyth.pxd """ dpath, fname = split(py_fpath) name, ext = splitext(fname) assert ext == '.py', 'not a python file' cy_fpath = utool.unixpath(join(dpath, get_cyth_name(name) + '.pxd')) return cy_fpath
def translate_all(): """ Translates a all python paths in directory """ dpaths = utool.ls_moduledirs('.') #print('[cyth] translate_all: %r' % (dpaths,)) globkw = {'recursive': True, 'with_dirs': False, 'with_files': True} # Find all unique python files in directory fpaths_iter = [ utool.glob(utool.unixpath(dpath), '*.py', **globkw) for dpath in dpaths ] fpath_iter = utool.iflatten(fpaths_iter) abspath_iter = map(utool.unixpath, fpath_iter) fpath_list = list(set(list(abspath_iter))) #print('[cyth] translate_all: %s' % ('\n'.join(fpath_list),)) # Try to translate each translate(*fpath_list)
def translate_all(): """ Translates a all python paths in directory """ dpaths = utool.ls_moduledirs('.') #print('[cyth] translate_all: %r' % (dpaths,)) globkw = { 'recursive': True, 'with_dirs': False, 'with_files': True } # Find all unique python files in directory fpaths_iter = [utool.glob(utool.unixpath(dpath), '*.py', **globkw) for dpath in dpaths] fpath_iter = utool.iflatten(fpaths_iter) abspath_iter = map(utool.unixpath, fpath_iter) fpath_list = list(set(list(abspath_iter))) #print('[cyth] translate_all: %s' % ('\n'.join(fpath_list),)) # Try to translate each translate(*fpath_list)
def translate(*paths): """ Translates a list of paths """ cy_bench_list = [] for fpath in paths: if isfile(fpath): abspath = utool.unixpath(fpath) cy_bench = translate_fpath(abspath) if cy_bench is not None: cy_bench_list.append(cy_bench) if len(cy_bench_list) > 0: runbench_shtext = cyth_benchmarks.build_runbench_shell_text(cy_bench_list) runbench_pytext = cyth_benchmarks.build_runbench_pyth_text(cy_bench_list) utool.write_to('_old_run_cyth_benchmarks.sh', runbench_shtext) utool.write_to('run_cyth_benchmarks.py', runbench_pytext) #try: os.chmod('_old_run_cyth_benchmarks.sh', 33277) os.chmod('run_cyth_benchmarks.py', 33277)
def translate(*paths): """ Translates a list of paths """ cy_bench_list = [] for fpath in paths: if isfile(fpath): abspath = utool.unixpath(fpath) cy_bench = translate_fpath(abspath) if cy_bench is not None: cy_bench_list.append(cy_bench) if len(cy_bench_list) > 0: runbench_shtext = cyth_benchmarks.build_runbench_shell_text( cy_bench_list) runbench_pytext = cyth_benchmarks.build_runbench_pyth_text( cy_bench_list) utool.write_to('_old_run_cyth_benchmarks.sh', runbench_shtext) utool.write_to('run_cyth_benchmarks.py', runbench_pytext) #try: os.chmod('_old_run_cyth_benchmarks.sh', 33277) os.chmod('run_cyth_benchmarks.py', 33277)
pkgname = basename(repodir) packages = utool.ls_moduledirs(repodir, full=False) print(pkgname) setup_text = setup_text_fmt.format( packages=packages, repodir=repodir, timestamp=timestamp, pkgname=pkgname, ) return setup_text if __name__ == '__main__': writeflag = utool.get_argflag(('--write', '-w')) overwriteflag = utool.get_argflag(('--yes', '-y')) repodir = utool.unixpath(os.getcwd()) print('[utool] making setup.py for: %r' % repodir) setup_text = make_setup(repodir) if writeflag: setup_fpath = utool.unixjoin(repodir, 'setup.py') if utool.checkpath(setup_fpath): confirm_flag = overwriteflag else: confirm_flag = True if confirm_flag: utool.write_to(setup_fpath, setup_text) else: print('setup.py file exists not writing') else: print(setup_text)
def bench_fpath_to_modname(bench): bench_upath = utool.unixpath(bench) bench_relpath = relpath(bench_upath, os.getcwd()) bench_relname, _ = splitext(bench_relpath) bench_modname = bench_relname.replace('\\', '/').replace('/', '.') return bench_modname
DCVS_DEFAULT = ut.ParamInfoList('distinctivness', [ ut.ParamInfo('dcvs_power', 1.0, 'p', varyvals=[.5, 1.0, 1.5, 2.0]), ut.ParamInfo('dcvs_min_clip', .2, 'mn', varyvals=[.2, .02, .03][0:1]), ut.ParamInfo('dcvs_max_clip', .5, 'mx', varyvals=[.05, .3, .4, .45, .5, 1.0][1:4]), ut.ParamInfo('dcvs_K', 5, 'dcvsK', varyvals=[5, 7, 15][0:1]), ]) DISTINCTIVENESS_NORMALIZER_CACHE = {} BASELINE_DISTINCTIVNESS_URLS = { # TODO: Populate 'zebra_grevys': const.ZIPPED_URLS.GZ_DISTINCTIVE, 'zebra_plains': const.ZIPPED_URLS.PZ_DISTINCTIVE, } PUBLISH_DIR = ut.unixpath('~/Dropbox/IBEIS') def testdata_distinctiveness(): """ Example: >>> # SLOW_DOCTEST >>> from ibeis.algo.hots.distinctiveness_normalizer import * # NOQA >>> dstcnvs_normer, qreq_ = testdata_distinctiveness() """ import ibeis # build test data db = ut.get_argval('--db', str, 'testdb1') species = ut.get_argval('--species', str, None) aid = ut.get_argval('--aid', int, None) ibs = ibeis.opendb(db)
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None, repodir=None, exclude_list=[]): """ Autogeneration function TODO move to util_autogen or just depricate Examples: >>> from utool.util_tests import * # NOQA >>> import utool # NOQA >>> testdirs = ['~/code/ibeis/test_ibs*.py'] """ import utool as ut from os.path import relpath, join, dirname # NOQA exclude_list += ['__init__.py'] # General format of the testing script script_fmtstr = ut.codeblock( r''' #!/bin/bash # Runs all tests # Win32 path hacks export CWD=$(pwd) export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")" # <CORRECT_PYTHON> # GET CORRECT PYTHON ON ALL PLATFORMS export SYSNAME="$(expr substr $(uname -s) 1 10)" if [ "$SYSNAME" = "MINGW32_NT" ]; then export PYEXE=python else if [ "$PYMAJOR" = "3" ]; then # virtual env? export PYEXE=python else export PYEXE=python2.7 fi fi # </CORRECT_PYTHON> PRINT_DELIMETER() {{ printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n" }} export TEST_ARGV="{test_argvs} $@" {dirdef_block} # Default tests to run set_test_flags() {{ export DEFAULT=$1 {testdefault_block} }} set_test_flags OFF {testdefaulton_block} # Parse for bash commandline args for i in "$@" do case $i in --testall) set_test_flags ON ;; esac {testcmdline_block} done BEGIN_TESTS() {{ cat <<EOF {runtests_bubbletext} EOF echo "BEGIN: TEST_ARGV=$TEST_ARGV" PRINT_DELIMETER num_passed=0 num_ran=0 export FAILED_TESTS='' }} RUN_TEST() {{ echo "RUN_TEST: $@" export TEST="$PYEXE $@ $TEST_ARGV" $TEST export RETURN_CODE=$? echo "RETURN_CODE=$RETURN_CODE" PRINT_DELIMETER num_ran=$(($num_ran + 1)) if [ "$RETURN_CODE" == "0" ] ; then num_passed=$(($num_passed + 1)) fi if [ "$RETURN_CODE" != "0" ] ; then export FAILED_TESTS="$FAILED_TESTS\n$TEST" fi }} END_TESTS() {{ echo "RUN_TESTS: DONE" if [ "$FAILED_TESTS" != "" ] ; then echo "-----" printf "Failed Tests:" printf "$FAILED_TESTS\n" printf "$FAILED_TESTS\n" >> failed_shelltests.txt echo "-----" fi echo "$num_passed / $num_ran tests passed" }} #--------------------------------------------- # START TESTS BEGIN_TESTS {quicktest_block} {test_block} #--------------------------------------------- # END TESTING END_TESTS ''') testcmdline_fmtstr = ut.codeblock( r''' case $i in --notest{header_lower}) export {testflag}=OFF ;; esac case $i in --test{header_lower}) export {testflag}=ON ;; esac ''') header_test_block_fmstr = ut.codeblock( r''' #--------------------------------------------- #{header_text} if [ "${testflag}" = "ON" ] ; then cat <<EOF {header_bubble_text} EOF {testlines_block} fi ''') #specialargv = '--noshow' specialargv = '' testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}' testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}' def format_testline(fpath, dirvar): if dirvar is None: return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv) else: return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv) default_flag_line_list = [] defaulton_flag_line_list = [] testcmdline_list = [] dirdef_list = [] header_test_block_list = [] known_tests = ut.ddict(list) # Tests to always run if quick_tests is not None: quicktest_block = '\n'.join( ['# Quick Tests (always run)'] + ['RUN_TEST ' + testline for testline in quick_tests]) else: quicktest_block = '# No quick tests' # Loop over different test types for testdef_tup in test_headers: header, default, modname, dpath, pats, testcmds = testdef_tup # Build individual test type information header_upper = header.upper() header_lower = header.lower() testflag = header_upper + '_TEST' if modname is not None: dirvar = header_upper + '_DIR' dirdef = ''.join([ 'export {dirvar}=$($PYEXE -c "', 'import os, {modname};', 'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))', '")']).format(dirvar=dirvar, modname=modname) dirdef_list.append(dirdef) else: dirvar = None # Build test dir #dirvar = header_upper + '_DIR' #dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname) #dirdef_list.append(dirdef) # Build command line flags default_flag_line = 'export {testflag}=$DEFAULT'.format(testflag=testflag) if default: defaulton_flag_line = 'export {testflag}=ON'.format(testflag=testflag) defaulton_flag_line_list.append(defaulton_flag_line) testcmdline_fmtdict = dict(header_lower=header_lower, testflag=testflag,) testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict) #ut.ls(dpath) # VERY HACK BIT OF CODE # Get list of tests from patterns if testcmds is None: if modname is not None: module = __import__(modname) repo_path = dirname(dirname(module.__file__)) else: repo_path = repodir dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath)) if header_upper == 'OTHER': # Hacky way to grab any other tests not explicitly seen in this directory _testfpath_list = list(set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_])) #_testfpath_list = ut.glob(dpath_, '*.py') #set(known_tests[dpath_]) else: _testfpath_list = ut.flatten([ut.glob(dpath_, pat) for pat in pats]) def not_excluded(x): return not any([x.find(exclude) > -1 for exclude in exclude_list]) _testfpath_list = list(filter(not_excluded, _testfpath_list)) known_tests[dpath_].extend(_testfpath_list) #print(_testfpath_list) testfpath_list = [util_path.unixjoin(dpath, relpath(fpath, dpath_)) for fpath in _testfpath_list] testline_list = [format_testline(fpath, dirvar) for fpath in testfpath_list] else: testline_list = testcmds testlines_block = ut.indentjoin(testline_list).strip('\n') # Construct test block for this type header_text = header_upper + ' TESTS' headerfont = 'cybermedium' header_bubble_text = ut.indent(ut.bubbletext(header_text, headerfont).strip()) header_test_block_dict = dict( testflag=testflag, header_text=header_text, testlines_block=testlines_block, header_bubble_text=header_bubble_text,) header_test_block = header_test_block_fmstr.format(**header_test_block_dict) # Append to script lists header_test_block_list.append(header_test_block) default_flag_line_list.append(default_flag_line) testcmdline_list.append(testcmdline) runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge') test_block = '\n'.join(header_test_block_list) dirdef_block = '\n'.join(dirdef_list) testdefault_block = ut.indent('\n'.join(default_flag_line_list)) testdefaulton_block = '\n'.join(defaulton_flag_line_list) testcmdline_block = '\n'.join(testcmdline_list) script_fmtdict = dict( quicktest_block=quicktest_block, runtests_bubbletext=runtests_bubbletext, test_argvs=test_argvs, dirdef_block=dirdef_block, testdefault_block=testdefault_block, testdefaulton_block=testdefaulton_block, testcmdline_block=testcmdline_block, test_block=test_block,) script_text = script_fmtstr.format(**script_fmtdict) return script_text
module_name = ut.get_modname_from_modpath(module_path) IMPORT_TUPLES = util_importer.make_import_tuples(module_path, exclude_modnames=exclude_modnames) initstr = util_importer.make_initstr(module_name, IMPORT_TUPLES) regen_command = 'cd %s\n' % (module_path) regen_command += ' makeinit.py' if len(exclude_modnames ) > 0: regen_command += ' -x ' + ' '.join(exclude_modnames) regen_block = (ut.codeblock(''' """ Regen Command: {regen_command} """ ''').format(regen_command=regen_command)) print('### __init__.py ###') print(initstr) print('\nIMPORT_TUPLES = ' + ut.list_str(IMPORT_TUPLES)) print(regen_block) if __name__ == '__main__': exclude_modnames = ut.get_argval(('--exclude', '-x'), list, default=[]) modname = ut.get_argval('--modname', str, default=None) if modname is None: dpath = os.getcwd() else: dpath = ut.get_modpath_from_modname(modname) module_path = ut.unixpath(dpath) makeinit(module_path, exclude_modnames=exclude_modnames) print('# autogenerated __init__.py for: %r' % module_path)
def write_default_ipython_profile(): """ CommandLine: python ~/local/init/init_ipython_config.py python -c "import utool as ut; ut.vd(ut.unixpath('~/.ipython/profile_default'))" python -c "import utool as ut; ut.editfile(ut.unixpath('~/.ipython/profile_default/ipython_config.py'))" References: http://2sn.org/python/ipython_config.py """ dpath = ut.unixpath('~/.ipython/profile_default') ut.ensuredir(dpath, info=True, verbose=True) ipy_config_fpath = ut.unixjoin(dpath, 'ipython_config.py') ipy_config_text = ut.codeblock( r''' # STARTBLOCK c = get_config() # NOQA c.InteractiveShellApp.exec_lines = [] future_line = ( 'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals') c.InteractiveShellApp.exec_lines.append(future_line) # Fix sip versions try: import sip # http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string sip.setapi('QVariant', 2) sip.setapi('QString', 2) sip.setapi('QTextStream', 2) sip.setapi('QTime', 2) sip.setapi('QUrl', 2) sip.setapi('QDate', 2) sip.setapi('QDateTime', 2) if hasattr(sip, 'setdestroyonexit'): sip.setdestroyonexit(False) # This prevents a crash on windows except ImportError as ex: pass except ValueError as ex: print('Warning: Value Error: %s' % str(ex)) pass c.InteractiveShellApp.exec_lines.append('%load_ext autoreload') c.InteractiveShellApp.exec_lines.append('%autoreload 2') #c.InteractiveShellApp.exec_lines.append('%pylab qt4') c.InteractiveShellApp.exec_lines.append('import numpy as np') c.InteractiveShellApp.exec_lines.append('import utool as ut') #c.InteractiveShellApp.exec_lines.append('import plottool as pt') c.InteractiveShellApp.exec_lines.append('from os.path import *') c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle') #c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True') import utool as ut c.InteractiveShellApp.exec_lines.append(ut.codeblock( """ class classproperty(property): def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() class vim(object): @classproperty def focus(cls): import utool.util_ubuntu utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),) @classproperty def copy(cls): import utool.util_ubuntu utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),) import utool as ut import IPython ipy = IPython.get_ipython() lastline = ipy.history_manager.input_hist_parsed[-2] ut.copy_text_to_clipboard(lastline) # import utool as ut import utool.util_ubuntu utool.util_ubuntu.xctrl.do( ('focus', 'GVIM'), ('key', 'ctrl+v'), ('focus', 'x-terminal-emulator.X-terminal-emulator') ) """ )) #c.InteractiveShell.autoindent = True #c.InteractiveShell.colors = 'LightBG' #c.InteractiveShell.confirm_exit = False #c.InteractiveShell.deep_reload = True c.InteractiveShell.editor = 'gvim' #c.InteractiveShell.xmode = 'Context' # ENDBOCK ''' ) ut.write_to(ipy_config_fpath, ipy_config_text)
'dcvs_min_clip', 0.2, 'mn', varyvals=[0.2, 0.02, 0.03][0:1]), ut.ParamInfo('dcvs_max_clip', 0.5, 'mx', varyvals=[0.05, 0.3, 0.4, 0.45, 0.5, 1.0][1:4]), ut.ParamInfo('dcvs_K', 5, 'dcvsK', varyvals=[5, 7, 15][0:1]), ], ) DISTINCTIVENESS_NORMALIZER_CACHE = {} BASELINE_DISTINCTIVNESS_URLS = { # TODO: Populate 'zebra_grevys': const.ZIPPED_URLS.GZ_DISTINCTIVE, 'zebra_plains': const.ZIPPED_URLS.PZ_DISTINCTIVE, } PUBLISH_DIR = ut.unixpath('~/Dropbox/IBEIS') def testdata_distinctiveness(): """ Example: >>> # SLOW_DOCTEST >>> from wbia.algo.hots.distinctiveness_normalizer import * # NOQA >>> dstcnvs_normer, qreq_ = testdata_distinctiveness() """ import wbia # build test data db = ut.get_argval('--db', str, 'testdb1') species = ut.get_argval('--species', str, None) aid = ut.get_argval('--aid', int, None)
def make_run_tests_script_text(test_headers, test_argvs, quick_tests=None, repodir=None, exclude_list=[]): """ Autogeneration function TODO move to util_autogen or just depricate Examples: >>> from utool.util_tests import * # NOQA >>> import utool # NOQA >>> testdirs = ['~/code/ibeis/test_ibs*.py'] """ import utool as ut from os.path import relpath, join, dirname # NOQA exclude_list += ['__init__.py'] # General format of the testing script script_fmtstr = ut.codeblock(r''' #!/bin/bash # Runs all tests # Win32 path hacks export CWD=$(pwd) export PYMAJOR="$(python -c "import sys; print(sys.version_info[0])")" # <CORRECT_PYTHON> # GET CORRECT PYTHON ON ALL PLATFORMS export SYSNAME="$(expr substr $(uname -s) 1 10)" if [ "$SYSNAME" = "MINGW32_NT" ]; then export PYEXE=python else if [ "$PYMAJOR" = "3" ]; then # virtual env? export PYEXE=python else export PYEXE=python2.7 fi fi # </CORRECT_PYTHON> PRINT_DELIMETER() {{ printf "\n#\n#\n#>>>>>>>>>>> next_test\n\n" }} export TEST_ARGV="{test_argvs} $@" {dirdef_block} # Default tests to run set_test_flags() {{ export DEFAULT=$1 {testdefault_block} }} set_test_flags OFF {testdefaulton_block} # Parse for bash commandline args for i in "$@" do case $i in --testall) set_test_flags ON ;; esac {testcmdline_block} done BEGIN_TESTS() {{ cat <<EOF {runtests_bubbletext} EOF echo "BEGIN: TEST_ARGV=$TEST_ARGV" PRINT_DELIMETER num_passed=0 num_ran=0 export FAILED_TESTS='' }} RUN_TEST() {{ echo "RUN_TEST: $@" export TEST="$PYEXE $@ $TEST_ARGV" $TEST export RETURN_CODE=$? echo "RETURN_CODE=$RETURN_CODE" PRINT_DELIMETER num_ran=$(($num_ran + 1)) if [ "$RETURN_CODE" == "0" ] ; then num_passed=$(($num_passed + 1)) fi if [ "$RETURN_CODE" != "0" ] ; then export FAILED_TESTS="$FAILED_TESTS\n$TEST" fi }} END_TESTS() {{ echo "RUN_TESTS: DONE" if [ "$FAILED_TESTS" != "" ] ; then echo "-----" printf "Failed Tests:" printf "$FAILED_TESTS\n" printf "$FAILED_TESTS\n" >> failed_shelltests.txt echo "-----" fi echo "$num_passed / $num_ran tests passed" }} #--------------------------------------------- # START TESTS BEGIN_TESTS {quicktest_block} {test_block} #--------------------------------------------- # END TESTING END_TESTS ''') testcmdline_fmtstr = ut.codeblock(r''' case $i in --notest{header_lower}) export {testflag}=OFF ;; esac case $i in --test{header_lower}) export {testflag}=ON ;; esac ''') header_test_block_fmstr = ut.codeblock(r''' #--------------------------------------------- #{header_text} if [ "${testflag}" = "ON" ] ; then cat <<EOF {header_bubble_text} EOF {testlines_block} fi ''') #specialargv = '--noshow' specialargv = '' testline_fmtstr = 'RUN_TEST ${dirvar}/{fpath} {specialargv}' testline_fmtstr2 = 'RUN_TEST {fpath} {specialargv}' def format_testline(fpath, dirvar): if dirvar is None: return testline_fmtstr2.format(fpath=fpath, specialargv=specialargv) else: return testline_fmtstr.format(dirvar=dirvar, fpath=fpath, specialargv=specialargv) default_flag_line_list = [] defaulton_flag_line_list = [] testcmdline_list = [] dirdef_list = [] header_test_block_list = [] known_tests = ut.ddict(list) # Tests to always run if quick_tests is not None: quicktest_block = '\n'.join( ['# Quick Tests (always run)'] + ['RUN_TEST ' + testline for testline in quick_tests]) else: quicktest_block = '# No quick tests' # Loop over different test types for testdef_tup in test_headers: header, default, modname, dpath, pats, testcmds = testdef_tup # Build individual test type information header_upper = header.upper() header_lower = header.lower() testflag = header_upper + '_TEST' if modname is not None: dirvar = header_upper + '_DIR' dirdef = ''.join([ 'export {dirvar}=$($PYEXE -c "', 'import os, {modname};', 'print(str(os.path.dirname(os.path.dirname({modname}.__file__))))', '")' ]).format(dirvar=dirvar, modname=modname) dirdef_list.append(dirdef) else: dirvar = None # Build test dir #dirvar = header_upper + '_DIR' #dirdef = 'export {dirvar}={dirname}'.format(dirvar=dirvar, dirname=dirname) #dirdef_list.append(dirdef) # Build command line flags default_flag_line = 'export {testflag}=$DEFAULT'.format( testflag=testflag) if default: defaulton_flag_line = 'export {testflag}=ON'.format( testflag=testflag) defaulton_flag_line_list.append(defaulton_flag_line) testcmdline_fmtdict = dict( header_lower=header_lower, testflag=testflag, ) testcmdline = testcmdline_fmtstr.format(**testcmdline_fmtdict) #ut.ls(dpath) # VERY HACK BIT OF CODE # Get list of tests from patterns if testcmds is None: if modname is not None: module = __import__(modname) repo_path = dirname(dirname(module.__file__)) else: repo_path = repodir dpath_ = ut.unixpath(util_path.unixjoin(repo_path, dpath)) if header_upper == 'OTHER': # Hacky way to grab any other tests not explicitly seen in this directory _testfpath_list = list( set(ut.glob(dpath_, '*.py')) - set(known_tests[dpath_])) #_testfpath_list = ut.glob(dpath_, '*.py') #set(known_tests[dpath_]) else: _testfpath_list = ut.flatten( [ut.glob(dpath_, pat) for pat in pats]) def not_excluded(x): return not any( [x.find(exclude) > -1 for exclude in exclude_list]) _testfpath_list = list(filter(not_excluded, _testfpath_list)) known_tests[dpath_].extend(_testfpath_list) #print(_testfpath_list) testfpath_list = [ util_path.unixjoin(dpath, relpath(fpath, dpath_)) for fpath in _testfpath_list ] testline_list = [ format_testline(fpath, dirvar) for fpath in testfpath_list ] else: testline_list = testcmds testlines_block = ut.indentjoin(testline_list).strip('\n') # Construct test block for this type header_text = header_upper + ' TESTS' headerfont = 'cybermedium' header_bubble_text = ut.indent( ut.bubbletext(header_text, headerfont).strip()) header_test_block_dict = dict( testflag=testflag, header_text=header_text, testlines_block=testlines_block, header_bubble_text=header_bubble_text, ) header_test_block = header_test_block_fmstr.format( **header_test_block_dict) # Append to script lists header_test_block_list.append(header_test_block) default_flag_line_list.append(default_flag_line) testcmdline_list.append(testcmdline) runtests_bubbletext = ut.bubbletext('RUN TESTS', 'cyberlarge') test_block = '\n'.join(header_test_block_list) dirdef_block = '\n'.join(dirdef_list) testdefault_block = ut.indent('\n'.join(default_flag_line_list)) testdefaulton_block = '\n'.join(defaulton_flag_line_list) testcmdline_block = '\n'.join(testcmdline_list) script_fmtdict = dict( quicktest_block=quicktest_block, runtests_bubbletext=runtests_bubbletext, test_argvs=test_argvs, dirdef_block=dirdef_block, testdefault_block=testdefault_block, testdefaulton_block=testdefaulton_block, testcmdline_block=testcmdline_block, test_block=test_block, ) script_text = script_fmtstr.format(**script_fmtdict) return script_text
page = urllib2.urlopen(req) page_str = page.read() page_str = ut.ensure_unicode(page_str) next = False lines = page_str.split('\n') for index, x in enumerate(lines): if next: print(x) import parse url_suffix = parse.parse('{foo}href="{href}"{other}', x)['href'] url = r'https://cmake.org' + url_suffix break if 'Linux x86_64' in x: next = True url = url.replace('.sh', '.tar.gz') cmake_unzipped_fpath = ut.grab_zipped_url(url) install_prefix = ut.unixpath('~') for dname in ['bin', 'doc', 'man', 'share']: install_dst = join(install_prefix, dname) install_src = join(cmake_unzipped_fpath, dname) # FIXME: this broke #ut.util_path.copy(install_src, install_dst) # HACK AROUND IT from os.path import dirname cmd = str('cp -r "' + install_src + '" "' + dirname(install_dst) + '"') print(cmd) ut.cmd(cmd) #os.system(cmd) print(cmake_unzipped_fpath)
def test_pyrf(): category = 'zebra_plains' detect_config = { 'save_detection_images': True, 'save_scales': True, 'percentage_top': 0.40, } #================================= # Train / Detect Initialization #================================= testdata_dir = utool.unixpath('~/code/pyrf/results') # assert utool.checkpath(testdata_dir) if utool.get_argflag('--vd'): print(utool.ls(testdata_dir)) # Create detector detector = Random_Forest_Detector() test_path = utool.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool') models_path = utool.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool') trees_path = join(models_path, category) results_path = join(utool.unixpath('~/code/pyrf/results'), category) # detect_path = join(results_path, 'detect') trees_path = join(results_path, 'trees') detect_path = join(test_path, category, 'detect') utool.ensuredir(detect_path) utool.ensuredir(test_path) utool.ensuredir(trees_path) #================================= # Detect using Random Forest #================================= # Get input images from vtool import image big_gpath_list = utool.list_images(test_path, fullpath=True, recursive=False) print(big_gpath_list) # Resize images to standard size if utool.get_argflag('--small'): big_gpath_list = big_gpath_list[0:8] #big_gpath_list = big_gpath_list[0:8] output_dir = join(test_path, 'resized') std_gpath_list = image.resize_imagelist_to_sqrtarea(big_gpath_list, sqrt_area=800, output_dir=output_dir, checkexists=True) dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list] #utool.view_directory(test_path) #utool.view_directory('.') print(std_gpath_list) num_images = len(std_gpath_list) #assert num_images == 16, 'the test has diverged!' print('Testing on %r images' % num_images) # Load forest, so we don't have to reload every time forest = detector.load(trees_path, category + '-', num_trees=25) detector.set_detect_params(**detect_config) results_list1 = [] with utool.Timer('[test_pyrf] for loop detector.detect') as t1: if not utool.get_argflag('--skip1'): for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)): #img_fname = split(img_fpath)[1] #dst_fpath = join(detect_path, img_fname) #print(' * img_fpath = %r' % img_fpath) #print(' * dst_fpath = %r' % dst_fpath) with utool.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)): results = detector.detect(forest, img_fpath, dst_fpath) results_list1.append(results) print('num results = %r' % len(results)) else: print('...skipped') # with utool.Timer('[test_pyrf] detector.detect_many') as t2: # results_list2 = detector.detect_many(forest, std_gpath_list, # dst_gpath_list, use_openmp=True) print('') print('+ --------------') print('| total time1: %r' % t1.ellapsed) # print('| total time2: %r' % t2.ellapsed) print('|') print('| num results1 = %r' % (list(map(len, results_list1)))) # print('| num results2 = %r' % (list(map(len, results_list2)))) #assert results_list2 == results_list1 return locals()
def write_default_ipython_profile(): """ CommandLine: python ~/local/init/init_ipython_config.py python -c "import utool as ut; ut.vd(ut.unixpath('~/.ipython/profile_default'))" python -c "import utool as ut; ut.editfile(ut.unixpath('~/.ipython/profile_default/ipython_config.py'))" References: http://2sn.org/python/ipython_config.py """ dpath = ut.unixpath('~/.ipython/profile_default') ut.ensuredir(dpath, info=True, verbose=True) ipy_config_fpath = ut.unixjoin(dpath, 'ipython_config.py') ipy_config_text = ut.codeblock(r''' # STARTBLOCK c = get_config() # NOQA c.InteractiveShellApp.exec_lines = [] import six if six.PY2: future_line = ( 'from __future__ import absolute_import, division, print_function, with_statement, unicode_literals') c.InteractiveShellApp.exec_lines.append(future_line) # Fix sip versions try: import sip # http://stackoverflow.com/questions/21217399/pyqt4-qtcore-qvariant-object-instead-of-a-string sip.setapi('QVariant', 2) sip.setapi('QString', 2) sip.setapi('QTextStream', 2) sip.setapi('QTime', 2) sip.setapi('QUrl', 2) sip.setapi('QDate', 2) sip.setapi('QDateTime', 2) if hasattr(sip, 'setdestroyonexit'): sip.setdestroyonexit(False) # This prevents a crash on windows except ImportError as ex: pass except ValueError as ex: print('Warning: Value Error: %s' % str(ex)) pass c.InteractiveShellApp.exec_lines.append('%load_ext autoreload') c.InteractiveShellApp.exec_lines.append('%autoreload 2') #c.InteractiveShellApp.exec_lines.append('%pylab qt4') c.InteractiveShellApp.exec_lines.append('import numpy as np') c.InteractiveShellApp.exec_lines.append('import ubelt as ub') c.InteractiveShellApp.exec_lines.append('import utool as ut') c.InteractiveShellApp.exec_lines.append('import pandas as pd') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_columns = 40') c.InteractiveShellApp.exec_lines.append('pd.options.display.width = 160') c.InteractiveShellApp.exec_lines.append('pd.options.display.max_rows = 20') c.InteractiveShellApp.exec_lines.append('pd.options.display.float_format = lambda x: \'%.4f\' % (x,)') c.InteractiveShellApp.exec_lines.append('import networkx as nx') #c.InteractiveShellApp.exec_lines.append('import plottool as pt') c.InteractiveShellApp.exec_lines.append('from os.path import *') c.InteractiveShellApp.exec_lines.append('from six.moves import cPickle as pickle') #c.InteractiveShellApp.exec_lines.append('if \'verbose\' not in vars():\\n verbose = True') import utool as ut c.InteractiveShellApp.exec_lines.append(ut.codeblock( """ class classproperty(property): def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() class vim(object): @classproperty def focus(cls): import utool.util_ubuntu utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),) @classproperty def copy(cls): import utool.util_ubuntu utool.util_ubuntu.xctrl.do(('focus', 'GVIM'),) import utool as ut import IPython ipy = IPython.get_ipython() lastline = ipy.history_manager.input_hist_parsed[-2] ut.copy_text_to_clipboard(lastline) # import utool as ut import utool.util_ubuntu utool.util_ubuntu.xctrl.do( ('focus', 'GVIM'), ('key', 'ctrl+v'), ('focus', 'x-terminal-emulator.X-terminal-emulator') ) """ )) #c.InteractiveShell.autoindent = True #c.InteractiveShell.colors = 'LightBG' #c.InteractiveShell.confirm_exit = False #c.InteractiveShell.deep_reload = True c.InteractiveShell.editor = 'gvim' #c.InteractiveShell.xmode = 'Context' # ENDBOCK ''') ut.write_to(ipy_config_fpath, ipy_config_text)
def train_pyrf(): # boosting = 3 num_trees = 5 category = 'zebra_grevys' #================================= # Train / Detect Configurations #================================= train_config = { 'object_min_width': 32, 'object_min_height': 32, 'mine_negatives': True, 'mine_max_keep': 1, 'mine_exclude_categories': [category], 'mine_width_min': 128, 'mine_width_max': 512, 'mine_height_min': 128, 'mine_height_max': 512, 'neg_exclude_categories': [category], 'max_rois_pos': 900, 'max_rois_neg': 550, 'num_trees': num_trees, } detect_config = { 'save_detection_images': True, 'percentage_top': 0.40, } #================================= # Train / Detect Initialization #================================= # Create detector detector = Random_Forest_Detector() # Gather Dataset dataset_path = utool.unixpath('~/code/IBEIS2014/') dataset = IBEIS_Data(dataset_path, **train_config) results_path = join(utool.unixpath('~/code/pyrf/results'), category) # pos_path = join(results_path, 'train-positives') # neg_path = join(results_path, 'train-negatives') # val_path = join(results_path, 'val') test_path = join(results_path, 'test') # test_pos_path = join(results_path, 'test-positives') # test_neg_path = join(results_path, 'test-negatives') detect_path = join(results_path, 'detect') trees_path = join(results_path, 'trees') # # Ensure result path for the category # # rmtreedir(results_path) # ensuredir(results_path) # for phase in range(1, boosting + 1): # print("*********************") # print("Phase: %s" % phase) # print("*********************") # # raw_input() # # ================================= # # Train Random Forest # #================================= # detector.train(dataset, category, pos_path, neg_path, val_path, # test_path, test_pos_path, test_neg_path, # trees_path, reshuffle=(phase == 1), **train_config) # if phase < boosting: # #================================= # # Detect using Random Forest # #================================= # # Load forest, so we don't have to reload every time # forest = detector.load(trees_path, category + '-', num_trees=(phase * num_trees)) # detector.set_detect_params(**detect_config) # # Ensure output detection paths # rmtreedir(detect_path) # ensuredir(detect_path) # # Calculate error on test set # direct = Directory(test_path, include_file_extensions=["jpg"]) # accuracy_list = [] # image_filepath_list = direct.files() # dst_filepath_list = [ join(detect_path, split(image_filepath)[1]) for image_filepath in image_filepath_list ] # predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True) # for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)): # image_path, image_filename = split(image_filepath) # image = dataset[image_filename] # accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category) # accuracy_list.append(accuracy) # progress = "%0.2f" % (float(index) / len(image_filepath_list)) # print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r') # sys.stdout.flush() # # image.show(prediction_list=predictions, category=category) # print(' ' * 100, end='\r') # print("TEST ERROR: %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list)))) # #================================= # # Eval and prep boosting train set # #================================= # detector.boosting(phase, forest, dataset, category, pos_path, neg_path, # test_pos_path, test_neg_path, detect_path) #################################### # New FAST #################################### detector = Random_Forest_Detector( scales='6 1.0 0.75 0.55 0.40 0.30 0.20' ) # Ensure output detection paths detect_path_temp = detect_path + "_1" rmtreedir(detect_path_temp) ensuredir(detect_path_temp) # Load forest, so we don't have to reload every time forest = detector.load(trees_path, category + '-', num_trees=25) detector.set_detect_params(**detect_config) # Calculate error on test set direct = Directory(test_path, include_file_extensions=["jpg"]) accuracy_list = [] true_pos_list = [] false_pos_list = [] false_neg_list = [] image_filepath_list = direct.files() dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ] predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True) for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)): image_path, image_filename = split(image_filepath) image = dataset[image_filename] accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category) accuracy_list.append(accuracy) true_pos_list.append(true_pos) false_pos_list.append(false_pos) false_neg_list.append(false_neg) progress = "%0.2f" % (float(index) / len(image_filepath_list)) print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r') sys.stdout.flush() # image.show(prediction_list=predictions, category=category) print(' ' * 100, end='\r') print("1 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list)))) print("1 TEST TRUE POS : %d" % (sum(true_pos_list))) print("1 TEST FALSE POS : %d" % (sum(false_pos_list))) print("1 TEST FALSE NEG : %d" % (sum(false_neg_list))) #################################### # New SLOW #################################### detector = Random_Forest_Detector( scales='11 1.5 1.25 1.0 0.8 0.64 0.51 0.41 0.33 0.26 0.21 0.17' ) # Ensure output detection paths detect_path_temp = detect_path + "_2" rmtreedir(detect_path_temp) ensuredir(detect_path_temp) # Load forest, so we don't have to reload every time forest = detector.load(trees_path, category + '-', num_trees=25) detector.set_detect_params(**detect_config) # Calculate error on test set direct = Directory(test_path, include_file_extensions=["jpg"]) accuracy_list = [] true_pos_list = [] false_pos_list = [] false_neg_list = [] image_filepath_list = direct.files() dst_filepath_list = [ join(detect_path_temp, split(image_filepath)[1]) for image_filepath in image_filepath_list ] predictions_list = detector.detect_many(forest, image_filepath_list, dst_filepath_list, use_openmp=True) for index, (predictions, image_filepath) in enumerate(zip(predictions_list, image_filepath_list)): image_path, image_filename = split(image_filepath) image = dataset[image_filename] accuracy, true_pos, false_pos, false_neg = image.accuracy(predictions, category) accuracy_list.append(accuracy) true_pos_list.append(true_pos) false_pos_list.append(false_pos) false_neg_list.append(false_neg) progress = "%0.2f" % (float(index) / len(image_filepath_list)) print("TEST %s %0.4f %s" % (image, accuracy, progress), end='\r') sys.stdout.flush() # image.show(prediction_list=predictions, category=category) print(' ' * 100, end='\r') print("2 TEST ERROR : %0.4f" % (1.0 - (float(sum(accuracy_list)) / len(accuracy_list)))) print("2 TEST TRUE POS : %d" % (sum(true_pos_list))) print("2 TEST FALSE POS : %d" % (sum(false_pos_list))) print("2 TEST FALSE NEG : %d" % (sum(false_neg_list)))
def test_pyrf(): r""" CommandLine: python run_tests.py --test-test_pyrf Example: >>> # ENABLE_DOCTEST >>> from run_tests import * # NOQA >>> result = test_pyrf() >>> print(result) """ #================================= # Initialization #================================= category = 'zebra_plains' #detect_config = { # 'save_detection_images': True, # 'percentage_top': 0.40, #} testdata_dir = ut.unixpath('~/code/pyrf/results') # assert ut.checkpath(testdata_dir) if ut.get_argflag('--vd'): print(ut.ls(testdata_dir)) # Create detector detector = Random_Forest_Detector() test_path = ut.grab_zipped_url(TEST_DATA_DETECT_URL, appname='utool') models_path = ut.grab_zipped_url(TEST_DATA_MODEL_URL, appname='utool') trees_path = join(models_path, category) detect_path = join(test_path, category, 'detect') ut.ensuredir(detect_path) ut.ensuredir(test_path) ut.ensuredir(trees_path) #================================= # Load Input Images #================================= # Get input images big_gpath_list = ut.list_images(test_path, fullpath=True, recursive=False) print(big_gpath_list) # Resize images to standard size if ut.get_argflag('--small'): big_gpath_list = big_gpath_list[0:8] #big_gpath_list = big_gpath_list[0:8] output_dir = join(test_path, 'resized') std_gpath_list = resize_imagelist_to_sqrtarea(big_gpath_list, sqrt_area=800, output_dir=output_dir, checkexists=True) dst_gpath_list = [join(detect_path, split(gpath)[1]) for gpath in std_gpath_list] #ut.view_directory(test_path) #ut.view_directory('.') print(std_gpath_list) num_images = len(std_gpath_list) #assert num_images == 16, 'the test has diverged!' print('Testing on %r images' % num_images) #================================= # Load Pretrained Forests #================================= # Load forest, so we don't have to reload every time trees_fpath_list = ut.ls(trees_path, '*.txt') #forest = detector.load(trees_path, category + '-') forest = detector.forest(trees_fpath_list) #detector.set_detect_params(**detect_config) results_list1 = [] #================================= # Detect using Random Forest #================================= with ut.Timer('[test_pyrf] for loop detector.detect') as t1: if not ut.get_argflag('--skip1'): results_list1 = detector.detect(forest, std_gpath_list, output_gpath_list=dst_gpath_list) #for ix, (img_fpath, dst_fpath) in enumerate(zip(std_gpath_list, dst_gpath_list)): # #img_fname = split(img_fpath)[1] # #dst_fpath = join(detect_path, img_fname) # #print(' * img_fpath = %r' % img_fpath) # #print(' * dst_fpath = %r' % dst_fpath) # with ut.Timer('[test_pyrf] detector.detect ix=%r' % (ix,)): # results = detector.detect(forest, img_fpath, dst_fpath) # results_list1.append(results) # print('num results = %r' % len(results)) #else: # print('...skipped') #with ut.Timer('[test_pyrf] detector.detect_many') as t2: # results_list2 = detector.detect_many(forest, std_gpath_list, # dst_gpath_list, use_openmp=True) detector.free_forest(forest) print('') print('+ --------------') print('| total time1: %r' % t1.ellapsed) #print('| total time2: %r' % t2.ellapsed) print('|') print('| num results1 = %r' % (list(map(len, results_list1)))) #print('| num results2 = %r' % (list(map(len, results_list2)))) #assert results_list2 == results_list1 return locals()