def check_importable(repo):
import utool as ut
# import utool as ut
found = False
tried = []
errors = []
for modname in repo.aliases:
tried.append(modname)
try:
ut.import_modname(modname)
except ImportError as ex: # NOQA
tried[-1] += " but got ImportError"
errors.append(ex)
pass
except AttributeError as ex: # NOQA
tried[-1] += " but got AttributeError"
errors.append(ex)
else:
found = True
errors.append(None)
tried[-1] += " and it worked"
break
msg = "tried %s" % (", ".join(tried))
return found, msg, errors
def find_expt_func(e):
import utool as ut
import ibeis.dev
for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS:
modname, funcname = tup[:2]
aliases = tup[2] if len(tup) == 3 else []
if e == funcname or e in aliases:
module = ut.import_modname(modname)
func = module.__dict__[funcname]
return func
# hack in --tf magic
func = ut.find_testfunc('ibeis', funcname)[0]
return func
def find_expt_func(e):
import utool as ut
import ibeis.dev
for tup in ibeis.dev.REGISTERED_DOCTEST_EXPERIMENTS:
modname, funcname = tup[:2]
aliases = tup[2] if len(tup) == 3 else []
if e == funcname or e in aliases:
module = ut.import_modname(modname)
func = module.__dict__[funcname]
return func
# hack in --tf magic
func = ut.find_testfunc('ibeis', funcname)[0]
return func
def check_importable(repo):
import utool as ut
# import utool as ut
found = False
tried = []
errors = []
for modname in repo.aliases:
tried.append(modname)
try:
ut.import_modname(modname)
except ImportError as ex: # NOQA
tried[-1] += ' but got ImportError'
errors.append(ex)
pass
except AttributeError as ex: # NOQA
tried[-1] += ' but got AttributeError'
errors.append(ex)
else:
found = True
errors.append(None)
tried[-1] += ' and it worked'
break
msg = 'tried %s' % (', '.join(tried))
return found, msg, errors
def print_module_info(modname):
print('Checking modname = %r' % (modname,))
# Handles special cases for certain modules
if modname.lower() == 'pillow':
from PIL import Image
import PIL
pil_path = PIL.__path__
infodict = module_stdinfo_dict(Image, versionattr='PILLOW_VERSION',
image_version=Image.VERSION,
pil_path=pil_path)
elif modname.lower() == 'pyqt4':
from PyQt4 import QtCore
infodict = module_stdinfo_dict(QtCore, 'PYQT_VERSION_STR')
elif modname.lower() == 'pyqt5':
from PyQt5 import QtCore
infodict = module_stdinfo_dict(QtCore, 'PYQT_VERSION_STR')
else:
# Handle normal modules
module = ut.import_modname(modname)
infodict = module_stdinfo_dict(module)
if any([infodict['__file__'].endswith(ext) for ext in ut.LIB_EXT_LIST]):
infodict['libdep'] = ut.get_dynlib_dependencies(infodict['__file__'])
return print(ut.repr4(infodict, strvals=True))
ibeis_rman.issue(
'git push --set-upstream origin {upstream_push}'.format(**locals()))
if GET_ARGFLAG('--test'):
failures = []
for repo_dpath in ibeis_rman.repo_dirs:
# ut.getp_
mod_dpaths = ut.get_submodules_from_dpath(repo_dpath,
recursive=False,
only_packages=True)
modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths)
print('Checking modules = %r' % (modname_list, ))
for modname in modname_list:
try:
ut.import_modname(modname)
print(modname + ' success')
except ImportError as ex:
failures += [modname]
print(modname + ' failure')
print('failures = %s' % (ut.repr3(failures), ))
# print('repo_dpath = %r' % (repo_dpath,))
# print('modules = %r' % (modules,))
# import ibeis
# print('found ibeis=%r' % (ibeis,))
if False:
try:
from six.moves import input
except ImportError:
def _exec_doctest_func(modname, funcname):
module = ut.import_modname(modname)
func = module.__dict__[funcname]
testsrc = ut.get_doctest_examples(func)[0][0]
exec(testsrc, globals(), locals())
def checkstdmod_version():
mod = ut.import_modname(modname)
return module_stdinfo_dict(mod)
# Dont use this if you are a developer. Use develop instead.
ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS)
ut.gg_command('python setup.py install'.format(**locals()))
if GET_ARGFLAG('--test'):
failures = []
for repo_dpath in IBEIS_REPO_DIRS:
# ut.getp_
mod_dpaths = ut.get_submodules_from_dpath(repo_dpath, recursive=False,
only_packages=True)
modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths)
print('Checking modules = %r' % (modname_list,))
for modname in modname_list:
try:
ut.import_modname(modname)
print(modname + ' success')
except ImportError as ex:
failures += [modname]
print(modname + ' failure')
print('failures = %s' % (ut.repr3(failures),))
# print('repo_dpath = %r' % (repo_dpath,))
# print('modules = %r' % (modules,))
# import ibeis
# print('found ibeis=%r' % (ibeis,))
if GET_ARGFLAG('--push'):
ut.gg_command('git push')
def _exec_doctest_func(modname, funcname):
module = ut.import_modname(modname)
func = module.__dict__[funcname]
testsrc = ut.get_doctest_examples(func)[0][0]
exec(testsrc, globals(), locals())
def run_ibeis():
r"""
CommandLine:
python -m ibeis
python -m ibeis find_installed_tomcat
python -m ibeis get_annot_groundtruth:1
"""
#ut.set_process_title('IBEIS_main')
#main_locals = ibeis.main()
#ibeis.main_loop(main_locals)
#ut.set_process_title('IBEIS_main')
cmdline_varags = ut.get_cmdline_varargs()
if len(cmdline_varags) > 0 and cmdline_varags[0] == 'rsync':
from ibeis.scripts import rsync_ibeisdb
rsync_ibeisdb.rsync_ibsdb_main()
sys.exit(0)
if ut.get_argflag('--devcmd'):
# Hack to let devs mess around when using an installer version
# TODO: add more hacks
#import utool.tests.run_tests
#utool.tests.run_tests.run_tests()
ut.embed()
# Run the tests of other modules
elif ut.get_argflag('--run-utool-tests'):
import utool.tests.run_tests
retcode = utool.tests.run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
elif ut.get_argflag('--run-vtool-tests'):
import vtool.tests.run_tests
retcode = vtool.tests.run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
elif ut.get_argflag(('--run-ibeis-tests', '--run-tests')):
from ibeis.tests import run_tests
retcode = run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
if ut.get_argflag('-e'):
"""
ibeis -e print -a default -t default
"""
# Run dev script if -e given
import ibeis.dev # NOQA
ibeis.dev.devmain()
print('... exiting')
sys.exit(0)
# Attempt to run a test using the funciton name alone
# with the --tf flag
import ibeis.tests.run_tests
import ibeis.tests.reset_testdbs
ignore_prefix = [
#'ibeis.tests',
'ibeis.control.__SQLITE3__',
'_autogen_explicit_controller']
ignore_suffix = ['_grave']
func_to_module_dict = {
'demo_bayesnet': 'ibeis.algo.hots.demobayes',
}
ut.main_function_tester('ibeis', ignore_prefix, ignore_suffix,
func_to_module_dict=func_to_module_dict)
#if ut.get_argflag('-e'):
# import ibeis
# expt_kw = ut.get_arg_dict(ut.get_func_kwargs(ibeis.run_experiment),
# prefix_list=['--', '-'])
# ibeis.run_experiment(**expt_kw)
# sys.exit(0)
doctest_modname = ut.get_argval(
('--doctest-module', '--tmod', '-tm', '--testmod'),
type_=str, default=None, help_='specify a module to doctest')
if doctest_modname is not None:
"""
Allow any doctest to be run the main ibeis script
python -m ibeis --tmod utool.util_str --test-align:0
python -m ibeis --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show
python -m ibeis --tf request_ibeis_query_L0:0 --show
./dist/ibeis/IBEISApp --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show # NOQA
./dist/ibeis/IBEISApp --tmod utool.util_str --test-align:0
./dist/IBEIS.app/Contents/MacOS/IBEISApp --tmod utool.util_str --test-align:0
./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-utool-tests
./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-vtool-tests
"""
print('[ibeis] Testing module')
mod_alias_list = {
'exptdraw': 'ibeis.expt.experiment_drawing'
}
doctest_modname = mod_alias_list.get(doctest_modname, doctest_modname)
module = ut.import_modname(doctest_modname)
(nPass, nTotal, failed_list, error_report_list) = ut.doctest_funcs(module=module)
retcode = 1 - (len(failed_list) == 0)
#print(module)
sys.exit(retcode)
import ibeis
main_locals = ibeis.main()
execstr = ibeis.main_loop(main_locals)
# <DEBUG CODE>
if 'back' in main_locals and CMD:
#from ibeis.all_imports import * # NOQA
back = main_locals['back']
front = getattr(back, 'front', None) # NOQA
#front = back.front
#ui = front.ui
ibs = main_locals['ibs'] # NOQA
exec(execstr)
def execute_commands(tpl_rman, wbia_rman):
import utool as ut
GET_ARGVAL = ut.get_argval
ut.init_catch_ctrl_c()
if 0:
print('Version Check Source:')
for repo in tpl_rman.repos:
print('python -c "import {0}; print({0}.__file__)"'.format(
repo.modname))
print('python -c "import {0}; print({0}.__version__)"'.format(
repo.modname))
# -----------
# Execute Commands on Core Repos
# -----------
CODE_DIR, pythoncmd, WIN32, PY2, PY3 = get_sysinfo()
print('wbia_rman = %r' % (wbia_rman, ))
wildme_ssh_flags = GET_ARGFLAG('--move-wildme') or GET_ARGFLAG(
'--move-wildme-ssh')
wildme_https_flags = GET_ARGFLAG('--move-wildme-https') or GET_ARGFLAG(
'--move-wildme-http')
if wildme_ssh_flags or wildme_https_flags:
fmt = 'ssh' if wildme_ssh_flags else 'https'
move_wildme(wbia_rman, fmt)
# Commands on global git repos
if GET_ARGFLAG('--status'):
wbia_rman.issue('git status')
sys.exit(0)
wbia_rman.ensure()
if GET_ARGFLAG('--dump') or GET_ARGFLAG('--dump-scripts'):
dpath = '_super_scripts/' + 'scripts' + get_plat_specifier()
ut.ensuredir(dpath)
dumps = [
(tpl_rman, 'cv2', 'build'),
(tpl_rman, 'cv2', 'install'),
(wbia_rman, 'flann', 'build'),
(wbia_rman, 'flann', 'install'),
(wbia_rman, 'hesaff', 'build'),
(tpl_rman, 'PyQt', 'system_to_venv'),
(tpl_rman, 'libgpuarray', 'build'),
]
for rman, mod, sname in dumps:
from os.path import join
# if mod not in rman:
# print('mod=%r not available in rman=%r' % (mod, rman))
# continue
script = rman[mod].get_script(sname).text
suffix = get_plat_specifier()
sh_fpath = join(dpath, mod + '_' + sname + suffix + '.sh')
ut.write_to(sh_fpath, script)
if GET_ARGFLAG('--requirements'):
ut.cmd('pip install -r requirements.txt')
# HACKED IN SCRIPTS WHILE IM STILL FIGURING OUT TPL DEPS
if GET_ARGFLAG('--opencv'):
# There is now a pypi for opencv! Yay
# ut.cmd('pip install opencv-python')
# Bummer, but we need opencv source for pyhessaff
# we should just make a wheel for pyhessaff
cv_repo = tpl_rman['cv2']
cv_repo.clone()
script = cv_repo.get_script('build')
script.exec_()
cv_repo = tpl_rman['cv2']
script = cv_repo.get_script('install')
script.exec_()
if GET_ARGFLAG('--flann'):
script = wbia_rman['flann'].get_script('build')
script.exec_()
script = wbia_rman['flann'].get_script('install')
script.exec_()
if GET_ARGFLAG('--pyqt'):
script = tpl_rman['PyQt'].get_script('system_to_venv')
script.exec_()
if GET_ARGFLAG('--hesaff'):
script = wbia_rman['hesaff'].get_script('build')
script.exec_()
if GET_ARGFLAG('--pydarknet'):
script = wbia_rman['pydarknet'].get_script('build')
script.exec_()
if GET_ARGFLAG('--pyrf'):
script = wbia_rman['pyrf'].get_script('build')
script.exec_()
if GET_ARGFLAG('--torch'):
# Theano and lasange code should be moved to pytorch
tpl_rman['pytorch'].clone(recursive=True)
tpl_rman['pytorch'].issue('git submodule update --init')
tpl_rman['pytorch'].issue('python setup install')
tpl_rman['pytorch'].issue('pip install torchvision')
# tpl_rman['pytorch'].issue('NO_CUDNN=TRUE && python setup install')
# tpl_rman['pytorch'].issue('pip install -e .')
if GET_ARGFLAG('--libgpuarray') or GET_ARGFLAG('--dcnn'):
tpl_rman['libgpuarray'].clone()
script = tpl_rman['libgpuarray'].get_script('build')
script.exec_()
if GET_ARGFLAG('--dcnn'):
tpl_rman['theano'].clone()
# tpl_rman['pylearn2'].clone()
tpl_rman['lasagne'].clone()
tpl_rman['theano'].issue('pip install -e .')
# tpl_rman['pylearn2'].issue('pip install -e .')
tpl_rman['lasagne'].issue('pip install -e .')
# tpl_rman['pylearn2'].python_develop()
# tpl_rman['theano'].python_develop()
# tpl_rman['lasagne'].python_develop()
# _===
if GET_ARGFLAG('--fix') or GET_ARGFLAG('--check'):
missing_dynlib = tpl_rman.check_cpp_build()
missing_dynlib += wbia_rman.check_cpp_build()
missing_install = tpl_rman.check_installed()
missing_install += wbia_rman.check_installed()
problems = []
problems += wbia_rman.check_importable()
problems += tpl_rman.check_importable()
if GET_ARGFLAG('--fix'):
print('Trying to fix problems')
for repo in missing_dynlib:
repo.custom_build()
for repo, recommended_fix in problems:
print('Trying to fix repo = %r' % (repo, ))
print(' * recommended_fix = %r' % (recommended_fix, ))
if recommended_fix == 'rebuild':
repo.custom_build()
print(
'Can currently only fix one module at a time. Please re-run'
)
sys.exit(1)
else:
print('Not sure how to fix %r' % (repo, ))
if GET_ARGFLAG('--pull'):
wbia_rman.issue('git pull')
if GET_ARGFLAG('--build'):
# Build tpl repos
# tpl_rman.custom_build()
# wbia_rman.custom_build()
# Build only IBEIS repos with setup.py
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py build'.format(pythoncmd=pythoncmd))
# Like install, but better if you are developing
if GET_ARGFLAG('--develop'):
_rman = wbia_rman.only_with_pysetup()
# # _rman.issue('{pythoncmd} setup.py develop'.format(pythoncmd=pythoncmd),
# # sudo=not ut.in_virtual_env())
_rman.issue(
'{pythoncmd} -m pip install -e .'.format(pythoncmd=pythoncmd),
sudo=not ut.in_virtual_env(),
)
if GET_ARGFLAG('--clean'):
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py clean'.format(pythoncmd=pythoncmd))
if GET_ARGFLAG('--install'):
print(
'WARNING: Dont use install if you are a developer. Use develop instead.'
)
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py install'.format(pythoncmd=pythoncmd))
if GET_ARGFLAG('--push'):
wbia_rman.issue('git push')
if GET_ARGFLAG('--branch'):
wbia_rman.issue('git branch')
sys.exit(0)
if GET_ARGFLAG('--tag-status'):
wbia_rman.issue('git tag')
# Tag everything
tag_name = GET_ARGVAL('--newtag', type_=str, default=None)
if tag_name is not None:
wbia_rman.issue(
'git tag -a "{tag_name}" -m "super_setup autotag {tag_name}"'.
format(**locals()))
wbia_rman.issue('git push --tags')
if GET_ARGFLAG('--bext'):
wbia_rman.issue('{pythoncmd} setup.py build_ext --inplace'.format(
pythoncmd=pythoncmd))
commit_msg = GET_ARGVAL('--commit', type_=str, default=None)
if commit_msg is not None:
wbia_rman.issue('git commit -am "{commit_msg}"'.format(**locals()))
# Change Branch
branch_name = GET_ARGVAL('--checkout', type_=str, default=None)
if branch_name is not None:
try:
wbia_rman.issue('git checkout "{branch_name}"'.format(**locals()))
except Exception:
print('ERROR: Could not checkout branch: %r' % (branch_name, ))
# Creates new branches
newbranch_name = GET_ARGVAL('--newbranch', type_=str, default=None)
if newbranch_name is not None:
# rman.issue('git stash"'.format(**locals()))
wbia_rman.issue(
'git checkout -b "{newbranch_name}"'.format(**locals()))
wbia_rman.issue(
'git push --set-upstream origin {newbranch_name}'.format(
**locals()))
# rman.issue('git stash pop"'.format(**locals()))
# Creates new branches
newlocalbranch_name = GET_ARGVAL('--newlocalbranch',
type_=str,
default=None)
if newlocalbranch_name is not None:
# rman.issue('git stash"'.format(**locals()))
wbia_rman.issue(
'git checkout -b "{newlocalbranch_name}"'.format(**locals()))
# rman.issue('git push --set-upstream origin {newlocalbranch_name}'.format(**locals()))
# rman.issue('git stash pop"'.format(**locals()))
# Creates new branches
mergebranch_name = GET_ARGVAL('--merge', type_=str, default=None)
if mergebranch_name is not None:
wbia_rman.issue('git merge "{mergebranch_name}"'.format(**locals()))
# Change ownership
if GET_ARGFLAG('--serverchmod'):
wbia_rman.issue('chmod -R 755 *')
if GET_ARGFLAG('--chown'):
# Fixes problems where repos are checked out as root
username = os.environ.get('USERNAME', ut.get_argval('--username'))
if username is None:
username = os.environ.get('USER', None)
if username is None:
raise AssertionError(
'cannot find username in commandline or environment vars')
usergroup = username
wbia_rman.issue('chown -R {username}:{usergroup} *'.format(**locals()),
sudo=True)
upstream_branch = GET_ARGVAL('--set-upstream', type_=str, default=None)
if upstream_branch is not None:
# git 2.0
wbia_rman.issue(
'git branch --set-upstream-to=origin/{upstream_branch} {upstream_branch}'
.format(**locals()))
upstream_push = GET_ARGVAL('--upstream-push', type_=str, default=None)
if upstream_push is not None:
wbia_rman.issue(
'git push --set-upstream origin {upstream_push}'.format(
**locals()))
if GET_ARGFLAG('--test'):
failures = []
for repo_dpath in wbia_rman.repo_dirs:
# ut.getp_
mod_dpaths = ut.get_submodules_from_dpath(repo_dpath,
recursive=False,
only_packages=True)
modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths)
print('Checking modules = %r' % (modname_list, ))
for modname in modname_list:
try:
ut.import_modname(modname)
print(modname + ' success')
except ImportError:
failures += [modname]
print(modname + ' failure')
print('failures = %s' % (ut.repr3(failures), ))
if False:
try:
from six.moves import input
except ImportError:
input = raw_input # NOQA
# General global git command
gg_cmd = GET_ARGVAL('--gg', None) # global command
if gg_cmd is not None:
ans = ('yes' if GET_ARGFLAG('-y') else input(
'Are you sure you want to run: %r on all directories? ' %
(gg_cmd, )))
if ans == 'yes':
wbia_rman.issue(gg_cmd)
def run_ibeis():
r"""
CommandLine:
python -m ibeis
python -m ibeis find_installed_tomcat
python -m ibeis get_annot_groundtruth:1
"""
#ut.set_process_title('IBEIS_main')
#main_locals = ibeis.main()
#ibeis.main_loop(main_locals)
#ut.set_process_title('IBEIS_main')
cmdline_varags = ut.get_cmdline_varargs()
if len(cmdline_varags) > 0 and cmdline_varags[0] == 'rsync':
from ibeis.scripts import rsync_ibeisdb
rsync_ibeisdb.rsync_ibsdb_main()
sys.exit(0)
if ut.get_argflag('--devcmd'):
# Hack to let devs mess around when using an installer version
# TODO: add more hacks
#import utool.tests.run_tests
#utool.tests.run_tests.run_tests()
ut.embed()
# Run the tests of other modules
elif ut.get_argflag('--run-utool-tests'):
import utool.tests.run_tests
retcode = utool.tests.run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
elif ut.get_argflag('--run-vtool-tests'):
import vtool.tests.run_tests
retcode = vtool.tests.run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
elif ut.get_argflag(('--run-ibeis-tests', '--run-tests')):
from ibeis.tests import run_tests
retcode = run_tests.run_tests()
print('... exiting')
sys.exit(retcode)
if ut.get_argflag('-e'):
"""
ibeis -e print -a default -t default
"""
# Run dev script if -e given
import ibeis.dev # NOQA
ibeis.dev.devmain()
print('... exiting')
sys.exit(0)
# Attempt to run a test using the funciton name alone
# with the --tf flag
import ibeis.tests.run_tests
import ibeis.tests.reset_testdbs
import ibeis.scripts.thesis
ignore_prefix = [
#'ibeis.tests',
'ibeis.control.__SQLITE3__',
'_autogen_explicit_controller'
]
ignore_suffix = ['_grave']
func_to_module_dict = {
'demo_bayesnet': 'ibeis.unstable.demobayes',
}
ut.main_function_tester('ibeis',
ignore_prefix,
ignore_suffix,
func_to_module_dict=func_to_module_dict)
#if ut.get_argflag('-e'):
# import ibeis
# expt_kw = ut.get_arg_dict(ut.get_func_kwargs(ibeis.run_experiment),
# prefix_list=['--', '-'])
# ibeis.run_experiment(**expt_kw)
# sys.exit(0)
doctest_modname = ut.get_argval(
('--doctest-module', '--tmod', '-tm', '--testmod'),
type_=str,
default=None,
help_='specify a module to doctest')
if doctest_modname is not None:
"""
Allow any doctest to be run the main ibeis script
python -m ibeis --tmod utool.util_str --test-align:0
python -m ibeis --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show
python -m ibeis --tf request_ibeis_query_L0:0 --show
./dist/ibeis/IBEISApp --tmod ibeis.algo.hots.pipeline --test-request_ibeis_query_L0:0 --show # NOQA
./dist/ibeis/IBEISApp --tmod utool.util_str --test-align:0
./dist/IBEIS.app/Contents/MacOS/IBEISApp --tmod utool.util_str --test-align:0
./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-utool-tests
./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-vtool-tests
"""
print('[ibeis] Testing module')
mod_alias_list = {'exptdraw': 'ibeis.expt.experiment_drawing'}
doctest_modname = mod_alias_list.get(doctest_modname, doctest_modname)
module = ut.import_modname(doctest_modname)
(nPass, nTotal, failed_list,
error_report_list) = ut.doctest_funcs(module=module)
retcode = 1 - (len(failed_list) == 0)
#print(module)
sys.exit(retcode)
import ibeis
main_locals = ibeis.main()
execstr = ibeis.main_loop(main_locals)
# <DEBUG CODE>
if 'back' in main_locals and CMD:
back = main_locals['back']
front = getattr(back, 'front', None) # NOQA
#front = back.front
#ui = front.ui
ibs = main_locals['ibs'] # NOQA
exec(execstr)
