def tryimport(modname, pipiname=None, ensure=False):
"""
CommandLine:
python -m utool.util_import --test-tryimport
Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_tests import * # NOQA
>>> import utool as ut
>>> modname = 'pyfiglet'
>>> pipiname = 'git+https://github.com/pwaller/pyfiglet'
>>> pyfiglet = ut.tryimport(modname, pipiname)
>>> assert pyfiglet is None or isinstance(pyfiglet, types.ModuleType), 'unknown error'
Example2:
>>> # UNSTABLE_DOCTEST
>>> # disabled because not everyone has access to being a super user
>>> from utool.util_tests import * # NOQA
>>> import utool as ut
>>> modname = 'lru'
>>> pipiname = 'git+https://github.com/amitdev/lru-dict'
>>> lru = ut.tryimport(modname, pipiname, ensure=True)
>>> assert isinstance(lru, types.ModuleType), 'did not ensure lru'
"""
if pipiname is None:
pipiname = modname
try:
if util_inject.PRINT_INJECT_ORDER:
if modname not in sys.modules:
util_inject.noinject(modname, N=2, via='ut.tryimport')
module = __import__(modname)
return module
except ImportError as ex:
import utool as ut
base_pipcmd = 'pip install %s' % pipiname
sudo = not ut.WIN32 and not ut.in_virtual_env()
if sudo:
pipcmd = 'sudo ' + base_pipcmd
else:
pipcmd = base_pipcmd
msg = 'unable to find module %s. Please install: %s' % ((modname), (pipcmd))
print(msg)
ut.printex(ex, msg, iswarning=True)
if ensure:
raise AssertionError('Ensure is dangerous behavior and is is no longer supported.')
#raise NotImplementedError('not ensuring')
ut.cmd(base_pipcmd, sudo=sudo)
module = tryimport(modname, pipiname, ensure=False)
if module is None:
raise AssertionError('Cannot ensure modname=%r please install using %r' % (modname, pipcmd))
return module
return None
def get_local_dist_packages_dir():
"""
Attempts to work around virtualenvs and find the system dist_pacakges.
Essentially this is implmenented as a lookuptable
"""
import utool as ut
if not ut.in_virtual_env():
# Non venv case
return get_site_packages_dir()
else:
candidates = []
if ut.LINUX:
candidates += [
'/usr/local/lib/python2.7/dist-packages',
]
else:
raise NotImplementedError()
for path in candidates:
if ut.checkpath(path):
return path
def get_local_dist_packages_dir():
"""
Attempts to work around virtualenvs and find the system dist_pacakges.
Essentially this is implmenented as a lookuptable
"""
import utool as ut
if not ut.in_virtual_env():
# Non venv case
return get_site_packages_dir()
else:
candidates = []
if ut.LINUX:
candidates += [
'/usr/local/lib/python2.7/dist-packages',
]
else:
raise NotImplementedError()
for path in candidates:
if ut.checkpath(path):
return path
def define_custom_scripts(tpl_rman):
if ut.in_virtual_env():
fmtdict = {
'sys_dist_packages': ut.get_global_dist_packages_dir(),
'venv_site_packages': ut.get_site_packages_dir(),
}
# Allows us to use a system qt install in a virtual environment.
tpl_rman['PyQt4'].add_script(
'system_to_venv',
ut.codeblock(r"""
# STARTBLOCK bash
ln -s {sys_dist_packages}/PyQt4/ {venv_site_packages}/PyQt4
ln -s {sys_dist_packages}/sip*.so {venv_site_packages}/
ln -s {sys_dist_packages}/sip*.py {venv_site_packages}/
# ENDBLOCK bash
""")).format(**fmtdict)
# TODO: add custom build alternative
pass
else:
pass
ibeis_rman['pyflann'].add_script(
'install',
ut.codeblock(r'''
# STARTBLOCK bash
# The pyflann source lives here
cd {repo_dir}/src/python
# But the setup script is generated during build
python {repo_dir}/build/src/python/setup.py develop
# ENDBLOCK bash
''').format(repo_dir=ibeis_rman['pyflann'].dpath))
# TODO: allow system installation as well
tpl_rman['cv2'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
cd $CODE_DIR
git clone https://github.com/Itseez/opencv.git
cd opencv
# Get Extras
git clone https://github.com/Itseez/opencv_contrib.git
mkdir -p build27
cd build27
if [[ "$VIRTUAL_ENV" == "" ]]; then
export LOCAL_PREFIX=/usr/local
export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/dist-packages
else
export LOCAL_PREFIX=$VIRTUAL_ENV/local
export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/site-packages
fi
echo "LOCAL_PREFIX = $LOCAL_PREFIX"
echo "PYTHON2_PACKAGES_PATH = $PYTHON2_PACKAGES_PATH"
# use dist packages on ubuntu. may need to change for other platforms
cmake -G "Unix Makefiles" \
-D WITH_OPENMP=ON \
-D CMAKE_BUILD_TYPE=RELEASE \
-D PYTHON2_PACKAGES_PATH=$PYTHON2_PACKAGES_PATH \
-D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \
-D OPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules \
..
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
# ENDBLOCK
"""))
tpl_rman['cv2'].add_script(
'install',
ut.codeblock(r"""
# STARTBLOCK bash
cd $CODE_DIR
sudo make install
# Hack because cv2 does not want to be installed for some reason
cp lib/cv2.so $PYTHON2_PACKAGES_PATH
# Test makesure things working
python -c "import numpy; print(numpy.__file__)"
python -c "import numpy; print(numpy.__version__)"
python -c "import cv2; print(cv2.__version__)"
python -c "import cv2; print(cv2.__file__)"
#python -c "import vtool"
# Check if we have contrib modules
python -c "import cv2; print(cv2.xfeatures2d)"
# ENDBLOCK
"""))
if GET_ARGFLAG('--pull'):
ibeis_rman.issue('git pull')
if GET_ARGFLAG('--build'):
# Build tpl repos
tpl_rman.custom_build()
ibeis_rman.custom_build()
# Build only IBEIS repos with setup.py
_rman = ibeis_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py build'.format(**locals()))
# Like install, but better if you are developing
if GET_ARGFLAG('--develop'):
_rman = ibeis_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py develop'.format(**locals()),
sudo=not ut.in_virtual_env())
if GET_ARGFLAG('--clean'):
_rman = ibeis_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py clean'.format(**locals()))
if GET_ARGFLAG('--install'):
print(
'WARNING: Dont use install if you are a developer. Use develop instead.'
)
_rman = ibeis_rman.only_with_pysetup()
_rman.issue('python setup.py install'.format(**locals()))
if GET_ARGFLAG('--push'):
ibeis_rman.issue('git push')
if GET_ARGFLAG('--bext'):
ut.gg_command('{pythoncmd} setup.py build_ext --inplace'.format(**locals()))
if GET_ARGFLAG('--build'):
# Build tpl repos
for repo in TPL_REPO_DIRS:
ut.util_git.std_build_command(repo) # Executes {plat}_build.{ext}
# Build only IBEIS repos with setup.py
ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS)
ut.gg_command('{pythoncmd} setup.py build'.format(**locals()))
if GET_ARGFLAG('--develop'):
# Like install, but better if you are developing
ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS)
ut.gg_command('{pythoncmd} setup.py develop'.format(**locals()),
sudo=not ut.in_virtual_env())
if GET_ARGFLAG('--install'):
# Dont use this if you are a developer. Use develop instead.
ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS)
ut.gg_command('python setup.py install'.format(**locals()))
if GET_ARGFLAG('--test'):
failures = []
for repo_dpath in IBEIS_REPO_DIRS:
# ut.getp_
mod_dpaths = ut.get_submodules_from_dpath(repo_dpath, recursive=False,
only_packages=True)
modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths)
print('Checking modules = %r' % (modname_list,))
def python_develop(repo):
import utool as ut
repo.issue("{pythoncmd} setup.py develop".format(pythoncmd=repo.pythoncmd), sudo=not ut.in_virtual_env())
def define_custom_scripts(tpl_rman, wbia_rman, PY2, PY3):
"""
export THEANO_FLAGS="device=cpu,print_active_device=True,enable_initial_driver_test=True"
set THEANO_FLAGS=device=cpu,print_active_device=True,enable_initial_driver_test=True,print_test_value=True
python -c "import pydot; print(pydot.__file__)"
python -c "import pydot; print(pydot.__version__)"
python -c "import pydot; print(pydot.find_graphviz())"
DEVICE="cuda" python -c "import pygpu;pygpu.test()"
python -c "import theano; print(theano.__file__)"
# python -c "import pylearn2; print(pylearn2.__file__)"
python -c "import lasagne; print(lasagne.__file__)"
python -c "import wbia_cnn; print(wbia_cnn.__file__)"
python -c "import detecttools; print(detecttools.__file__)"
# http://stackoverflow.com/questions/18042919/how-to-install-pyqt5-on-a-new-virtualenv-and-work-on-an-idle
pip install vext.pyqt5
sudo apt-get install pyqt5-dev
sudo apt-get install python3-pyqt5
python
python -c "import sip; print('[test] Python can import sip')"
python -c "import sip; print('sip.__file__=%r' % (sip.__file__,))"
python -c "import sip; print('sip.SIP_VERSION=%r' % (sip.SIP_VERSION,))"
python -c "import sip; print('sip.SIP_VERSION_STR=%r' % (sip.SIP_VERSION_STR,))"
ln -s /usr/lib/python3/dist-packages/PyQt5/ /home/joncrall/venv3/lib/python3.4/site-packages/PyQt5
ln -s /usr/lib/python3/dist-packages/sip*.so /home/joncrall/venv3/lib/python3.4/site-packages/
ln -s /usr/lib/python3/dist-packages/sip*.py /home/joncrall/venv3/lib/python3.4/site-packages/
"""
import utool as ut
major = str(sys.version_info.major)
minor = str(sys.version_info.minor)
majorminor = [major, minor]
pyoff = '2' if sys.version_info.major == 3 else '3'
pyon = majorminor[0]
plat_spec = get_plat_specifier()
# build_dname = 'build' + ''.join(majorminor)
build_dname = 'cmake_builds/build' + plat_spec
script_fmtdict = {
'pyexe': sys.executable,
'pyversion': 'python' + '.'.join(majorminor),
'pypkg_var': 'PYTHON' + pyon + '_PACKAGES_PATH',
'build_dname': build_dname,
'pyoff': pyoff,
'pyon': pyon,
'cv_pyon_var': 'BUILD_opencv_python' + pyon,
'cv_pyoff_var': 'BUILD_opencv_python' + pyoff,
'plat_spec': plat_spec,
'source_dpath': '../..',
'libext': ut.get_lib_ext(),
}
if os.environ.get('VIRTUAL_ENV', '') == '':
if sys.platform.startswith('darwin'):
local_prefix = '/opt/local'
else:
local_prefix = '/usr/local'
else:
local_prefix = os.environ['VIRTUAL_ENV']
opencv_dir = os.path.join(local_prefix, '/share/OpenCV')
if not os.path.exists(opencv_dir):
if not ut.get_argflag('--opencv'):
opencv_dir = ''
print(
'OpenCV is not installed in the expected location: {}'.format(
opencv_dir))
print(
'Running this script with --opencv will build and install it there'
)
# define bash variables for different combinations of python distros and
# virtual environments
python_bash_setup = ut.codeblock(r"""
# STARTBLOCK bash
if [[ "$VIRTUAL_ENV" == "" ]]; then
# The case where we are installying system-wide
# It is recommended that a virtual enviornment is used instead
export PYTHON_EXECUTABLE=$(which {pyversion})
if [[ '$OSTYPE' == 'darwin'* ]]; then
# Mac system info
export LOCAL_PREFIX=/opt/local
export {pypkg_var}=$($PYTHON_EXECUTABLE -c "import site; print(site.getsitepackages()[0])")
export PYTHON_PACKAGES_PATH=${pypkg_var}
export _SUDO="sudo"
else
# Linux system info
export LOCAL_PREFIX=/usr/local
export {pypkg_var}=$LOCAL_PREFIX/lib/{pyversion}/dist-packages
export PYTHON_PACKAGES_PATH=${pypkg_var}
export _SUDO="sudo"
fi
# No windows support here
else
# The prefered case where we are in a virtual environment
export PYTHON_EXECUTABLE=$(which python)
# export LOCAL_PREFIX=$VIRTUAL_ENV/local
export LOCAL_PREFIX=$VIRTUAL_ENV
export {pypkg_var}=$LOCAL_PREFIX/lib/{pyversion}/site-packages
export PYTHON_PACKAGES_PATH=${pypkg_var}
export _SUDO=""
fi
echo "LOCAL_PREFIX = $LOCAL_PREFIX"
echo "{pypkg_var} = ${pypkg_var}"
# ENDBLOCK bash
""").format(**script_fmtdict)
script_fmtdict['python_bash_setup'] = python_bash_setup
# ===================
# PYFLANN SETUP SCRIPTS
# ===================
wbia_rman['pyflann'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
cd {repo_dir}
mkdir -p {build_dname}
cd {build_dname}
cmake -G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE="Release" \
-DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
-DBUILD_EXAMPLES=Off \
-DBUILD_TESTS=Off \
-DBUILD_PYTHON_BINDINGS=On \
-DBUILD_MATLAB_BINDINGS=Off \
-DBUILD_CUDA_LIB=Off\
-DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX\
{source_dpath}
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
# ENDBLOCK bash
""").format(repo_dir=wbia_rman['pyflann'].dpath, **script_fmtdict),
)
wbia_rman['pyflann'].add_script(
'install',
ut.codeblock(r"""
# STARTBLOCK bash
# The pyflann source lives here
cd {repo_dir}/src/python
# Need to run build to move the libs to the build directory
python setup.py build
# Use pip to editable install
pip install -e {repo_dir}/src/python
# Old way of doing it
# But the setup script is generated during build
# python {repo_dir}/build/src/python/setup.py develop
python -c "from vtool_ibeis._pyflann_backend import pyflann as pyflann; print(pyflann.__file__)" --verb-flann
python -c "from vtool_ibeis._pyflann_backend import pyflann as pyflann; print(pyflann)" --verb-flann
# ENDBLOCK bash
""").format(repo_dir=wbia_rman['pyflann'].dpath),
)
# ===================
# HESAFF
# ===================
wbia_rman['hesaff'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
cd $CODE_DIR/hesaff
mkdir -p {build_dname}
cd {build_dname}
# only specify an explicit opencv directory if we know one exists
if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then
OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV"
else
OPENCV_ARGS=""
fi
echo 'Configuring with cmake'
if [[ '$OSTYPE' == 'darwin'* ]]; then
cmake -G "Unix Makefiles" \
-DCMAKE_OSX_ARCHITECTURES=x86_64 \
-DCMAKE_C_COMPILER=clang2 \
-DCMAKE_CXX_COMPILER=clang2++ \
-DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \
$OPENCV_ARGS \
{source_dpath}
else
cmake -G "Unix Makefiles" \
-DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \
$OPENCV_ARGS \
{source_dpath}
fi
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
export MAKE_EXITCODE=$?
echo "MAKE_EXITCODE=$MAKE_EXITCODE"
# Move the compiled library into the source folder
if [[ $MAKE_EXITCODE == 0 ]]; then
#make VERBOSE=1
cp -v libhesaff{libext} {source_dpath}/pyhesaff/libhesaff{plat_spec}{libext}
fi
# ENDBLOCK
""").format(**script_fmtdict),
)
# ===================
# PYDARKNET
# ===================
wbia_rman['pydarknet'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
cd $CODE_DIR/pydarknet
mkdir -p {build_dname}
cd {build_dname}
if [[ "$(which nvcc)" == "" ]]; then
export CMAKE_CUDA=Off
else
export CMAKE_CUDA=On
fi
# only specify an explicit opencv directory if we know one exists
if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then
OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV"
else
OPENCV_ARGS=""
fi
echo 'Configuring with cmake'
if [[ '$OSTYPE' == 'darwin'* ]]; then
export CONFIG="-DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_C_COMPILER=clang2 -DCMAKE_CXX_COMPILER=clang2++ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS"
else
export CONFIG="-DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS"
fi
export CONFIG="$CONFIG -DCUDA=$CMAKE_CUDA"
echo "CONFIG = $CONFIG"
cmake $CONFIG -G 'Unix Makefiles' {source_dpath}
#################################
echo 'Building with make'
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS -w
#################################
export MAKE_EXITCODE=$?
echo "MAKE_EXITCODE=$MAKE_EXITCODE"
# Move the compiled library into the source folder
if [[ $MAKE_EXITCODE == 0 ]]; then
echo 'Moving the shared library'
# cp -v lib* ../pydarknet
cp -v lib*{libext} {source_dpath}/pydarknet
# cp -v libdarknet{libext} {source_dpath}/pydarknet/libdarknet{plat_spec}{libext}
fi
# ENDBLOCK
""").format(**script_fmtdict),
)
# ===================
# PYRF
# ===================
wbia_rman['pyrf'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
cd $CODE_DIR/pyrf
mkdir -p {build_dname}
cd {build_dname}
# only specify an explicit opencv directory if we know one exists
if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then
OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV"
else
OPENCV_ARGS=""
fi
echo 'Configuring with cmake'
if [[ '$OSTYPE' == 'darwin'* ]]; then
export CONFIG="-DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_C_COMPILER=clang2 -DCMAKE_CXX_COMPILER=clang2++ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS"
else
export CONFIG="-DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS"
fi
echo "CONFIG = $CONFIG"
cmake $CONFIG -G 'Unix Makefiles' {source_dpath}
#################################
echo 'Building with make'
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS -w
#################################
export MAKE_EXITCODE=$?
echo "MAKE_EXITCODE=$MAKE_EXITCODE"
# Move the compiled library into the source folder
if [[ $MAKE_EXITCODE == 0 ]]; then
echo 'Moving the shared library'
# cp -v lib* ../pyrf
cp -v lib*{libext} {source_dpath}/pyrf
# cp -v libpyrf{libext} {source_dpath}/pyrf/libpyrf{plat_spec}{libext}
fi
# ENDBLOCK
""").format(**script_fmtdict),
)
# ===================
# OPENCV SETUP SCRIPTS
# ===================
"""
./super_setup.py --dump-scripts
"""
tpl_rman['cv2'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
# Checkout opencv core
cd $CODE_DIR
# export REPO_DIR=$CODE_DIR/opencv
export REPO_DIR={repo_dpath}
# git clone https://github.com/Itseez/opencv.git
cd $REPO_DIR
# Checkout opencv extras
git clone https://github.com/Itseez/opencv_contrib.git
# cd opencv_contrib
# git pull
# cd ..
# git pull
mkdir -p $REPO_DIR/{build_dname}
cd $REPO_DIR/{build_dname}
cmake -G "Unix Makefiles" \
-D WITH_OPENMP=ON \
-D CMAKE_BUILD_TYPE=RELEASE \
-D {cv_pyoff_var}=Off \
-D {cv_pyon_var}=On \
-D PYTHON_DEFAULT_EXECUTABLE="{pyexe}" \
-D {pypkg_var}=${pypkg_var} \
-D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \
-D OPENCV_EXTRA_MODULES_PATH=$REPO_DIR/opencv_contrib/modules \
-D WITH_CUDA=Off \
-D BUILD_opencv_dnn=Off \
-D BUILD_opencv_dnn_modern=Off \
-D WITH_VTK=Off \
-D WITH_CUDA=Off \
-D WITH_MATLAB=Off \
$REPO_DIR
# -D WITH_OPENCL=Off \
# -D BUILD_opencv_face=Off \
# -D BUILD_opencv_objdetect=Off \
# -D BUILD_opencv_video=Off \
# -D BUILD_opencv_videoio=Off \
# -D BUILD_opencv_videostab=Off \
# -D BUILD_opencv_ximgproc=Off \
# -D BUILD_opencv_xobjdetect=Off \
# -D BUILD_opencv_xphoto=Off \
# -D BUILD_opencv_datasets=Off \
# -D CXX_FLAGS="-std=c++11" \ %TODO
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
# ENDBLOCK
""").format(repo_dpath=ut.unexpanduser(tpl_rman['cv2'].dpath),
**script_fmtdict),
)
tpl_rman['cv2'].add_script(
'install',
ut.codeblock(r"""
# STARTBLOCK bash
{python_bash_setup}
cd $CODE_DIR/opencv/{build_dname}
$_SUDO make install
# Hack because cv2 does not want to be installed for some reason
# cp lib/cv2.so $PYTHON_PACKAGES_PATH
# Seems to work now that local is removed from prefix
# cp -v lib/cv2.so $PYTHON_PACKAGES_PATH
# Test makesure things working
python -c "import numpy; print(numpy.__file__)"
python -c "import numpy; print(numpy.__version__)"
python -c "import cv2; print(cv2.__version__)"
python -c "import cv2; print(cv2.__file__)"
#python -c "import vtool_ibeis"
# Check if we have contrib modules
python -c "import cv2; print(cv2.xfeatures2d)"
# ENDBLOCK
""").format(**script_fmtdict),
)
# if GET_ARGFLAG('--libgpuarray'):
tpl_rman['libgpuarray'].add_script(
'build',
ut.codeblock(r"""
# STARTBLOCK bash
# Ensure the repo was checked out
if [ ! -d {repo_dpath} ]; then
git clone https://github.com/Theano/libgpuarray.git {repo_dpath}
fi
{python_bash_setup}
cd {repo_dpath}
# need a specific version of libgpuarray
git checkout tags/v0.6.2 -b v0.6.2
mkdir -p {repo_dpath}/{build_dname}
cd {repo_dpath}/{build_dname}
# First build the C library
cmake {repo_dpath} -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
$_SUDO make install
# Now build the python libarary
cd {repo_dpath}
python setup.py build_ext -L $LOCAL_PREFIX/lib -I $LOCAL_PREFIX/include
python setup.py build
# python setup.py install
$_SUDO pip install -e {repo_dpath}
# DEVICE="<test device>" python -c "import pygpu;pygpu.test()"
# DEVICE="gpu0" python -c "import pygpu;pygpu.test()"
cd ~
$_SUDO pip install nose
DEVICE="cuda" python -c "import pygpu;pygpu.test()"
# pip uninstall pygpu
# ENDBLOCK
""").format(repo_dpath=ut.unexpanduser(tpl_rman['libgpuarray'].dpath),
**script_fmtdict),
)
# ===================
# PYQT SETUP SCRIPTS
# ===================
if ut.in_virtual_env():
try:
fmtdict = {
'sys_dist_packages':
ut.get_global_dist_packages_dir(),
'venv_site_packages':
ut.get_site_packages_dir(),
'pyqt':
'PyQt4' if PY2 else 'PyQt5',
# Need the PyQT5 SVG module for IPython to work properly
'debian-python-qt':
('python-qt4'
if PY2 else 'qt5-default python3-pyqt5 debian-python-qt-svg'),
'pip-python-qt':
'python-qt4' if PY2 else 'python-qt5',
}
# sys_dist_packages = ut.get_global_dist_packages_dir()
# sys_pyqt_dir = sys_dist_packages + '/{pyqt}'
# Allows us to use a system qt install in a virtual environment.
system_to_venv = ut.codeblock(r"""
# STARTBLOCK bash
# Creates a symlink to the global PyQt in a virtual env
export GLOBAL_DIST_PACKAGES="{sys_dist_packages}"
export VENV_DIST_PACKAGES="{venv_site_packages}"
if [ -d $GLOBAL_DIST_PACKAGES/{pyqt} ]; then
echo "have qt"
ls $GLOBAL_DIST_PACKAGES/{pyqt}
ls $VENV_DIST_PACKAGES/{pyqt}
else
# Ensure PyQt is installed first (FIXME make this work for non-debian systems)
sudo apt-get install {debian-python-qt}
# pip install {pip-python-qt}
fi
if [ -d $GLOBAL_DIST_PACKAGES/{pyqt} ]; then
# Install system pyqt packages to virtual envirment via symlink
ln -s $GLOBAL_DIST_PACKAGES/{pyqt}/ $VENV_DIST_PACKAGES/{pyqt}
ln -s $GLOBAL_DIST_PACKAGES/sip*.so $VENV_DIST_PACKAGES/
ln -s $GLOBAL_DIST_PACKAGES/sip*.py $VENV_DIST_PACKAGES/
else
echo "{pyqt} DOES NOT SEEM TO BE INSTALLED ON THE SYSTEM"
fi
echo "testing"
python -c "import {pyqt}; print({pyqt})"
# ENDBLOCK bash
""").format(**fmtdict)
# TODO: add custom build alternative
tpl_rman['PyQt'].add_script('system_to_venv', system_to_venv)
except NotImplementedError:
pass
def execute_commands(tpl_rman, wbia_rman):
import utool as ut
GET_ARGVAL = ut.get_argval
ut.init_catch_ctrl_c()
if 0:
print('Version Check Source:')
for repo in tpl_rman.repos:
print('python -c "import {0}; print({0}.__file__)"'.format(
repo.modname))
print('python -c "import {0}; print({0}.__version__)"'.format(
repo.modname))
# -----------
# Execute Commands on Core Repos
# -----------
CODE_DIR, pythoncmd, WIN32, PY2, PY3 = get_sysinfo()
print('wbia_rman = %r' % (wbia_rman, ))
wildme_ssh_flags = GET_ARGFLAG('--move-wildme') or GET_ARGFLAG(
'--move-wildme-ssh')
wildme_https_flags = GET_ARGFLAG('--move-wildme-https') or GET_ARGFLAG(
'--move-wildme-http')
if wildme_ssh_flags or wildme_https_flags:
fmt = 'ssh' if wildme_ssh_flags else 'https'
move_wildme(wbia_rman, fmt)
# Commands on global git repos
if GET_ARGFLAG('--status'):
wbia_rman.issue('git status')
sys.exit(0)
wbia_rman.ensure()
if GET_ARGFLAG('--dump') or GET_ARGFLAG('--dump-scripts'):
dpath = '_super_scripts/' + 'scripts' + get_plat_specifier()
ut.ensuredir(dpath)
dumps = [
(tpl_rman, 'cv2', 'build'),
(tpl_rman, 'cv2', 'install'),
(wbia_rman, 'flann', 'build'),
(wbia_rman, 'flann', 'install'),
(wbia_rman, 'hesaff', 'build'),
(tpl_rman, 'PyQt', 'system_to_venv'),
(tpl_rman, 'libgpuarray', 'build'),
]
for rman, mod, sname in dumps:
from os.path import join
# if mod not in rman:
# print('mod=%r not available in rman=%r' % (mod, rman))
# continue
script = rman[mod].get_script(sname).text
suffix = get_plat_specifier()
sh_fpath = join(dpath, mod + '_' + sname + suffix + '.sh')
ut.write_to(sh_fpath, script)
if GET_ARGFLAG('--requirements'):
ut.cmd('pip install -r requirements.txt')
# HACKED IN SCRIPTS WHILE IM STILL FIGURING OUT TPL DEPS
if GET_ARGFLAG('--opencv'):
# There is now a pypi for opencv! Yay
# ut.cmd('pip install opencv-python')
# Bummer, but we need opencv source for pyhessaff
# we should just make a wheel for pyhessaff
cv_repo = tpl_rman['cv2']
cv_repo.clone()
script = cv_repo.get_script('build')
script.exec_()
cv_repo = tpl_rman['cv2']
script = cv_repo.get_script('install')
script.exec_()
if GET_ARGFLAG('--flann'):
script = wbia_rman['flann'].get_script('build')
script.exec_()
script = wbia_rman['flann'].get_script('install')
script.exec_()
if GET_ARGFLAG('--pyqt'):
script = tpl_rman['PyQt'].get_script('system_to_venv')
script.exec_()
if GET_ARGFLAG('--hesaff'):
script = wbia_rman['hesaff'].get_script('build')
script.exec_()
if GET_ARGFLAG('--pydarknet'):
script = wbia_rman['pydarknet'].get_script('build')
script.exec_()
if GET_ARGFLAG('--pyrf'):
script = wbia_rman['pyrf'].get_script('build')
script.exec_()
if GET_ARGFLAG('--torch'):
# Theano and lasange code should be moved to pytorch
tpl_rman['pytorch'].clone(recursive=True)
tpl_rman['pytorch'].issue('git submodule update --init')
tpl_rman['pytorch'].issue('python setup install')
tpl_rman['pytorch'].issue('pip install torchvision')
# tpl_rman['pytorch'].issue('NO_CUDNN=TRUE && python setup install')
# tpl_rman['pytorch'].issue('pip install -e .')
if GET_ARGFLAG('--libgpuarray') or GET_ARGFLAG('--dcnn'):
tpl_rman['libgpuarray'].clone()
script = tpl_rman['libgpuarray'].get_script('build')
script.exec_()
if GET_ARGFLAG('--dcnn'):
tpl_rman['theano'].clone()
# tpl_rman['pylearn2'].clone()
tpl_rman['lasagne'].clone()
tpl_rman['theano'].issue('pip install -e .')
# tpl_rman['pylearn2'].issue('pip install -e .')
tpl_rman['lasagne'].issue('pip install -e .')
# tpl_rman['pylearn2'].python_develop()
# tpl_rman['theano'].python_develop()
# tpl_rman['lasagne'].python_develop()
# _===
if GET_ARGFLAG('--fix') or GET_ARGFLAG('--check'):
missing_dynlib = tpl_rman.check_cpp_build()
missing_dynlib += wbia_rman.check_cpp_build()
missing_install = tpl_rman.check_installed()
missing_install += wbia_rman.check_installed()
problems = []
problems += wbia_rman.check_importable()
problems += tpl_rman.check_importable()
if GET_ARGFLAG('--fix'):
print('Trying to fix problems')
for repo in missing_dynlib:
repo.custom_build()
for repo, recommended_fix in problems:
print('Trying to fix repo = %r' % (repo, ))
print(' * recommended_fix = %r' % (recommended_fix, ))
if recommended_fix == 'rebuild':
repo.custom_build()
print(
'Can currently only fix one module at a time. Please re-run'
)
sys.exit(1)
else:
print('Not sure how to fix %r' % (repo, ))
if GET_ARGFLAG('--pull'):
wbia_rman.issue('git pull')
if GET_ARGFLAG('--build'):
# Build tpl repos
# tpl_rman.custom_build()
# wbia_rman.custom_build()
# Build only IBEIS repos with setup.py
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py build'.format(pythoncmd=pythoncmd))
# Like install, but better if you are developing
if GET_ARGFLAG('--develop'):
_rman = wbia_rman.only_with_pysetup()
# # _rman.issue('{pythoncmd} setup.py develop'.format(pythoncmd=pythoncmd),
# # sudo=not ut.in_virtual_env())
_rman.issue(
'{pythoncmd} -m pip install -e .'.format(pythoncmd=pythoncmd),
sudo=not ut.in_virtual_env(),
)
if GET_ARGFLAG('--clean'):
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py clean'.format(pythoncmd=pythoncmd))
if GET_ARGFLAG('--install'):
print(
'WARNING: Dont use install if you are a developer. Use develop instead.'
)
_rman = wbia_rman.only_with_pysetup()
_rman.issue('{pythoncmd} setup.py install'.format(pythoncmd=pythoncmd))
if GET_ARGFLAG('--push'):
wbia_rman.issue('git push')
if GET_ARGFLAG('--branch'):
wbia_rman.issue('git branch')
sys.exit(0)
if GET_ARGFLAG('--tag-status'):
wbia_rman.issue('git tag')
# Tag everything
tag_name = GET_ARGVAL('--newtag', type_=str, default=None)
if tag_name is not None:
wbia_rman.issue(
'git tag -a "{tag_name}" -m "super_setup autotag {tag_name}"'.
format(**locals()))
wbia_rman.issue('git push --tags')
if GET_ARGFLAG('--bext'):
wbia_rman.issue('{pythoncmd} setup.py build_ext --inplace'.format(
pythoncmd=pythoncmd))
commit_msg = GET_ARGVAL('--commit', type_=str, default=None)
if commit_msg is not None:
wbia_rman.issue('git commit -am "{commit_msg}"'.format(**locals()))
# Change Branch
branch_name = GET_ARGVAL('--checkout', type_=str, default=None)
if branch_name is not None:
try:
wbia_rman.issue('git checkout "{branch_name}"'.format(**locals()))
except Exception:
print('ERROR: Could not checkout branch: %r' % (branch_name, ))
# Creates new branches
newbranch_name = GET_ARGVAL('--newbranch', type_=str, default=None)
if newbranch_name is not None:
# rman.issue('git stash"'.format(**locals()))
wbia_rman.issue(
'git checkout -b "{newbranch_name}"'.format(**locals()))
wbia_rman.issue(
'git push --set-upstream origin {newbranch_name}'.format(
**locals()))
# rman.issue('git stash pop"'.format(**locals()))
# Creates new branches
newlocalbranch_name = GET_ARGVAL('--newlocalbranch',
type_=str,
default=None)
if newlocalbranch_name is not None:
# rman.issue('git stash"'.format(**locals()))
wbia_rman.issue(
'git checkout -b "{newlocalbranch_name}"'.format(**locals()))
# rman.issue('git push --set-upstream origin {newlocalbranch_name}'.format(**locals()))
# rman.issue('git stash pop"'.format(**locals()))
# Creates new branches
mergebranch_name = GET_ARGVAL('--merge', type_=str, default=None)
if mergebranch_name is not None:
wbia_rman.issue('git merge "{mergebranch_name}"'.format(**locals()))
# Change ownership
if GET_ARGFLAG('--serverchmod'):
wbia_rman.issue('chmod -R 755 *')
if GET_ARGFLAG('--chown'):
# Fixes problems where repos are checked out as root
username = os.environ.get('USERNAME', ut.get_argval('--username'))
if username is None:
username = os.environ.get('USER', None)
if username is None:
raise AssertionError(
'cannot find username in commandline or environment vars')
usergroup = username
wbia_rman.issue('chown -R {username}:{usergroup} *'.format(**locals()),
sudo=True)
upstream_branch = GET_ARGVAL('--set-upstream', type_=str, default=None)
if upstream_branch is not None:
# git 2.0
wbia_rman.issue(
'git branch --set-upstream-to=origin/{upstream_branch} {upstream_branch}'
.format(**locals()))
upstream_push = GET_ARGVAL('--upstream-push', type_=str, default=None)
if upstream_push is not None:
wbia_rman.issue(
'git push --set-upstream origin {upstream_push}'.format(
**locals()))
if GET_ARGFLAG('--test'):
failures = []
for repo_dpath in wbia_rman.repo_dirs:
# ut.getp_
mod_dpaths = ut.get_submodules_from_dpath(repo_dpath,
recursive=False,
only_packages=True)
modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths)
print('Checking modules = %r' % (modname_list, ))
for modname in modname_list:
try:
ut.import_modname(modname)
print(modname + ' success')
except ImportError:
failures += [modname]
print(modname + ' failure')
print('failures = %s' % (ut.repr3(failures), ))
if False:
try:
from six.moves import input
except ImportError:
input = raw_input # NOQA
# General global git command
gg_cmd = GET_ARGVAL('--gg', None) # global command
if gg_cmd is not None:
ans = ('yes' if GET_ARGFLAG('-y') else input(
'Are you sure you want to run: %r on all directories? ' %
(gg_cmd, )))
if ans == 'yes':
wbia_rman.issue(gg_cmd)
def python_develop(repo):
import utool as ut
repo.issue('{pythoncmd} setup.py develop'.format(
pythoncmd=repo.pythoncmd), sudo=not ut.in_virtual_env())
def python_develop(repo):
import utool as ut
repo.issue('{pythoncmd} -m pip install -e {dpath}'.format(
pythoncmd=repo.pythoncmd, dpath=repo.dpath),
sudo=not ut.in_virtual_env())
def define_custom_scripts(tpl_rman):
if ut.in_virtual_env():
fmtdict = {
'sys_dist_packages': ut.get_global_dist_packages_dir(),
'venv_site_packages': ut.get_site_packages_dir(),
}
# Allows us to use a system qt install in a virtual environment.
tpl_rman['PyQt4'].add_script('system_to_venv', ut.codeblock(
r"""
# STARTBLOCK bash
ln -s {sys_dist_packages}/PyQt4/ {venv_site_packages}/PyQt4
ln -s {sys_dist_packages}/sip*.so {venv_site_packages}/
ln -s {sys_dist_packages}/sip*.py {venv_site_packages}/
# ENDBLOCK bash
""")).format(**fmtdict)
# TODO: add custom build alternative
pass
else:
pass
ibeis_rman['pyflann'].add_script('install', ut.codeblock(
r'''
# STARTBLOCK bash
# The pyflann source lives here
cd {repo_dir}/src/python
# But the setup script is generated during build
python {repo_dir}/build/src/python/setup.py develop
# ENDBLOCK bash
''').format(repo_dir=ibeis_rman['pyflann'].dpath)
)
# TODO: allow system installation as well
tpl_rman['cv2'].add_script('build', ut.codeblock(
r"""
# STARTBLOCK bash
cd $CODE_DIR
git clone https://github.com/Itseez/opencv.git
cd opencv
# Get Extras
git clone https://github.com/Itseez/opencv_contrib.git
mkdir -p build27
cd build27
if [[ "$VIRTUAL_ENV" == "" ]]; then
export LOCAL_PREFIX=/usr/local
export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/dist-packages
else
export LOCAL_PREFIX=$VIRTUAL_ENV/local
export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/site-packages
fi
echo "LOCAL_PREFIX = $LOCAL_PREFIX"
echo "PYTHON2_PACKAGES_PATH = $PYTHON2_PACKAGES_PATH"
# use dist packages on ubuntu. may need to change for other platforms
cmake -G "Unix Makefiles" \
-D WITH_OPENMP=ON \
-D CMAKE_BUILD_TYPE=RELEASE \
-D PYTHON2_PACKAGES_PATH=$PYTHON2_PACKAGES_PATH \
-D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \
-D OPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules \
..
export NCPUS=$(grep -c ^processor /proc/cpuinfo)
make -j$NCPUS
# ENDBLOCK
"""))
tpl_rman['cv2'].add_script('install', ut.codeblock(
r"""
# STARTBLOCK bash
cd $CODE_DIR
sudo make install
# Hack because cv2 does not want to be installed for some reason
cp lib/cv2.so $PYTHON2_PACKAGES_PATH
# Test makesure things working
python -c "import numpy; print(numpy.__file__)"
python -c "import numpy; print(numpy.__version__)"
python -c "import cv2; print(cv2.__version__)"
python -c "import cv2; print(cv2.__file__)"
#python -c "import vtool"
# Check if we have contrib modules
python -c "import cv2; print(cv2.xfeatures2d)"
# ENDBLOCK
""")
)
