def tryimport(modname, pipiname=None, ensure=False): """ CommandLine: python -m utool.util_import --test-tryimport Example: >>> # ENABLE_DOCTEST >>> from utool.util_tests import * # NOQA >>> import utool as ut >>> modname = 'pyfiglet' >>> pipiname = 'git+https://github.com/pwaller/pyfiglet' >>> pyfiglet = ut.tryimport(modname, pipiname) >>> assert pyfiglet is None or isinstance(pyfiglet, types.ModuleType), 'unknown error' Example2: >>> # UNSTABLE_DOCTEST >>> # disabled because not everyone has access to being a super user >>> from utool.util_tests import * # NOQA >>> import utool as ut >>> modname = 'lru' >>> pipiname = 'git+https://github.com/amitdev/lru-dict' >>> lru = ut.tryimport(modname, pipiname, ensure=True) >>> assert isinstance(lru, types.ModuleType), 'did not ensure lru' """ if pipiname is None: pipiname = modname try: if util_inject.PRINT_INJECT_ORDER: if modname not in sys.modules: util_inject.noinject(modname, N=2, via='ut.tryimport') module = __import__(modname) return module except ImportError as ex: import utool as ut base_pipcmd = 'pip install %s' % pipiname sudo = not ut.WIN32 and not ut.in_virtual_env() if sudo: pipcmd = 'sudo ' + base_pipcmd else: pipcmd = base_pipcmd msg = 'unable to find module %s. Please install: %s' % ((modname), (pipcmd)) print(msg) ut.printex(ex, msg, iswarning=True) if ensure: raise AssertionError('Ensure is dangerous behavior and is is no longer supported.') #raise NotImplementedError('not ensuring') ut.cmd(base_pipcmd, sudo=sudo) module = tryimport(modname, pipiname, ensure=False) if module is None: raise AssertionError('Cannot ensure modname=%r please install using %r' % (modname, pipcmd)) return module return None
def get_local_dist_packages_dir(): """ Attempts to work around virtualenvs and find the system dist_pacakges. Essentially this is implmenented as a lookuptable """ import utool as ut if not ut.in_virtual_env(): # Non venv case return get_site_packages_dir() else: candidates = [] if ut.LINUX: candidates += [ '/usr/local/lib/python2.7/dist-packages', ] else: raise NotImplementedError() for path in candidates: if ut.checkpath(path): return path
def define_custom_scripts(tpl_rman): if ut.in_virtual_env(): fmtdict = { 'sys_dist_packages': ut.get_global_dist_packages_dir(), 'venv_site_packages': ut.get_site_packages_dir(), } # Allows us to use a system qt install in a virtual environment. tpl_rman['PyQt4'].add_script( 'system_to_venv', ut.codeblock(r""" # STARTBLOCK bash ln -s {sys_dist_packages}/PyQt4/ {venv_site_packages}/PyQt4 ln -s {sys_dist_packages}/sip*.so {venv_site_packages}/ ln -s {sys_dist_packages}/sip*.py {venv_site_packages}/ # ENDBLOCK bash """)).format(**fmtdict) # TODO: add custom build alternative pass else: pass ibeis_rman['pyflann'].add_script( 'install', ut.codeblock(r''' # STARTBLOCK bash # The pyflann source lives here cd {repo_dir}/src/python # But the setup script is generated during build python {repo_dir}/build/src/python/setup.py develop # ENDBLOCK bash ''').format(repo_dir=ibeis_rman['pyflann'].dpath)) # TODO: allow system installation as well tpl_rman['cv2'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash cd $CODE_DIR git clone https://github.com/Itseez/opencv.git cd opencv # Get Extras git clone https://github.com/Itseez/opencv_contrib.git mkdir -p build27 cd build27 if [[ "$VIRTUAL_ENV" == "" ]]; then export LOCAL_PREFIX=/usr/local export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/dist-packages else export LOCAL_PREFIX=$VIRTUAL_ENV/local export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/site-packages fi echo "LOCAL_PREFIX = $LOCAL_PREFIX" echo "PYTHON2_PACKAGES_PATH = $PYTHON2_PACKAGES_PATH" # use dist packages on ubuntu. may need to change for other platforms cmake -G "Unix Makefiles" \ -D WITH_OPENMP=ON \ -D CMAKE_BUILD_TYPE=RELEASE \ -D PYTHON2_PACKAGES_PATH=$PYTHON2_PACKAGES_PATH \ -D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \ -D OPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules \ .. export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS # ENDBLOCK """)) tpl_rman['cv2'].add_script( 'install', ut.codeblock(r""" # STARTBLOCK bash cd $CODE_DIR sudo make install # Hack because cv2 does not want to be installed for some reason cp lib/cv2.so $PYTHON2_PACKAGES_PATH # Test makesure things working python -c "import numpy; print(numpy.__file__)" python -c "import numpy; print(numpy.__version__)" python -c "import cv2; print(cv2.__version__)" python -c "import cv2; print(cv2.__file__)" #python -c "import vtool" # Check if we have contrib modules python -c "import cv2; print(cv2.xfeatures2d)" # ENDBLOCK """))
if GET_ARGFLAG('--pull'): ibeis_rman.issue('git pull') if GET_ARGFLAG('--build'): # Build tpl repos tpl_rman.custom_build() ibeis_rman.custom_build() # Build only IBEIS repos with setup.py _rman = ibeis_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py build'.format(**locals())) # Like install, but better if you are developing if GET_ARGFLAG('--develop'): _rman = ibeis_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py develop'.format(**locals()), sudo=not ut.in_virtual_env()) if GET_ARGFLAG('--clean'): _rman = ibeis_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py clean'.format(**locals())) if GET_ARGFLAG('--install'): print( 'WARNING: Dont use install if you are a developer. Use develop instead.' ) _rman = ibeis_rman.only_with_pysetup() _rman.issue('python setup.py install'.format(**locals())) if GET_ARGFLAG('--push'): ibeis_rman.issue('git push')
if GET_ARGFLAG('--bext'): ut.gg_command('{pythoncmd} setup.py build_ext --inplace'.format(**locals())) if GET_ARGFLAG('--build'): # Build tpl repos for repo in TPL_REPO_DIRS: ut.util_git.std_build_command(repo) # Executes {plat}_build.{ext} # Build only IBEIS repos with setup.py ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS) ut.gg_command('{pythoncmd} setup.py build'.format(**locals())) if GET_ARGFLAG('--develop'): # Like install, but better if you are developing ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS) ut.gg_command('{pythoncmd} setup.py develop'.format(**locals()), sudo=not ut.in_virtual_env()) if GET_ARGFLAG('--install'): # Dont use this if you are a developer. Use develop instead. ut.set_project_repos(IBEIS_REPO_URLS, IBEIS_REPO_DIRS) ut.gg_command('python setup.py install'.format(**locals())) if GET_ARGFLAG('--test'): failures = [] for repo_dpath in IBEIS_REPO_DIRS: # ut.getp_ mod_dpaths = ut.get_submodules_from_dpath(repo_dpath, recursive=False, only_packages=True) modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths) print('Checking modules = %r' % (modname_list,))
def python_develop(repo): import utool as ut repo.issue("{pythoncmd} setup.py develop".format(pythoncmd=repo.pythoncmd), sudo=not ut.in_virtual_env())
def define_custom_scripts(tpl_rman, wbia_rman, PY2, PY3): """ export THEANO_FLAGS="device=cpu,print_active_device=True,enable_initial_driver_test=True" set THEANO_FLAGS=device=cpu,print_active_device=True,enable_initial_driver_test=True,print_test_value=True python -c "import pydot; print(pydot.__file__)" python -c "import pydot; print(pydot.__version__)" python -c "import pydot; print(pydot.find_graphviz())" DEVICE="cuda" python -c "import pygpu;pygpu.test()" python -c "import theano; print(theano.__file__)" # python -c "import pylearn2; print(pylearn2.__file__)" python -c "import lasagne; print(lasagne.__file__)" python -c "import wbia_cnn; print(wbia_cnn.__file__)" python -c "import detecttools; print(detecttools.__file__)" # http://stackoverflow.com/questions/18042919/how-to-install-pyqt5-on-a-new-virtualenv-and-work-on-an-idle pip install vext.pyqt5 sudo apt-get install pyqt5-dev sudo apt-get install python3-pyqt5 python python -c "import sip; print('[test] Python can import sip')" python -c "import sip; print('sip.__file__=%r' % (sip.__file__,))" python -c "import sip; print('sip.SIP_VERSION=%r' % (sip.SIP_VERSION,))" python -c "import sip; print('sip.SIP_VERSION_STR=%r' % (sip.SIP_VERSION_STR,))" ln -s /usr/lib/python3/dist-packages/PyQt5/ /home/joncrall/venv3/lib/python3.4/site-packages/PyQt5 ln -s /usr/lib/python3/dist-packages/sip*.so /home/joncrall/venv3/lib/python3.4/site-packages/ ln -s /usr/lib/python3/dist-packages/sip*.py /home/joncrall/venv3/lib/python3.4/site-packages/ """ import utool as ut major = str(sys.version_info.major) minor = str(sys.version_info.minor) majorminor = [major, minor] pyoff = '2' if sys.version_info.major == 3 else '3' pyon = majorminor[0] plat_spec = get_plat_specifier() # build_dname = 'build' + ''.join(majorminor) build_dname = 'cmake_builds/build' + plat_spec script_fmtdict = { 'pyexe': sys.executable, 'pyversion': 'python' + '.'.join(majorminor), 'pypkg_var': 'PYTHON' + pyon + '_PACKAGES_PATH', 'build_dname': build_dname, 'pyoff': pyoff, 'pyon': pyon, 'cv_pyon_var': 'BUILD_opencv_python' + pyon, 'cv_pyoff_var': 'BUILD_opencv_python' + pyoff, 'plat_spec': plat_spec, 'source_dpath': '../..', 'libext': ut.get_lib_ext(), } if os.environ.get('VIRTUAL_ENV', '') == '': if sys.platform.startswith('darwin'): local_prefix = '/opt/local' else: local_prefix = '/usr/local' else: local_prefix = os.environ['VIRTUAL_ENV'] opencv_dir = os.path.join(local_prefix, '/share/OpenCV') if not os.path.exists(opencv_dir): if not ut.get_argflag('--opencv'): opencv_dir = '' print( 'OpenCV is not installed in the expected location: {}'.format( opencv_dir)) print( 'Running this script with --opencv will build and install it there' ) # define bash variables for different combinations of python distros and # virtual environments python_bash_setup = ut.codeblock(r""" # STARTBLOCK bash if [[ "$VIRTUAL_ENV" == "" ]]; then # The case where we are installying system-wide # It is recommended that a virtual enviornment is used instead export PYTHON_EXECUTABLE=$(which {pyversion}) if [[ '$OSTYPE' == 'darwin'* ]]; then # Mac system info export LOCAL_PREFIX=/opt/local export {pypkg_var}=$($PYTHON_EXECUTABLE -c "import site; print(site.getsitepackages()[0])") export PYTHON_PACKAGES_PATH=${pypkg_var} export _SUDO="sudo" else # Linux system info export LOCAL_PREFIX=/usr/local export {pypkg_var}=$LOCAL_PREFIX/lib/{pyversion}/dist-packages export PYTHON_PACKAGES_PATH=${pypkg_var} export _SUDO="sudo" fi # No windows support here else # The prefered case where we are in a virtual environment export PYTHON_EXECUTABLE=$(which python) # export LOCAL_PREFIX=$VIRTUAL_ENV/local export LOCAL_PREFIX=$VIRTUAL_ENV export {pypkg_var}=$LOCAL_PREFIX/lib/{pyversion}/site-packages export PYTHON_PACKAGES_PATH=${pypkg_var} export _SUDO="" fi echo "LOCAL_PREFIX = $LOCAL_PREFIX" echo "{pypkg_var} = ${pypkg_var}" # ENDBLOCK bash """).format(**script_fmtdict) script_fmtdict['python_bash_setup'] = python_bash_setup # =================== # PYFLANN SETUP SCRIPTS # =================== wbia_rman['pyflann'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} cd {repo_dir} mkdir -p {build_dname} cd {build_dname} cmake -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE="Release" \ -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \ -DBUILD_EXAMPLES=Off \ -DBUILD_TESTS=Off \ -DBUILD_PYTHON_BINDINGS=On \ -DBUILD_MATLAB_BINDINGS=Off \ -DBUILD_CUDA_LIB=Off\ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX\ {source_dpath} export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS # ENDBLOCK bash """).format(repo_dir=wbia_rman['pyflann'].dpath, **script_fmtdict), ) wbia_rman['pyflann'].add_script( 'install', ut.codeblock(r""" # STARTBLOCK bash # The pyflann source lives here cd {repo_dir}/src/python # Need to run build to move the libs to the build directory python setup.py build # Use pip to editable install pip install -e {repo_dir}/src/python # Old way of doing it # But the setup script is generated during build # python {repo_dir}/build/src/python/setup.py develop python -c "from vtool_ibeis._pyflann_backend import pyflann as pyflann; print(pyflann.__file__)" --verb-flann python -c "from vtool_ibeis._pyflann_backend import pyflann as pyflann; print(pyflann)" --verb-flann # ENDBLOCK bash """).format(repo_dir=wbia_rman['pyflann'].dpath), ) # =================== # HESAFF # =================== wbia_rman['hesaff'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} cd $CODE_DIR/hesaff mkdir -p {build_dname} cd {build_dname} # only specify an explicit opencv directory if we know one exists if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV" else OPENCV_ARGS="" fi echo 'Configuring with cmake' if [[ '$OSTYPE' == 'darwin'* ]]; then cmake -G "Unix Makefiles" \ -DCMAKE_OSX_ARCHITECTURES=x86_64 \ -DCMAKE_C_COMPILER=clang2 \ -DCMAKE_CXX_COMPILER=clang2++ \ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \ $OPENCV_ARGS \ {source_dpath} else cmake -G "Unix Makefiles" \ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \ $OPENCV_ARGS \ {source_dpath} fi export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS export MAKE_EXITCODE=$? echo "MAKE_EXITCODE=$MAKE_EXITCODE" # Move the compiled library into the source folder if [[ $MAKE_EXITCODE == 0 ]]; then #make VERBOSE=1 cp -v libhesaff{libext} {source_dpath}/pyhesaff/libhesaff{plat_spec}{libext} fi # ENDBLOCK """).format(**script_fmtdict), ) # =================== # PYDARKNET # =================== wbia_rman['pydarknet'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} cd $CODE_DIR/pydarknet mkdir -p {build_dname} cd {build_dname} if [[ "$(which nvcc)" == "" ]]; then export CMAKE_CUDA=Off else export CMAKE_CUDA=On fi # only specify an explicit opencv directory if we know one exists if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV" else OPENCV_ARGS="" fi echo 'Configuring with cmake' if [[ '$OSTYPE' == 'darwin'* ]]; then export CONFIG="-DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_C_COMPILER=clang2 -DCMAKE_CXX_COMPILER=clang2++ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS" else export CONFIG="-DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS" fi export CONFIG="$CONFIG -DCUDA=$CMAKE_CUDA" echo "CONFIG = $CONFIG" cmake $CONFIG -G 'Unix Makefiles' {source_dpath} ################################# echo 'Building with make' export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS -w ################################# export MAKE_EXITCODE=$? echo "MAKE_EXITCODE=$MAKE_EXITCODE" # Move the compiled library into the source folder if [[ $MAKE_EXITCODE == 0 ]]; then echo 'Moving the shared library' # cp -v lib* ../pydarknet cp -v lib*{libext} {source_dpath}/pydarknet # cp -v libdarknet{libext} {source_dpath}/pydarknet/libdarknet{plat_spec}{libext} fi # ENDBLOCK """).format(**script_fmtdict), ) # =================== # PYRF # =================== wbia_rman['pyrf'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} cd $CODE_DIR/pyrf mkdir -p {build_dname} cd {build_dname} # only specify an explicit opencv directory if we know one exists if [ -d "$LOCAL_PREFIX/share/OpenCV" ]; then OPENCV_ARGS="-DOpenCV_DIR=$LOCAL_PREFIX/share/OpenCV" else OPENCV_ARGS="" fi echo 'Configuring with cmake' if [[ '$OSTYPE' == 'darwin'* ]]; then export CONFIG="-DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_C_COMPILER=clang2 -DCMAKE_CXX_COMPILER=clang2++ -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS" else export CONFIG="-DCMAKE_BUILD_TYPE='Release' -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX $OPENCV_ARGS" fi echo "CONFIG = $CONFIG" cmake $CONFIG -G 'Unix Makefiles' {source_dpath} ################################# echo 'Building with make' export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS -w ################################# export MAKE_EXITCODE=$? echo "MAKE_EXITCODE=$MAKE_EXITCODE" # Move the compiled library into the source folder if [[ $MAKE_EXITCODE == 0 ]]; then echo 'Moving the shared library' # cp -v lib* ../pyrf cp -v lib*{libext} {source_dpath}/pyrf # cp -v libpyrf{libext} {source_dpath}/pyrf/libpyrf{plat_spec}{libext} fi # ENDBLOCK """).format(**script_fmtdict), ) # =================== # OPENCV SETUP SCRIPTS # =================== """ ./super_setup.py --dump-scripts """ tpl_rman['cv2'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} # Checkout opencv core cd $CODE_DIR # export REPO_DIR=$CODE_DIR/opencv export REPO_DIR={repo_dpath} # git clone https://github.com/Itseez/opencv.git cd $REPO_DIR # Checkout opencv extras git clone https://github.com/Itseez/opencv_contrib.git # cd opencv_contrib # git pull # cd .. # git pull mkdir -p $REPO_DIR/{build_dname} cd $REPO_DIR/{build_dname} cmake -G "Unix Makefiles" \ -D WITH_OPENMP=ON \ -D CMAKE_BUILD_TYPE=RELEASE \ -D {cv_pyoff_var}=Off \ -D {cv_pyon_var}=On \ -D PYTHON_DEFAULT_EXECUTABLE="{pyexe}" \ -D {pypkg_var}=${pypkg_var} \ -D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \ -D OPENCV_EXTRA_MODULES_PATH=$REPO_DIR/opencv_contrib/modules \ -D WITH_CUDA=Off \ -D BUILD_opencv_dnn=Off \ -D BUILD_opencv_dnn_modern=Off \ -D WITH_VTK=Off \ -D WITH_CUDA=Off \ -D WITH_MATLAB=Off \ $REPO_DIR # -D WITH_OPENCL=Off \ # -D BUILD_opencv_face=Off \ # -D BUILD_opencv_objdetect=Off \ # -D BUILD_opencv_video=Off \ # -D BUILD_opencv_videoio=Off \ # -D BUILD_opencv_videostab=Off \ # -D BUILD_opencv_ximgproc=Off \ # -D BUILD_opencv_xobjdetect=Off \ # -D BUILD_opencv_xphoto=Off \ # -D BUILD_opencv_datasets=Off \ # -D CXX_FLAGS="-std=c++11" \ %TODO export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS # ENDBLOCK """).format(repo_dpath=ut.unexpanduser(tpl_rman['cv2'].dpath), **script_fmtdict), ) tpl_rman['cv2'].add_script( 'install', ut.codeblock(r""" # STARTBLOCK bash {python_bash_setup} cd $CODE_DIR/opencv/{build_dname} $_SUDO make install # Hack because cv2 does not want to be installed for some reason # cp lib/cv2.so $PYTHON_PACKAGES_PATH # Seems to work now that local is removed from prefix # cp -v lib/cv2.so $PYTHON_PACKAGES_PATH # Test makesure things working python -c "import numpy; print(numpy.__file__)" python -c "import numpy; print(numpy.__version__)" python -c "import cv2; print(cv2.__version__)" python -c "import cv2; print(cv2.__file__)" #python -c "import vtool_ibeis" # Check if we have contrib modules python -c "import cv2; print(cv2.xfeatures2d)" # ENDBLOCK """).format(**script_fmtdict), ) # if GET_ARGFLAG('--libgpuarray'): tpl_rman['libgpuarray'].add_script( 'build', ut.codeblock(r""" # STARTBLOCK bash # Ensure the repo was checked out if [ ! -d {repo_dpath} ]; then git clone https://github.com/Theano/libgpuarray.git {repo_dpath} fi {python_bash_setup} cd {repo_dpath} # need a specific version of libgpuarray git checkout tags/v0.6.2 -b v0.6.2 mkdir -p {repo_dpath}/{build_dname} cd {repo_dpath}/{build_dname} # First build the C library cmake {repo_dpath} -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$LOCAL_PREFIX export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS $_SUDO make install # Now build the python libarary cd {repo_dpath} python setup.py build_ext -L $LOCAL_PREFIX/lib -I $LOCAL_PREFIX/include python setup.py build # python setup.py install $_SUDO pip install -e {repo_dpath} # DEVICE="<test device>" python -c "import pygpu;pygpu.test()" # DEVICE="gpu0" python -c "import pygpu;pygpu.test()" cd ~ $_SUDO pip install nose DEVICE="cuda" python -c "import pygpu;pygpu.test()" # pip uninstall pygpu # ENDBLOCK """).format(repo_dpath=ut.unexpanduser(tpl_rman['libgpuarray'].dpath), **script_fmtdict), ) # =================== # PYQT SETUP SCRIPTS # =================== if ut.in_virtual_env(): try: fmtdict = { 'sys_dist_packages': ut.get_global_dist_packages_dir(), 'venv_site_packages': ut.get_site_packages_dir(), 'pyqt': 'PyQt4' if PY2 else 'PyQt5', # Need the PyQT5 SVG module for IPython to work properly 'debian-python-qt': ('python-qt4' if PY2 else 'qt5-default python3-pyqt5 debian-python-qt-svg'), 'pip-python-qt': 'python-qt4' if PY2 else 'python-qt5', } # sys_dist_packages = ut.get_global_dist_packages_dir() # sys_pyqt_dir = sys_dist_packages + '/{pyqt}' # Allows us to use a system qt install in a virtual environment. system_to_venv = ut.codeblock(r""" # STARTBLOCK bash # Creates a symlink to the global PyQt in a virtual env export GLOBAL_DIST_PACKAGES="{sys_dist_packages}" export VENV_DIST_PACKAGES="{venv_site_packages}" if [ -d $GLOBAL_DIST_PACKAGES/{pyqt} ]; then echo "have qt" ls $GLOBAL_DIST_PACKAGES/{pyqt} ls $VENV_DIST_PACKAGES/{pyqt} else # Ensure PyQt is installed first (FIXME make this work for non-debian systems) sudo apt-get install {debian-python-qt} # pip install {pip-python-qt} fi if [ -d $GLOBAL_DIST_PACKAGES/{pyqt} ]; then # Install system pyqt packages to virtual envirment via symlink ln -s $GLOBAL_DIST_PACKAGES/{pyqt}/ $VENV_DIST_PACKAGES/{pyqt} ln -s $GLOBAL_DIST_PACKAGES/sip*.so $VENV_DIST_PACKAGES/ ln -s $GLOBAL_DIST_PACKAGES/sip*.py $VENV_DIST_PACKAGES/ else echo "{pyqt} DOES NOT SEEM TO BE INSTALLED ON THE SYSTEM" fi echo "testing" python -c "import {pyqt}; print({pyqt})" # ENDBLOCK bash """).format(**fmtdict) # TODO: add custom build alternative tpl_rman['PyQt'].add_script('system_to_venv', system_to_venv) except NotImplementedError: pass
def execute_commands(tpl_rman, wbia_rman): import utool as ut GET_ARGVAL = ut.get_argval ut.init_catch_ctrl_c() if 0: print('Version Check Source:') for repo in tpl_rman.repos: print('python -c "import {0}; print({0}.__file__)"'.format( repo.modname)) print('python -c "import {0}; print({0}.__version__)"'.format( repo.modname)) # ----------- # Execute Commands on Core Repos # ----------- CODE_DIR, pythoncmd, WIN32, PY2, PY3 = get_sysinfo() print('wbia_rman = %r' % (wbia_rman, )) wildme_ssh_flags = GET_ARGFLAG('--move-wildme') or GET_ARGFLAG( '--move-wildme-ssh') wildme_https_flags = GET_ARGFLAG('--move-wildme-https') or GET_ARGFLAG( '--move-wildme-http') if wildme_ssh_flags or wildme_https_flags: fmt = 'ssh' if wildme_ssh_flags else 'https' move_wildme(wbia_rman, fmt) # Commands on global git repos if GET_ARGFLAG('--status'): wbia_rman.issue('git status') sys.exit(0) wbia_rman.ensure() if GET_ARGFLAG('--dump') or GET_ARGFLAG('--dump-scripts'): dpath = '_super_scripts/' + 'scripts' + get_plat_specifier() ut.ensuredir(dpath) dumps = [ (tpl_rman, 'cv2', 'build'), (tpl_rman, 'cv2', 'install'), (wbia_rman, 'flann', 'build'), (wbia_rman, 'flann', 'install'), (wbia_rman, 'hesaff', 'build'), (tpl_rman, 'PyQt', 'system_to_venv'), (tpl_rman, 'libgpuarray', 'build'), ] for rman, mod, sname in dumps: from os.path import join # if mod not in rman: # print('mod=%r not available in rman=%r' % (mod, rman)) # continue script = rman[mod].get_script(sname).text suffix = get_plat_specifier() sh_fpath = join(dpath, mod + '_' + sname + suffix + '.sh') ut.write_to(sh_fpath, script) if GET_ARGFLAG('--requirements'): ut.cmd('pip install -r requirements.txt') # HACKED IN SCRIPTS WHILE IM STILL FIGURING OUT TPL DEPS if GET_ARGFLAG('--opencv'): # There is now a pypi for opencv! Yay # ut.cmd('pip install opencv-python') # Bummer, but we need opencv source for pyhessaff # we should just make a wheel for pyhessaff cv_repo = tpl_rman['cv2'] cv_repo.clone() script = cv_repo.get_script('build') script.exec_() cv_repo = tpl_rman['cv2'] script = cv_repo.get_script('install') script.exec_() if GET_ARGFLAG('--flann'): script = wbia_rman['flann'].get_script('build') script.exec_() script = wbia_rman['flann'].get_script('install') script.exec_() if GET_ARGFLAG('--pyqt'): script = tpl_rman['PyQt'].get_script('system_to_venv') script.exec_() if GET_ARGFLAG('--hesaff'): script = wbia_rman['hesaff'].get_script('build') script.exec_() if GET_ARGFLAG('--pydarknet'): script = wbia_rman['pydarknet'].get_script('build') script.exec_() if GET_ARGFLAG('--pyrf'): script = wbia_rman['pyrf'].get_script('build') script.exec_() if GET_ARGFLAG('--torch'): # Theano and lasange code should be moved to pytorch tpl_rman['pytorch'].clone(recursive=True) tpl_rman['pytorch'].issue('git submodule update --init') tpl_rman['pytorch'].issue('python setup install') tpl_rman['pytorch'].issue('pip install torchvision') # tpl_rman['pytorch'].issue('NO_CUDNN=TRUE && python setup install') # tpl_rman['pytorch'].issue('pip install -e .') if GET_ARGFLAG('--libgpuarray') or GET_ARGFLAG('--dcnn'): tpl_rman['libgpuarray'].clone() script = tpl_rman['libgpuarray'].get_script('build') script.exec_() if GET_ARGFLAG('--dcnn'): tpl_rman['theano'].clone() # tpl_rman['pylearn2'].clone() tpl_rman['lasagne'].clone() tpl_rman['theano'].issue('pip install -e .') # tpl_rman['pylearn2'].issue('pip install -e .') tpl_rman['lasagne'].issue('pip install -e .') # tpl_rman['pylearn2'].python_develop() # tpl_rman['theano'].python_develop() # tpl_rman['lasagne'].python_develop() # _=== if GET_ARGFLAG('--fix') or GET_ARGFLAG('--check'): missing_dynlib = tpl_rman.check_cpp_build() missing_dynlib += wbia_rman.check_cpp_build() missing_install = tpl_rman.check_installed() missing_install += wbia_rman.check_installed() problems = [] problems += wbia_rman.check_importable() problems += tpl_rman.check_importable() if GET_ARGFLAG('--fix'): print('Trying to fix problems') for repo in missing_dynlib: repo.custom_build() for repo, recommended_fix in problems: print('Trying to fix repo = %r' % (repo, )) print(' * recommended_fix = %r' % (recommended_fix, )) if recommended_fix == 'rebuild': repo.custom_build() print( 'Can currently only fix one module at a time. Please re-run' ) sys.exit(1) else: print('Not sure how to fix %r' % (repo, )) if GET_ARGFLAG('--pull'): wbia_rman.issue('git pull') if GET_ARGFLAG('--build'): # Build tpl repos # tpl_rman.custom_build() # wbia_rman.custom_build() # Build only IBEIS repos with setup.py _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py build'.format(pythoncmd=pythoncmd)) # Like install, but better if you are developing if GET_ARGFLAG('--develop'): _rman = wbia_rman.only_with_pysetup() # # _rman.issue('{pythoncmd} setup.py develop'.format(pythoncmd=pythoncmd), # # sudo=not ut.in_virtual_env()) _rman.issue( '{pythoncmd} -m pip install -e .'.format(pythoncmd=pythoncmd), sudo=not ut.in_virtual_env(), ) if GET_ARGFLAG('--clean'): _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py clean'.format(pythoncmd=pythoncmd)) if GET_ARGFLAG('--install'): print( 'WARNING: Dont use install if you are a developer. Use develop instead.' ) _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py install'.format(pythoncmd=pythoncmd)) if GET_ARGFLAG('--push'): wbia_rman.issue('git push') if GET_ARGFLAG('--branch'): wbia_rman.issue('git branch') sys.exit(0) if GET_ARGFLAG('--tag-status'): wbia_rman.issue('git tag') # Tag everything tag_name = GET_ARGVAL('--newtag', type_=str, default=None) if tag_name is not None: wbia_rman.issue( 'git tag -a "{tag_name}" -m "super_setup autotag {tag_name}"'. format(**locals())) wbia_rman.issue('git push --tags') if GET_ARGFLAG('--bext'): wbia_rman.issue('{pythoncmd} setup.py build_ext --inplace'.format( pythoncmd=pythoncmd)) commit_msg = GET_ARGVAL('--commit', type_=str, default=None) if commit_msg is not None: wbia_rman.issue('git commit -am "{commit_msg}"'.format(**locals())) # Change Branch branch_name = GET_ARGVAL('--checkout', type_=str, default=None) if branch_name is not None: try: wbia_rman.issue('git checkout "{branch_name}"'.format(**locals())) except Exception: print('ERROR: Could not checkout branch: %r' % (branch_name, )) # Creates new branches newbranch_name = GET_ARGVAL('--newbranch', type_=str, default=None) if newbranch_name is not None: # rman.issue('git stash"'.format(**locals())) wbia_rman.issue( 'git checkout -b "{newbranch_name}"'.format(**locals())) wbia_rman.issue( 'git push --set-upstream origin {newbranch_name}'.format( **locals())) # rman.issue('git stash pop"'.format(**locals())) # Creates new branches newlocalbranch_name = GET_ARGVAL('--newlocalbranch', type_=str, default=None) if newlocalbranch_name is not None: # rman.issue('git stash"'.format(**locals())) wbia_rman.issue( 'git checkout -b "{newlocalbranch_name}"'.format(**locals())) # rman.issue('git push --set-upstream origin {newlocalbranch_name}'.format(**locals())) # rman.issue('git stash pop"'.format(**locals())) # Creates new branches mergebranch_name = GET_ARGVAL('--merge', type_=str, default=None) if mergebranch_name is not None: wbia_rman.issue('git merge "{mergebranch_name}"'.format(**locals())) # Change ownership if GET_ARGFLAG('--serverchmod'): wbia_rman.issue('chmod -R 755 *') if GET_ARGFLAG('--chown'): # Fixes problems where repos are checked out as root username = os.environ.get('USERNAME', ut.get_argval('--username')) if username is None: username = os.environ.get('USER', None) if username is None: raise AssertionError( 'cannot find username in commandline or environment vars') usergroup = username wbia_rman.issue('chown -R {username}:{usergroup} *'.format(**locals()), sudo=True) upstream_branch = GET_ARGVAL('--set-upstream', type_=str, default=None) if upstream_branch is not None: # git 2.0 wbia_rman.issue( 'git branch --set-upstream-to=origin/{upstream_branch} {upstream_branch}' .format(**locals())) upstream_push = GET_ARGVAL('--upstream-push', type_=str, default=None) if upstream_push is not None: wbia_rman.issue( 'git push --set-upstream origin {upstream_push}'.format( **locals())) if GET_ARGFLAG('--test'): failures = [] for repo_dpath in wbia_rman.repo_dirs: # ut.getp_ mod_dpaths = ut.get_submodules_from_dpath(repo_dpath, recursive=False, only_packages=True) modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths) print('Checking modules = %r' % (modname_list, )) for modname in modname_list: try: ut.import_modname(modname) print(modname + ' success') except ImportError: failures += [modname] print(modname + ' failure') print('failures = %s' % (ut.repr3(failures), )) if False: try: from six.moves import input except ImportError: input = raw_input # NOQA # General global git command gg_cmd = GET_ARGVAL('--gg', None) # global command if gg_cmd is not None: ans = ('yes' if GET_ARGFLAG('-y') else input( 'Are you sure you want to run: %r on all directories? ' % (gg_cmd, ))) if ans == 'yes': wbia_rman.issue(gg_cmd)
def python_develop(repo): import utool as ut repo.issue('{pythoncmd} setup.py develop'.format( pythoncmd=repo.pythoncmd), sudo=not ut.in_virtual_env())
def python_develop(repo): import utool as ut repo.issue('{pythoncmd} -m pip install -e {dpath}'.format( pythoncmd=repo.pythoncmd, dpath=repo.dpath), sudo=not ut.in_virtual_env())
def define_custom_scripts(tpl_rman): if ut.in_virtual_env(): fmtdict = { 'sys_dist_packages': ut.get_global_dist_packages_dir(), 'venv_site_packages': ut.get_site_packages_dir(), } # Allows us to use a system qt install in a virtual environment. tpl_rman['PyQt4'].add_script('system_to_venv', ut.codeblock( r""" # STARTBLOCK bash ln -s {sys_dist_packages}/PyQt4/ {venv_site_packages}/PyQt4 ln -s {sys_dist_packages}/sip*.so {venv_site_packages}/ ln -s {sys_dist_packages}/sip*.py {venv_site_packages}/ # ENDBLOCK bash """)).format(**fmtdict) # TODO: add custom build alternative pass else: pass ibeis_rman['pyflann'].add_script('install', ut.codeblock( r''' # STARTBLOCK bash # The pyflann source lives here cd {repo_dir}/src/python # But the setup script is generated during build python {repo_dir}/build/src/python/setup.py develop # ENDBLOCK bash ''').format(repo_dir=ibeis_rman['pyflann'].dpath) ) # TODO: allow system installation as well tpl_rman['cv2'].add_script('build', ut.codeblock( r""" # STARTBLOCK bash cd $CODE_DIR git clone https://github.com/Itseez/opencv.git cd opencv # Get Extras git clone https://github.com/Itseez/opencv_contrib.git mkdir -p build27 cd build27 if [[ "$VIRTUAL_ENV" == "" ]]; then export LOCAL_PREFIX=/usr/local export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/dist-packages else export LOCAL_PREFIX=$VIRTUAL_ENV/local export PYTHON2_PACKAGES_PATH=$LOCAL_PREFIX/lib/python2.7/site-packages fi echo "LOCAL_PREFIX = $LOCAL_PREFIX" echo "PYTHON2_PACKAGES_PATH = $PYTHON2_PACKAGES_PATH" # use dist packages on ubuntu. may need to change for other platforms cmake -G "Unix Makefiles" \ -D WITH_OPENMP=ON \ -D CMAKE_BUILD_TYPE=RELEASE \ -D PYTHON2_PACKAGES_PATH=$PYTHON2_PACKAGES_PATH \ -D CMAKE_INSTALL_PREFIX=$LOCAL_PREFIX \ -D OPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules \ .. export NCPUS=$(grep -c ^processor /proc/cpuinfo) make -j$NCPUS # ENDBLOCK """)) tpl_rman['cv2'].add_script('install', ut.codeblock( r""" # STARTBLOCK bash cd $CODE_DIR sudo make install # Hack because cv2 does not want to be installed for some reason cp lib/cv2.so $PYTHON2_PACKAGES_PATH # Test makesure things working python -c "import numpy; print(numpy.__file__)" python -c "import numpy; print(numpy.__version__)" python -c "import cv2; print(cv2.__version__)" python -c "import cv2; print(cv2.__file__)" #python -c "import vtool" # Check if we have contrib modules python -c "import cv2; print(cv2.xfeatures2d)" # ENDBLOCK """) )