def shutdown_wildbook_server(verbose=ut.NOT_QUIET): r""" Args: verbose (bool): verbosity flag(default = True) CommandLine: python -m ibeis.control.manual_wildbook_funcs --exec-shutdown_wildbook_server --exec-mode python -m ibeis --tf shutdown_wildbook_server Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> verbose = True >>> wb_url = shutdown_wildbook_server() >>> ut.quit_if_noshow() >>> ut.get_prefered_browser(PREFERED_BROWSER).open_new_tab(wb_url) """ # TODO: allow custom specified tomcat directory from os.path import join import time tomcat_dpath = find_installed_tomcat(check_unpacked=False) # Ensure environment variables #os.environ['JAVA_HOME'] = find_java_jvm() #os.environ['TOMCAT_HOME'] = tomcat_dpath #os.environ['CATALINA_HOME'] = tomcat_dpath with ut.ChdirContext(get_tomcat_startup_tmpdir()): shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') #ut.cmd(shutdown_fpath) ut.cmd(ut.quote_single_command(shutdown_fpath)) time.sleep(.5)
def startup_wildbook_server(verbose=ut.NOT_QUIET): r""" Args: verbose (bool): verbosity flag(default = True) CommandLine: python -m ibeis startup_wildbook_server python -m ibeis startup_wildbook_server --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.wildbook_manager import * # NOQA >>> verbose = True >>> wb_url = startup_wildbook_server() >>> ut.quit_if_noshow() >>> ut.get_prefered_browser(PREFERED_BROWSER).open_new_tab(wb_url) """ # TODO: allow custom specified tomcat directory import ibeis tomcat_dpath = find_installed_tomcat() with ut.ChdirContext(get_tomcat_startup_tmpdir()): startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') ut.cmd(ut.quote_single_command(startup_fpath)) time.sleep(1) wb_url = 'http://localhost:8080/' + ibeis.const.WILDBOOK_TARGET # TODO go to http://localhost:8080/ibeis/createAssetStore.jsp return wb_url
def run_suite_test(): app_fpath = get_dist_app_fpath() ut.assert_exists(app_fpath, 'app fpath must exist', info=True, verbose=True) ut.cmd(app_fpath + ' --run-utool-tests')
def monitor_mouse(): """ CommandLine: python -m utool.util_ubuntu monitor_mouse Example: >>> # SCRIPT >>> from utool.util_ubuntu import * # NOQA >>> import utool as ut >>> monitor_mouse() """ import utool as ut import re import parse mouse_ids = ut.cmd('xinput --list ', verbose=False, quiet=True)[0] x = mouse_ids.decode('utf-8') pattern = 'mouse' pattern = 'trackball' print(x) grepres = ut.greplines(x.split('\n'), pattern, reflags=re.IGNORECASE) mouse_id = parse.parse('{left}id={id}{right}', grepres[0][0])['id'] print('mouse_id = %r' % (mouse_id,)) import time while True: time.sleep(.2) out = ut.cmd('xinput --query-state ' + mouse_id, verbose=False, quiet=True)[0] print(out)
def shutdown_wildbook_server(verbose=ut.NOT_QUIET): r""" Args: verbose (bool): verbosity flag(default = True) Ignore: tail -f ~/.config/ibeis/tomcat/logs/catalina.out CommandLine: python -m ibeis shutdown_wildbook_server Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.wildbook_manager import * # NOQA >>> verbose = True >>> wb_url = shutdown_wildbook_server() >>> ut.quit_if_noshow() >>> ut.get_prefered_browser(PREFERED_BROWSER).open_new_tab(wb_url) """ # TODO: allow custom specified tomcat directory tomcat_dpath = find_installed_tomcat(check_unpacked=False, strict=False) # TODO: allow custom specified tomcat directory #tomcat_dpath = find_installed_tomcat(check_unpacked=False) #catalina_out_fpath = join(tomcat_dpath, 'logs', 'catalina.out') if tomcat_dpath is not None: with ut.ChdirContext(get_tomcat_startup_tmpdir()): shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') #ut.cmd(shutdown_fpath) ut.cmd(ut.quote_single_command(shutdown_fpath)) time.sleep(.5)
def ensure_inno_isinstalled(): """ Ensures that the current machine has INNO installed. returns path to the executable """ assert ut.WIN32, 'Can only build INNO on windows' inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs()) # Make sure INNO is installed if inno_fpath is None: print('WARNING: cannot find inno_fpath') AUTO_FIXIT = ut.WIN32 print('Inno seems to not be installed. AUTO_FIXIT=%r' % AUTO_FIXIT) if AUTO_FIXIT: print('Automaticaly trying to downoad and install INNO') # Download INNO Installer inno_installer_url = 'http://www.jrsoftware.org/download.php/ispack.exe' inno_installer_fpath = ut.download_url(inno_installer_url) print('Automaticaly trying to install INNO') # Install INNO Installer ut.cmd(inno_installer_fpath) else: inno_homepage_url = 'http://www.jrsoftware.org/isdl.php' ut.open_url_in_browser(inno_homepage_url) raise AssertionError('Cannot find INNO and AUTOFIX it is false') # Ensure that it has now been installed inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs()) assert ut.checkpath( inno_fpath, verbose=True, info=True), 'inno installer is still not installed!' return inno_fpath
def std_build_command(repo="."): """ DEPRICATE My standard build script names. Calls mingw_build.bat on windows and unix_build.sh on unix """ import utool as ut print("+**** stdbuild *******") print("repo = %r" % (repo,)) if sys.platform.startswith("win32"): # vtool --rebuild-sver didnt work with this line # scriptname = './mingw_build.bat' scriptname = "mingw_build.bat" else: scriptname = "./unix_build.sh" if repo == "": # default to cwd repo = "." else: os.chdir(repo) ut.assert_exists(scriptname) normbuild_flag = "--no-rmbuild" if ut.get_argflag(normbuild_flag): scriptname += " " + normbuild_flag # Execute build ut.cmd(scriptname) # os.system(scriptname) print("L**** stdbuild *******")
def main(): r""" python win32bootstrap.py --dl numpy --nocache python win32bootstrap.py --dl numpy-1.9.2rc1 --force python win32bootstrap.py --dl numpy-1.9.2rc1 --run python win32bootstrap.py --force python win32bootstrap.py --dryrun python win32bootstrap.py --dryrun --dl numpy scipy python win32bootstrap.py --dl numpy C:\Users\jon.crall\AppData\Roaming\utool\numpy-1.9.2rc1+mkl-cp27-none-win32.whl pip install C:/Users/jon.crall/AppData/Roaming/utool/numpy-1.9.2rc1+mkl-cp27-none-win32.whl """ # Packages that you are requesting pkg_list = [] if ut.get_argflag('--all'): pkg_list = KNOWN_PKG_LIST else: print('specify --all to download all packages') print('or specify --dl pkgname to download that package') pkg_list.extend(ut.get_argval('--dl', list, [])) dryrun = ut.get_argflag('--dryrun') pkg_exe_list = bootstrap_sysreq(pkg_list, dryrun=dryrun) if ut.get_argflag('--run'): for pkg_exe in pkg_exe_list: if pkg_exe.endswith('.whl'): ut.cmd('pip install ' + pkg_exe)
def exec_(script): import utool as ut print('+**** exec %s script *******' % (script.type_)) print('repo = %r' % (repo,)) with ut.ChdirContext(repo.dpath): if script.is_fpath_valid(): normbuild_flag = '--no-rmbuild' if ut.get_argflag(normbuild_flag): ut.cmd(script.fpath + ' ' + normbuild_flag) else: ut.cmd(script.fpath) else: if script.text is not None: print('ABOUT TO EXECUTE') ut.print_code(script.text, 'bash') if ut.are_you_sure('execute above script?'): from os.path import join scriptdir = ut.ensure_app_resource_dir('utool', 'build_scripts') script_path = join(scriptdir, 'script_' + script.type_ + '_' + ut.hashstr27(script.text) + '.sh') ut.writeto(script_path, script.text) _ = ut.cmd('bash ', script_path) # NOQA else: print("CANT QUITE EXECUTE THIS YET") ut.print_code(script.text, 'bash') #os.system(scriptname) print('L**** exec %s script *******' % (script.type_))
def std_build_command(repo='.'): """ DEPRICATE My standard build script names. Calls mingw_build.bat on windows and unix_build.sh on unix """ import utool as ut print('+**** stdbuild *******') print('repo = %r' % (repo,)) if sys.platform.startswith('win32'): # vtool --rebuild-sver didnt work with this line #scriptname = './mingw_build.bat' scriptname = 'mingw_build.bat' else: scriptname = './unix_build.sh' if repo == '': # default to cwd repo = '.' else: os.chdir(repo) ut.assert_exists(scriptname) normbuild_flag = '--no-rmbuild' if ut.get_argflag(normbuild_flag): scriptname += ' ' + normbuild_flag # Execute build ut.cmd(scriptname) #os.system(scriptname) print('L**** stdbuild *******')
def std_build_command(repo='.'): """ My standard build script names. Calls mingw_build.bat on windows and unix_build.sh on unix """ import utool as ut print("+**** stdbuild *******") print('repo = %r' % (repo,)) if sys.platform.startswith('win32'): #scriptname = './mingw_build.bat' # vtool --rebuild-sver didnt work with this line scriptname = 'mingw_build.bat' else: scriptname = './unix_build.sh' if repo == '': # default to cwd repo = '.' else: os.chdir(repo) ut.assert_exists(scriptname) normbuild_flag = '--no-rmbuild' if ut.get_argflag(normbuild_flag): scriptname += ' ' + normbuild_flag # Execute build ut.cmd(scriptname) #os.system(scriptname) print("L**** stdbuild *******")
def exec_(script): import utool as ut print("+**** exec %s script *******" % (script.type_)) print("repo = %r" % (repo,)) with ut.ChdirContext(repo.dpath): if script.is_fpath_valid(): normbuild_flag = "--no-rmbuild" if ut.get_argflag(normbuild_flag): ut.cmd(script.fpath + " " + normbuild_flag) else: ut.cmd(script.fpath) else: if script.text is not None: print("ABOUT TO EXECUTE") ut.print_code(script.text, "bash") if ut.are_you_sure("execute above script?"): from os.path import join scriptdir = ut.ensure_app_resource_dir("utool", "build_scripts") script_path = join( scriptdir, "script_" + script.type_ + "_" + ut.hashstr27(script.text) + ".sh" ) ut.writeto(script_path, script.text) _ = ut.cmd("bash ", script_path) # NOQA else: print("CANT QUITE EXECUTE THIS YET") ut.print_code(script.text, "bash") # os.system(scriptname) print("L**** exec %s script *******" % (script.type_))
def startup_wildbook_server(verbose=ut.NOT_QUIET): r""" Args: verbose (bool): verbosity flag(default = True) CommandLine: python -m ibeis.control.manual_wildbook_funcs --test-startup_wildbook_server python -m ibeis --tf startup_wildbook_server --show Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> verbose = True >>> wb_url = startup_wildbook_server() >>> ut.quit_if_noshow() >>> ut.get_prefered_browser(PREFERED_BROWSER).open_new_tab(wb_url) """ # TODO: allow custom specified tomcat directory from os.path import join import time import ibeis tomcat_dpath = find_installed_tomcat() # Ensure environment variables #os.environ['JAVA_HOME'] = find_java_jvm() #os.environ['TOMCAT_HOME'] = tomcat_dpath #os.environ['CATALINA_HOME'] = tomcat_dpath with ut.ChdirContext(get_tomcat_startup_tmpdir()): startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') ut.cmd(ut.quote_single_command(startup_fpath)) time.sleep(1) wb_url = 'http://localhost:8080/' + ibeis.const.WILDBOOK_TARGET return wb_url
def ensure_inno_isinstalled(): """ Ensures that the current machine has INNO installed. returns path to the executable """ assert ut.WIN32, 'Can only build INNO on windows' inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs()) # Make sure INNO is installed if inno_fpath is None: print('WARNING: cannot find inno_fpath') AUTO_FIXIT = ut.WIN32 print('Inno seems to not be installed. AUTO_FIXIT=%r' % AUTO_FIXIT) if AUTO_FIXIT: print('Automaticaly trying to downoad and install INNO') # Download INNO Installer inno_installer_url = 'http://www.jrsoftware.org/download.php/ispack.exe' inno_installer_fpath = ut.download_url(inno_installer_url) print('Automaticaly trying to install INNO') # Install INNO Installer ut.cmd(inno_installer_fpath) else: inno_homepage_url = 'http://www.jrsoftware.org/isdl.php' ut.open_url_in_browser(inno_homepage_url) raise AssertionError('Cannot find INNO and AUTOFIX it is false') # Ensure that it has now been installed inno_fpath = ut.search_in_dirs('Inno Setup 5\ISCC.exe', ut.get_install_dirs()) assert ut.checkpath(inno_fpath, verbose=True, info=True), 'inno installer is still not installed!' return inno_fpath
def autogen_ipynb(ibs, launch=None, run=None): r""" Autogenerates standard IBEIS Image Analysis IPython notebooks. CommandLine: python -m ibeis --tf autogen_ipynb --run --db lynx python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --asreport python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --noexample --withtags python -m ibeis --tf autogen_ipynb --db PZ_MTEST # TODO: Add support for dbdir to be specified python -m ibeis --tf autogen_ipynb --db ~/work/PZ_MTEST python -m ibeis --tf autogen_ipynb --ipynb --db Oxford -a default:qhas_any=\(query,\),dpername=1,exclude_reference=True,dminqual=good python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST -a default -t best:lnbnn_normalizer=[None,normlnbnn-test] python -m ibeis.templates.generate_notebook --exec-autogen_ipynb --db wd_peter_blinston --ipynb python -m ibeis --tf autogen_ipynb --db PZ_Master1 --ipynb python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl:qindex=0:100 -t best best:normsum=True --ipynb --noexample python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl --run jupyter-notebook Experiments-lynx.ipynb killall python python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW -a default:has_any=hasnotch python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW default:proot=vsmany -a default:has_any=hasnotch,mingt=2,qindex=0:50 --noexample Example: >>> # SCRIPT >>> from ibeis.templates.generate_notebook import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> result = autogen_ipynb(ibs) >>> print(result) """ dbname = ibs.get_dbname() fname = 'Experiments-' + dbname nb_fpath = fname + '.ipynb' if ut.get_argflag('--cells'): notebook_cells = make_ibeis_cell_list(ibs) print('\n# ---- \n'.join(notebook_cells)) return # TODO: Add support for dbdir to be specified notebook_str = make_ibeis_notebook(ibs) ut.writeto(nb_fpath, notebook_str) run = ut.get_argflag('--run') if run is None else run launch = launch if launch is not None else ut.get_argflag('--ipynb') if run: run_nb = ut.run_ipython_notebook(notebook_str) output_fpath = ut.export_notebook(run_nb, fname) ut.startfile(output_fpath) elif launch: ut.cmd('jupyter-notebook', nb_fpath, detatch=True) #ut.cmd('ipython-notebook', nb_fpath) #ut.startfile(nb_fpath) else: print('notebook_str =\n%s' % (notebook_str, ))
def autogen_ipynb(ibs, launch=None, run=None): r""" Autogenerates standard IBEIS Image Analysis IPython notebooks. CommandLine: python -m ibeis --tf autogen_ipynb --run --db lynx python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --asreport python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --noexample --withtags python -m ibeis --tf autogen_ipynb --db PZ_MTEST # TODO: Add support for dbdir to be specified python -m ibeis --tf autogen_ipynb --db ~/work/PZ_MTEST python -m ibeis --tf autogen_ipynb --ipynb --db Oxford -a default:qhas_any=\(query,\),dpername=1,exclude_reference=True,dminqual=good python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST -a default -t best:lnbnn_normalizer=[None,normlnbnn-test] python -m ibeis.templates.generate_notebook --exec-autogen_ipynb --db wd_peter_blinston --ipynb python -m ibeis --tf autogen_ipynb --db PZ_Master1 --ipynb python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl:qindex=0:100 -t best best:normsum=True --ipynb --noexample python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl --run jupyter-notebook Experiments-lynx.ipynb killall python python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW -a default:has_any=hasnotch python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW default:proot=vsmany -a default:has_any=hasnotch,mingt=2,qindex=0:50 --noexample Example: >>> # SCRIPT >>> from ibeis.templates.generate_notebook import * # NOQA >>> import ibeis >>> ibs = ibeis.opendb(defaultdb='testdb1') >>> result = autogen_ipynb(ibs) >>> print(result) """ dbname = ibs.get_dbname() fname = 'Experiments-' + dbname nb_fpath = fname + '.ipynb' if ut.get_argflag('--cells'): notebook_cells = make_ibeis_cell_list(ibs) print('\n# ---- \n'.join(notebook_cells)) return # TODO: Add support for dbdir to be specified notebook_str = make_ibeis_notebook(ibs) ut.writeto(nb_fpath, notebook_str) run = ut.get_argflag('--run') if run is None else run launch = launch if launch is not None else ut.get_argflag('--ipynb') if run: run_nb = ut.run_ipython_notebook(notebook_str) output_fpath = ut.export_notebook(run_nb, fname) ut.startfile(output_fpath) elif launch: ut.cmd('jupyter-notebook', nb_fpath, detatch=True) #ut.cmd('ipython-notebook', nb_fpath) #ut.startfile(nb_fpath) else: print('notebook_str =\n%s' % (notebook_str,))
def scp_pull(remote_path, local_path='.', remote='localhost', user=None): r""" wrapper for scp """ import utool as ut if user is not None: remote_uri = user + '@' + remote + ':' + remote_path else: remote_uri = remote + ':' + remote_path scp_exe = 'scp' scp_args = (scp_exe, '-r', remote_uri, local_path) ut.cmd(scp_args)
def ensure_nongit_plugins(): import utool as ut import REPOS1 BUNDLE_DPATH = util_git1.BUNDLE_DPATH for url in REPOS1.VIM_NONGIT_PLUGINS: fpath = ut.grab_zipped_url(url, download_dir=BUNDLE_DPATH) if fpath.endswith(".vba"): cmd_ = "vim " + fpath + ' -c "so % | q"' ut.cmd(cmd_) print("url = %r" % (url,)) pass
def tryimport(modname, pipiname=None, ensure=False): """ CommandLine: python -m utool.util_import --test-tryimport Example: >>> # ENABLE_DOCTEST >>> from utool.util_tests import * # NOQA >>> import utool as ut >>> modname = 'pyfiglet' >>> pipiname = 'git+https://github.com/pwaller/pyfiglet' >>> pyfiglet = ut.tryimport(modname, pipiname) >>> assert pyfiglet is None or isinstance(pyfiglet, types.ModuleType), 'unknown error' Example2: >>> # UNSTABLE_DOCTEST >>> # disabled because not everyone has access to being a super user >>> from utool.util_tests import * # NOQA >>> import utool as ut >>> modname = 'lru' >>> pipiname = 'git+https://github.com/amitdev/lru-dict' >>> lru = ut.tryimport(modname, pipiname, ensure=True) >>> assert isinstance(lru, types.ModuleType), 'did not ensure lru' """ if pipiname is None: pipiname = modname try: if util_inject.PRINT_INJECT_ORDER: if modname not in sys.modules: util_inject.noinject(modname, N=2, via='ut.tryimport') module = __import__(modname) return module except ImportError as ex: import utool as ut base_pipcmd = 'pip install %s' % pipiname sudo = not ut.WIN32 and not ut.in_virtual_env() if sudo: pipcmd = 'sudo ' + base_pipcmd else: pipcmd = base_pipcmd msg = 'unable to find module %s. Please install: %s' % ((modname), (pipcmd)) print(msg) ut.printex(ex, msg, iswarning=True) if ensure: raise AssertionError('Ensure is dangerous behavior and is is no longer supported.') #raise NotImplementedError('not ensuring') ut.cmd(base_pipcmd, sudo=sudo) module = tryimport(modname, pipiname, ensure=False) if module is None: raise AssertionError('Cannot ensure modname=%r please install using %r' % (modname, pipcmd)) return module return None
def build_win32_inno_installer(): inno_dir = r'C:\Program Files (x86)\Inno Setup 5' inno_fname = 'ISCC.exe' inno_fpath = join(inno_dir, inno_fname) cwd = get_setup_dpath() iss_script = join(cwd, '_installers', 'win_installer_script.iss') assert utool.checkpath(inno_fpath, verbose=True) assert utool.checkpath(iss_script, verbose=True) utool.cmd([inno_fpath, iss_script]) import shutil installer_src = join(cwd, '_installers', 'Output', 'ibeis-win32-setup.exe') installer_dst = join(cwd, 'dist', 'ibeis-win32-setup.exe') shutil.move(installer_src, installer_dst)
def ensure_nongit_plugins(): try: import utool as ut import REPOS1 BUNDLE_DPATH = util_git1.BUNDLE_DPATH for url in REPOS1.VIM_NONGIT_PLUGINS: fpath = ut.grab_zipped_url(url, download_dir=BUNDLE_DPATH) if fpath.endswith('.vba'): cmd_ = 'vim ' + fpath + ' -c "so % | q"' ut.cmd(cmd_) print('url = %r' % (url,)) except ImportError: print('Cant do nongit plugins without utool')
def focus_window(winhandle, path=None, name=None, sleeptime=.01): """ sudo apt-get install xautomation apt-get install autokey-gtk wmctrl -xa gnome-terminal.Gnome-terminal wmctrl -xl """ import utool as ut import time print('focus: ' + winhandle) args = ['wmctrl', '-xa', winhandle] ut.cmd(*args, verbose=False, quiet=True) time.sleep(sleeptime)
def install_cockatrice(): cockatrice_url = 'http://www.woogerworks.com/files/cockatrice.weeklybuilds/Cockatrice-WindowsClient.exe' import utool as ut fpath = ut.grab_file_url(cockatrice_url) # run setup script ut.cmd(fpath) # press enter a few times import win32com.client as w32 shell = w32.Dispatch("WScript.Shell") shell.AppActivate('Cockatrice Setup') shell.SendKeys("{ENTER}") shell.SendKeys("{ENTER}") shell.SendKeys("{ENTER}") shell.SendKeys("{ENTER}")
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath, )) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath, )) gs_exe = find_ghostscript_exe() cmd_list = (gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath) ut.cmd(*cmd_list) return output_pdf_fpath
def killold(pattern, num=4): """ Leaves no more than `num` instances of a program alive. Ordering is determined by most recent usage. CommandLine: python -m utool.util_ubuntu XCtrl.killold gvim 2 >>> import utool as ut >>> from utool import util_ubuntu >>> XCtrl = util_ubuntu.XCtrl >>> pattern = 'gvim' >>> num = 2 """ import utool as ut cmdkw = dict(verbose=False, quiet=True, silence=True) num = int(num) winid_list = XCtrl.findall_window_ids(pattern) winid_list = XCtrl.sort_window_ids(winid_list, 'mru')[num:] output_lines = ut.cmd( """wmctrl -lxp | awk '{print $1 " " $3}'""", **cmdkw)[0].strip().split('\n') output_fields = [line.split(' ') for line in output_lines] output_fields = [(int(wid, 16), int(pid)) for wid, pid in output_fields] pid_list = [pid for wid, pid in output_fields if wid in winid_list] import psutil for pid in pid_list: proc = psutil.Process(pid=pid) proc.kill()
def get_sys_thread_limit(): import utool as ut if ut.LINUX: out, err, ret = ut.cmd('ulimit', '-u', verbose=False, quiet=True, shell=True) else: raise NotImplementedError('')
def build_pyinstaller(): clean_pyinstaller() # Run the pyinstaller command (does all the work) utool.cmd('pyinstaller', '_installers/pyinstaller-ibeis.spec') # Perform some post processing steps on the mac if sys.platform == 'darwin' and exists("dist/IBEIS.app/Contents/"): copy_list = [ ('ibsicon.icns', 'Resources/icon-windowed.icns'), ('Info.plist', 'Info.plist'), ] srcdir = '_installers' dstdir = 'dist/IBEIS.app/Contents/' for srcname, dstname in copy_list: src = join(srcdir, srcname) dst = join(dstdir, dstname) utool.copy(src, dst) print("RUN: ./_installers/mac_dmg_builder.sh")
def run_app_test(): """ Execute the installed app """ print('[installer] +--- TEST_APP ---') app_fpath = get_dist_app_fpath() ut.assert_exists(app_fpath, 'app fpath must exist', info=True, verbose=True) if ut.DARWIN: #ut.cmd('open ' + ut.unixpath('dist/IBEIS.app')) """ rm -rf ~/Desktop/IBEIS.app rm -rf /Applications/IBEIS.app ls /Applications/IBEIS.app cd /Volumes/IBEIS ib cd dist # Install to /Applications hdiutil attach ~/code/ibeis/dist/IBEIS.dmg cp -R /Volumes/IBEIS/IBEIS.app /Applications/IBEIS.app hdiutil unmount /Volumes/IBEIS open -a /Applications/IBEIS.app chmod +x /Applications/IBEIS.app/Contents/MacOS/IBEISApp cp -R /Volumes/IBEIS/IBEIS.app ~/Desktop open -a ~/Desktop/IBEIS.app chmod +x ~/code/ibeis/dist/IBEIS.app/Contents/MacOS/IBEISApp open -a ~/code/ibeis/dist/IBEIS.app open ~/code/ibeis/dist/IBEIS.app/Contents/MacOS/IBEISApp open ~/Desktop/IBEIS.app ./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-tests """ ut.cmd(app_fpath) else: ut.cmd(app_fpath) print('[installer] L___ FINISH TEST_APP ___')
def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, 'cant sudo on windows' if command == 'short_status': return repo.short_status() command_list = ut.ensure_iterable(command) cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) if not dry: print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ut.color_text(repo.dpath, 'yellow')) verbose = True with repo.chdir_context(): ret = None for count, cmd in enumerate(command_list): if dry: print(cmd) continue if not sudo or ut.WIN32: # ret = os.system(cmd) cmdinfo = ut.cmd2(cmd, verbout=True) out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret']) else: # cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1) out, err, ret = ut.cmd(cmd, sudo=True) if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: if error == 'raise': raise Exception('Failed command %r' % (cmd,)) elif error == 'return': return out else: raise ValueError('unknown flag error=%r' % (error,)) if return_out: return out if not dry: print('L____')
def get_commit_date(hashid): out, err, ret = ut.cmd('git show -s --format=%ci ' + hashid, verbose=False, quiet=True, pad_stdout=False) # from datetime import datetime from dateutil import parser # print('out = %r' % (out,)) stamp = out.strip('\n') # print('stamp = %r' % (stamp,)) dt = parser.parse(stamp) # dt = datetime.strptime(stamp, '%Y-%m-%d %H:%M:%S %Z') # print('dt = %r' % (dt,)) return dt
def test_run_app(): """ Execute the installed app """ print('[installer] +--- TEST_APP ---') app_fpath = get_dist_app_fpath() ut.assert_exists(app_fpath, 'app fpath must exist', info=True, verbose=True) if ut.DARWIN: #ut.cmd('open ' + ut.unixpath('dist/IBEIS.app')) """ rm -rf ~/Desktop/IBEIS.app rm -rf /Applications/IBEIS.app ls /Applications/IBEIS.app cd /Volumes/IBEIS ib cd dist # Install to /Applications hdiutil attach ~/code/ibeis/dist/IBEIS.dmg cp -R /Volumes/IBEIS/IBEIS.app /Applications/IBEIS.app hdiutil unmount /Volumes/IBEIS open -a /Applications/IBEIS.app chmod +x /Applications/IBEIS.app/Contents/MacOS/IBEISApp cp -R /Volumes/IBEIS/IBEIS.app ~/Desktop open -a ~/Desktop/IBEIS.app chmod +x ~/code/ibeis/dist/IBEIS.app/Contents/MacOS/IBEISApp open -a ~/code/ibeis/dist/IBEIS.app open ~/code/ibeis/dist/IBEIS.app/Contents/MacOS/IBEISApp open ~/Desktop/IBEIS.app ./dist/IBEIS.app/Contents/MacOS/IBEISApp --run-tests """ ut.cmd(app_fpath) else: ut.cmd(app_fpath) print('[installer] L___ FINISH TEST_APP ___')
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath,)) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath,)) gs_exe = find_ghostscript_exe() cmd_list = ( gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath ) ut.cmd(*cmd_list) return output_pdf_fpath
def check_installed_debian(pkgname): """ References: http://www.cyberciti.biz/faq/find-out-if-package-is-installed-in-linux/ """ import utool as ut #pkgname = 'espeak' #pkgname = 'sudo' #ut.cmd('hash ' + pkgname + ' 2>/dev/null') tup = ut.cmd('hash ' + pkgname + ' 2>/dev/null', quiet=True, pad_stdout=False) out, err, ret = tup is_installed = (ret == 0) return is_installed
def build_win32_inno_installer(): """ win32 self-executable package """ print('[installer] +--- BUILD_WIN32_INNO_INSTALLER ---') assert ut.WIN32, 'Can only build INNO on windows' # Get inno executable inno_fpath = ensure_inno_isinstalled() # Get IBEIS inno script iss_script_fpath = ensure_inno_script() print('Trying to run ' + ' '.join(['"' + inno_fpath + '"', '"' + iss_script_fpath + '"'])) try: command_args = ' '.join((inno_fpath, iss_script_fpath)) ut.cmd(command_args) except Exception as ex: ut.printex(ex, 'error running script') raise # Move the installer into dist and make a timestamped version # Uninstall exe in case we need to cleanup #uninstall_ibeis_exe = 'unins000.exe' cwd = get_setup_dpath() installer_fpath = join(cwd, '_installers', 'Output', 'ibeis-win32-setup.exe') print('[installer] L___ BUILD_WIN32_INNO_INSTALLER ___') return installer_fpath
def rsync(src_uri, dst_uri, exclude_dirs=[], port=22, dryrun=False): """ General function to push or pull a directory from a remote server to a local path References: http://www.tecmint.com/rsync-local-remote-file-synchronization-commands/ http://serverfault.com/questions/219013/showing-total-progress-in-rsync-is-it-possible Notes (rsync commandline options): rsync [OPTION]... SRC [SRC]... DEST -v : verbose -r : copies data recursively (but dont preserve timestamps and permission while transferring data -a : archive mode, allows recursive copying and preserves symlinks, permissions, user and group ownerships, and timestamps -z : compress file data -i, --itemize-changes output a change-summary for all updates -s, --protect-args : no space-splitting; only wildcard special-chars -h : human-readable, output numbers in a human-readable format -P same as --partial --progress """ rsync_exe = 'rsync' rsync_options = '-avhzP' #rsync_options += ' --port=%d' % (port,) rsync_options += ' -e "ssh -p %d"' % (port,) if len(exclude_dirs) > 0: exclude_tup = ['--exclude ' + dir_ for dir_ in exclude_dirs] exclude_opts = ' '.join(exclude_tup) rsync_options += ' ' + exclude_opts cmdtuple = (rsync_exe, rsync_options, src_uri, dst_uri) cmdstr = ' '.join(cmdtuple) print('[rsync] src_uri = %r ' % (src_uri,)) print('[rsync] dst_uri = %r ' % (dst_uri,)) print('[rsync] cmdstr = %r' % cmdstr) print(cmdstr) #if not dryrun: ut.cmd(cmdstr, dryrun=dryrun)
def list_remote(remote_uri, verbose=False): """ remote_uri = '*****@*****.**' """ remote_uri1, remote_dpath = remote_uri.split(':') if not remote_dpath: remote_dpath = '.' import utool as ut out = ut.cmd('ssh', remote_uri1, 'ls -l %s' % (remote_dpath,), verbose=verbose) import re # Find lines that look like ls output split_lines = [re.split(r'\s+', t) for t in out[0].split('\n')] paths = [' '.join(t2[8:]) for t2 in split_lines if len(t2) > 8] return paths
def monitor_wildbook_logs(verbose=ut.NOT_QUIET): r""" Args: verbose (bool): verbosity flag(default = True) CommandLine: python -m ibeis monitor_wildbook_logs --show Example: >>> # SCRIPT >>> from ibeis.control.wildbook_manager import * # NOQA >>> monitor_wildbook_logs() """ # TODO: allow custom specified tomcat directory import ibeis tomcat_dpath = find_installed_tomcat() with ut.ChdirContext(get_tomcat_startup_tmpdir()): startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') ut.cmd(ut.quote_single_command(startup_fpath)) time.sleep(1) wb_url = 'http://localhost:8080/' + ibeis.const.WILDBOOK_TARGET return wb_url
def build_pyinstaller(): """ build_pyinstaller creates build/ibeis/* and dist/ibeis/* """ print('[installer] +--- BUILD_PYINSTALLER ---') # 1) RUN: PYINSTALLER # Run the pyinstaller command (does all the work) utool_python_path = dirname(dirname(ut.__file__)) #import os #os.environ['PYTHONPATH'] = os.pathsep.join([utool_python_path] + os.environ['PYTHONPATH'].strip(os.pathsep).split(os.pathsep)) import os sys.path.insert(1, utool_python_path) if not ut.WIN32: pathcmd = 'export PYTHONPATH=%s%s$PYTHONPATH && ' % (utool_python_path, os.pathsep) else: pathcmd = '' installcmd = '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/bin/pyinstaller --runtime-hook _installers/rthook_pyqt4.py _installers/pyinstaller-ibeis.spec -y' output, err, ret = ut.cmd(pathcmd + installcmd) if ret != 0: raise AssertionError('Pyinstalled failed with return code = %r' % (ret, )) #ut.cmd(installcmd) #ut.cmd('pyinstaller --runtime-hook rthook_pyqt4.py _installers/pyinstaller-ibeis.spec -y') #else: #ut.cmd('pyinstaller', '_installers/pyinstaller-ibeis.spec', '-y') #ut.cmd('pyinstaller', '--runtime-hook rthook_pyqt4.py', '_installers/pyinstaller-ibeis.spec') # 2) POST: PROCESSING # Perform some post processing steps on the mac if sys.platform == 'darwin' and exists('dist/IBEIS.app/Contents/'): copy_list = [ ('ibsicon.icns', 'Resources/icon-windowed.icns'), ('Info.plist', 'Info.plist'), ] srcdir = '_installers' dstdir = 'dist/IBEIS.app/Contents/' for srcname, dstname in copy_list: src = join(srcdir, srcname) dst = join(dstdir, dstname) ut.copy(src, dst) # TODO: make this take arguments instead of defaulting to ~/code/ibeis/build #print("RUN: sudo ./_installers/mac_dmg_builder.sh") app_fpath = get_dist_app_fpath() print('[installer] app_fpath = %s' % (app_fpath, )) print('[installer] L___ FINISH BUILD_PYINSTALLER ___')
def do(*cmd_list, **kwargs): import utool as ut import time verbose = False # print('Running xctrl.do script') if verbose: print('Executing x do: %r' % (cmd_list,)) cmdkw = dict(verbose=False, quiet=True, silence=True) # http://askubuntu.com/questions/455762/xbindkeys-wont-work-properly # Make things work even if other keys are pressed defaultsleep = 0.0 sleeptime = kwargs.get('sleeptime', defaultsleep) time.sleep(.05) ut.cmd('xset r off', **cmdkw) for item in cmd_list: # print('item = %r' % (item,)) sleeptime = kwargs.get('sleeptime', defaultsleep) assert isinstance(item, tuple) assert len(item) >= 2 xcmd, key_ = item[0:2] if len(item) >= 3: sleeptime = float(item[2]) if xcmd == 'focus': args = ['wmctrl', '-xa', str(key_)] elif xcmd == 'type': args = [ 'xdotool', 'keyup', '--window', '0', '7', 'type', '--clearmodifiers', '--window', '0', str(key_) ] else: args = ['xdotool', str(xcmd), str(key_)] if verbose: print('args = %r' % (args,)) # print('args = %r' % (args,)) ut.cmd(*args, **cmdkw) if sleeptime > 0: time.sleep(sleeptime) ut.cmd('xset r on', verbose=False, quiet=True, silence=True)
def issue(repo, command, sudo=False): """ issues a command on a repo CommandLine: python -m utool.util_git --exec-repocmd Example: >>> # DISABLE_DOCTEST >>> from utool.util_git import * # NOQA >>> import utool as ut >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True)) >>> command = 'git status' >>> sudo = False >>> result = repocmd(repo, command, sudo) >>> print(result) """ import utool as ut if ut.WIN32: assert not sudo, 'cant sudo on windows' command_list = ut.ensure_iterable(command) cmdstr = '\n '.join([cmd_ for cmd_ in command_list]) print('+--- *** repocmd(%s) *** ' % (cmdstr,)) print('repo=%s' % ut.color_text(repo.dpath, 'yellow')) with ut.ChdirContext(repo.dpath, verbose=False): ret = None for count, cmd in enumerate(command_list): if not sudo or ut.WIN32: ret = os.system(cmd) else: out, err, ret = ut.cmd(cmd, sudo=True) verbose = True if verbose > 1: print('ret(%d) = %r' % (count, ret,)) if ret != 0: raise Exception('Failed command %r' % (cmd,)) print('L____')
def clean_mitex(): # if mitex does not install correctly install_dir = 'C:\Program Files (x86)\MiKTeX 2.9' ut.delete(install_dir) def install_mathtools(): """ wget http://mirrors.ctan.org/install/macros/latex/contrib/mathtools.tds.zip mkdir tmp2 #7z x -o"tmp2" mathtools.tds.zip 7z x -o"C:/Program Files (x86)/MiKTeX 2.9" mathtools.tds.zip """ pass if __name__ == '__main__': """ python %USERPROFILE%/local/windows/init_mitex.py """ assert ut.WIN32 url = 'http://mirrors.ctan.org/systems/win32/miktex/setup/basic-miktex-2.9.5105.exe' fpath = ut.grab_file_url(url) ut.cmd(fpath)
def install_wildbook(verbose=ut.NOT_QUIET): """ Script to setup wildbook on a unix based system (hopefully eventually this will generalize to win32) CommandLine: # Reset ibeis purge_local_wildbook ibeis ensure_wb_mysql ibeis ensure_local_war # Setup ibeis install_wildbook # ibeis install_wildbook --nomysql # Startup ibeis startup_wildbook_server --show Alternates: ibeis install_wildbook --redownload-war ibeis install_wildbook --assets ibeis startup_wildbook_server --show Example: >>> # SCRIPT >>> from ibeis.control.wildbook_manager import * # NOQA >>> verbose = True >>> result = install_wildbook() >>> print(result) """ import requests # Ensure that the war file has been unpacked tomcat_dpath, webapps_dpath, wb_target = ensure_local_war() unpacked_war_dpath = join(webapps_dpath, wb_target) tomcat_startup_dir = get_tomcat_startup_tmpdir() fresh_install = not ut.checkpath(unpacked_war_dpath, verbose=verbose) if fresh_install: # Need to make sure you start catalina in the same directory otherwise # the derby databsae gets put in in cwd with ut.ChdirContext(tomcat_startup_dir): # Starting and stoping catalina should be sufficient to unpack the # war startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') #shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') ut.cmd(ut.quote_single_command(startup_fpath)) print('It is NOT ok if the startup.sh fails\n') # wait for the war to be unpacked for retry_count in range(0, 6): time.sleep(1) if ut.checkpath(unpacked_war_dpath, verbose=True): break else: print('Retrying') # ensure that the server is ruuning print('Checking if we can ping the server') response = requests.get('http://localhost:8080') if response is None or response.status_code != 200: print('There may be an error starting the server') else: print('Seem able to ping the server') # assert tht the war was unpacked ut.assertpath(unpacked_war_dpath, ( 'Wildbook war might have not unpacked correctly. This may ' 'be ok. Try again. If it fails a second time, then there is a ' 'problem.'), verbose=True) # Don't shutdown just yet. Need to create assets update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) asset_flag_fpath = join(tomcat_startup_dir, 'made_assets.flag') # Pinging the server to create asset store # Ensureing that createAssetStore exists if not ut.checkpath(asset_flag_fpath): if not fresh_install: startup_wildbook_server() #web_url = startup_wildbook_server(verbose=False) print('Creating asset store') wb_url = 'http://localhost:8080/' + wb_target response = requests.get(wb_url + '/createAssetStore.jsp') if response is None or response.status_code != 200: print('There may be an error starting the server') #if response.status_code == 500: print(response.text) assert False, 'response error' else: print('Created asset store') # Create file signaling we did this ut.writeto(asset_flag_fpath, 'True') shutdown_wildbook_server(verbose=False) print('It is ok if the shutdown fails') elif fresh_install: shutdown_wildbook_server(verbose=False) #127.0.0.1:8080/wildbook_data_dir/test.txt print('Wildbook is installed and waiting to be started')
def fix_empty_dirs(drive): """ # --- FIND EMPTY DIRECTORIES --- """ print('Fixing Empty Dirs in %r' % (drive, )) fidxs_list = ut.dict_take(drive.dpath_to_fidx, drive.dpath_list) isempty_flags = [len(fidxs) == 0 for fidxs in fidxs_list] empty_dpaths = ut.compress(drive.dpath_list, isempty_flags) def is_cplat_link(path_): try: if islink(path_): return True os.listdir(d) return False except SystemErrors: return True valid_flags = [not is_cplat_link(d) for d in empty_dpaths] if not all(valid_flags): print('Filtered windows links %r / %r' % (len(empty_dpaths) - sum(valid_flags), len(empty_dpaths))) #print(ut.repr2(empty_dpaths[0:10])) empty_dpaths = ut.compress(empty_dpaths, valid_flags) print('Found %r / %r empty_dpaths' % (len(empty_dpaths), len(drive.dpath_list))) print(ut.repr2(empty_dpaths[0:10])) # Ensure actually still empty current_contents = [ ut.glob(d, with_dirs=False) for d in ut.ProgIter(empty_dpaths, 'checking empty status') ] current_lens = list(map(len, current_contents)) assert not any(current_lens), 'some dirs are not empty' # n ** 2 check to get only the base directories isbase_dir = [ not any([ d.startswith(dpath_) and d != dpath_ for dpath_ in empty_dpaths ]) for d in ut.ProgIter(empty_dpaths, 'finding base dirs') ] base_empty_dirs = ut.compress(empty_dpaths, isbase_dir) def list_only_files(dpath): # glob is too slow for root, dirs, fpaths in os.walk(dpath): for fpath in fpaths: yield fpath base_current_contents = [ list(list_only_files(d)) for d in ut.ProgIter( base_empty_dirs, 'checking emptyness', freq=10) ] is_actually_empty = [len(fs) == 0 for fs in base_current_contents] not_really_empty = ut.compress(base_empty_dirs, ut.not_list(is_actually_empty)) print('%d dirs are not actually empty' % (len(not_really_empty), )) print('not_really_empty = %s' % (ut.repr2(not_really_empty[0:10]), )) truly_empty_dirs = ut.compress(base_empty_dirs, is_actually_empty) def list_all(dpath): # glob is too slow for root, dirs, fpaths in os.walk(dpath): for dir_ in dirs: yield dir_ for fpath in fpaths: yield fpath exclude_base_dirs = [join(drive.root_dpath, 'AppData')] exclude_end_dirs = ['__pycache__'] truly_empty_dirs1 = truly_empty_dirs for ed in exclude_base_dirs: truly_empty_dirs1 = [ d for d in truly_empty_dirs1 if (not any(d.startswith(ed) for ed in exclude_base_dirs) and not any( d.endswith(ed) for ed in exclude_end_dirs)) ] # Ensure actually still empty (with recursive checks for hidden files) print('truly_empty_dirs1[::5] = %s' % (ut.repr2(truly_empty_dirs1[0::5], strvals=True), )) #print('truly_empty_dirs1 = %s' % (ut.repr2(truly_empty_dirs1, strvals=True),)) if not dryrun: # FIX PART #from os.path import normpath #for d in ut.ProgIter(truly_empty_dirs): # break # if ut.WIN32: # # http://www.sevenforums.com/system-security/53095-file-folder-read-only-attribute-wont-disable.html # ut.cmd('attrib', '-r', '-s', normpath(d), verbose=False) #x = ut.remove_fpaths(truly_empty_dirs, strict=False) print('Deleting %d truly_empty_dirs1' % (len(truly_empty_dirs1), )) for d in ut.ProgIter(truly_empty_dirs1, 'DELETE empty dirs', freq=1000): # NOQA ut.delete(d, quiet=True) if ut.WIN32 and False: # remove file that failed removing flags = list(map(exists, truly_empty_dirs1)) truly_empty_dirs1 = ut.compress(truly_empty_dirs1, flags) for d in ut.ProgIter(truly_empty_dirs1, 'rming', freq=1000): ut.cmd('rmdir', d)
def execute_commands(tpl_rman, wbia_rman): import utool as ut GET_ARGVAL = ut.get_argval ut.init_catch_ctrl_c() if 0: print('Version Check Source:') for repo in tpl_rman.repos: print('python -c "import {0}; print({0}.__file__)"'.format( repo.modname)) print('python -c "import {0}; print({0}.__version__)"'.format( repo.modname)) # ----------- # Execute Commands on Core Repos # ----------- CODE_DIR, pythoncmd, WIN32, PY2, PY3 = get_sysinfo() print('wbia_rman = %r' % (wbia_rman, )) wildme_ssh_flags = GET_ARGFLAG('--move-wildme') or GET_ARGFLAG( '--move-wildme-ssh') wildme_https_flags = GET_ARGFLAG('--move-wildme-https') or GET_ARGFLAG( '--move-wildme-http') if wildme_ssh_flags or wildme_https_flags: fmt = 'ssh' if wildme_ssh_flags else 'https' move_wildme(wbia_rman, fmt) # Commands on global git repos if GET_ARGFLAG('--status'): wbia_rman.issue('git status') sys.exit(0) wbia_rman.ensure() if GET_ARGFLAG('--dump') or GET_ARGFLAG('--dump-scripts'): dpath = '_super_scripts/' + 'scripts' + get_plat_specifier() ut.ensuredir(dpath) dumps = [ (tpl_rman, 'cv2', 'build'), (tpl_rman, 'cv2', 'install'), (wbia_rman, 'flann', 'build'), (wbia_rman, 'flann', 'install'), (wbia_rman, 'hesaff', 'build'), (tpl_rman, 'PyQt', 'system_to_venv'), (tpl_rman, 'libgpuarray', 'build'), ] for rman, mod, sname in dumps: from os.path import join # if mod not in rman: # print('mod=%r not available in rman=%r' % (mod, rman)) # continue script = rman[mod].get_script(sname).text suffix = get_plat_specifier() sh_fpath = join(dpath, mod + '_' + sname + suffix + '.sh') ut.write_to(sh_fpath, script) if GET_ARGFLAG('--requirements'): ut.cmd('pip install -r requirements.txt') # HACKED IN SCRIPTS WHILE IM STILL FIGURING OUT TPL DEPS if GET_ARGFLAG('--opencv'): # There is now a pypi for opencv! Yay # ut.cmd('pip install opencv-python') # Bummer, but we need opencv source for pyhessaff # we should just make a wheel for pyhessaff cv_repo = tpl_rman['cv2'] cv_repo.clone() script = cv_repo.get_script('build') script.exec_() cv_repo = tpl_rman['cv2'] script = cv_repo.get_script('install') script.exec_() if GET_ARGFLAG('--flann'): script = wbia_rman['flann'].get_script('build') script.exec_() script = wbia_rman['flann'].get_script('install') script.exec_() if GET_ARGFLAG('--pyqt'): script = tpl_rman['PyQt'].get_script('system_to_venv') script.exec_() if GET_ARGFLAG('--hesaff'): script = wbia_rman['hesaff'].get_script('build') script.exec_() if GET_ARGFLAG('--pydarknet'): script = wbia_rman['pydarknet'].get_script('build') script.exec_() if GET_ARGFLAG('--pyrf'): script = wbia_rman['pyrf'].get_script('build') script.exec_() if GET_ARGFLAG('--torch'): # Theano and lasange code should be moved to pytorch tpl_rman['pytorch'].clone(recursive=True) tpl_rman['pytorch'].issue('git submodule update --init') tpl_rman['pytorch'].issue('python setup install') tpl_rman['pytorch'].issue('pip install torchvision') # tpl_rman['pytorch'].issue('NO_CUDNN=TRUE && python setup install') # tpl_rman['pytorch'].issue('pip install -e .') if GET_ARGFLAG('--libgpuarray') or GET_ARGFLAG('--dcnn'): tpl_rman['libgpuarray'].clone() script = tpl_rman['libgpuarray'].get_script('build') script.exec_() if GET_ARGFLAG('--dcnn'): tpl_rman['theano'].clone() # tpl_rman['pylearn2'].clone() tpl_rman['lasagne'].clone() tpl_rman['theano'].issue('pip install -e .') # tpl_rman['pylearn2'].issue('pip install -e .') tpl_rman['lasagne'].issue('pip install -e .') # tpl_rman['pylearn2'].python_develop() # tpl_rman['theano'].python_develop() # tpl_rman['lasagne'].python_develop() # _=== if GET_ARGFLAG('--fix') or GET_ARGFLAG('--check'): missing_dynlib = tpl_rman.check_cpp_build() missing_dynlib += wbia_rman.check_cpp_build() missing_install = tpl_rman.check_installed() missing_install += wbia_rman.check_installed() problems = [] problems += wbia_rman.check_importable() problems += tpl_rman.check_importable() if GET_ARGFLAG('--fix'): print('Trying to fix problems') for repo in missing_dynlib: repo.custom_build() for repo, recommended_fix in problems: print('Trying to fix repo = %r' % (repo, )) print(' * recommended_fix = %r' % (recommended_fix, )) if recommended_fix == 'rebuild': repo.custom_build() print( 'Can currently only fix one module at a time. Please re-run' ) sys.exit(1) else: print('Not sure how to fix %r' % (repo, )) if GET_ARGFLAG('--pull'): wbia_rman.issue('git pull') if GET_ARGFLAG('--build'): # Build tpl repos # tpl_rman.custom_build() # wbia_rman.custom_build() # Build only IBEIS repos with setup.py _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py build'.format(pythoncmd=pythoncmd)) # Like install, but better if you are developing if GET_ARGFLAG('--develop'): _rman = wbia_rman.only_with_pysetup() # # _rman.issue('{pythoncmd} setup.py develop'.format(pythoncmd=pythoncmd), # # sudo=not ut.in_virtual_env()) _rman.issue( '{pythoncmd} -m pip install -e .'.format(pythoncmd=pythoncmd), sudo=not ut.in_virtual_env(), ) if GET_ARGFLAG('--clean'): _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py clean'.format(pythoncmd=pythoncmd)) if GET_ARGFLAG('--install'): print( 'WARNING: Dont use install if you are a developer. Use develop instead.' ) _rman = wbia_rman.only_with_pysetup() _rman.issue('{pythoncmd} setup.py install'.format(pythoncmd=pythoncmd)) if GET_ARGFLAG('--push'): wbia_rman.issue('git push') if GET_ARGFLAG('--branch'): wbia_rman.issue('git branch') sys.exit(0) if GET_ARGFLAG('--tag-status'): wbia_rman.issue('git tag') # Tag everything tag_name = GET_ARGVAL('--newtag', type_=str, default=None) if tag_name is not None: wbia_rman.issue( 'git tag -a "{tag_name}" -m "super_setup autotag {tag_name}"'. format(**locals())) wbia_rman.issue('git push --tags') if GET_ARGFLAG('--bext'): wbia_rman.issue('{pythoncmd} setup.py build_ext --inplace'.format( pythoncmd=pythoncmd)) commit_msg = GET_ARGVAL('--commit', type_=str, default=None) if commit_msg is not None: wbia_rman.issue('git commit -am "{commit_msg}"'.format(**locals())) # Change Branch branch_name = GET_ARGVAL('--checkout', type_=str, default=None) if branch_name is not None: try: wbia_rman.issue('git checkout "{branch_name}"'.format(**locals())) except Exception: print('ERROR: Could not checkout branch: %r' % (branch_name, )) # Creates new branches newbranch_name = GET_ARGVAL('--newbranch', type_=str, default=None) if newbranch_name is not None: # rman.issue('git stash"'.format(**locals())) wbia_rman.issue( 'git checkout -b "{newbranch_name}"'.format(**locals())) wbia_rman.issue( 'git push --set-upstream origin {newbranch_name}'.format( **locals())) # rman.issue('git stash pop"'.format(**locals())) # Creates new branches newlocalbranch_name = GET_ARGVAL('--newlocalbranch', type_=str, default=None) if newlocalbranch_name is not None: # rman.issue('git stash"'.format(**locals())) wbia_rman.issue( 'git checkout -b "{newlocalbranch_name}"'.format(**locals())) # rman.issue('git push --set-upstream origin {newlocalbranch_name}'.format(**locals())) # rman.issue('git stash pop"'.format(**locals())) # Creates new branches mergebranch_name = GET_ARGVAL('--merge', type_=str, default=None) if mergebranch_name is not None: wbia_rman.issue('git merge "{mergebranch_name}"'.format(**locals())) # Change ownership if GET_ARGFLAG('--serverchmod'): wbia_rman.issue('chmod -R 755 *') if GET_ARGFLAG('--chown'): # Fixes problems where repos are checked out as root username = os.environ.get('USERNAME', ut.get_argval('--username')) if username is None: username = os.environ.get('USER', None) if username is None: raise AssertionError( 'cannot find username in commandline or environment vars') usergroup = username wbia_rman.issue('chown -R {username}:{usergroup} *'.format(**locals()), sudo=True) upstream_branch = GET_ARGVAL('--set-upstream', type_=str, default=None) if upstream_branch is not None: # git 2.0 wbia_rman.issue( 'git branch --set-upstream-to=origin/{upstream_branch} {upstream_branch}' .format(**locals())) upstream_push = GET_ARGVAL('--upstream-push', type_=str, default=None) if upstream_push is not None: wbia_rman.issue( 'git push --set-upstream origin {upstream_push}'.format( **locals())) if GET_ARGFLAG('--test'): failures = [] for repo_dpath in wbia_rman.repo_dirs: # ut.getp_ mod_dpaths = ut.get_submodules_from_dpath(repo_dpath, recursive=False, only_packages=True) modname_list = ut.lmap(ut.get_modname_from_modpath, mod_dpaths) print('Checking modules = %r' % (modname_list, )) for modname in modname_list: try: ut.import_modname(modname) print(modname + ' success') except ImportError: failures += [modname] print(modname + ' failure') print('failures = %s' % (ut.repr3(failures), )) if False: try: from six.moves import input except ImportError: input = raw_input # NOQA # General global git command gg_cmd = GET_ARGVAL('--gg', None) # global command if gg_cmd is not None: ans = ('yes' if GET_ARGFLAG('-y') else input( 'Are you sure you want to run: %r on all directories? ' % (gg_cmd, ))) if ans == 'yes': wbia_rman.issue(gg_cmd)
def build_osx_dmg_installer(): # outputs dmg to ut.cmd('./_installers/mac_dmg_builder.sh', sudo=True) cwd = get_setup_dpath() installer_fpath = join(cwd, 'dist', 'IBEIS.dmg') return installer_fpath
def autogen_sphinx_apidoc(): r""" autogen_sphinx_docs.py Ignore: C:\Python27\Scripts\autogen_sphinx_docs.py autogen_sphinx_docs.py pip uninstall sphinx pip install sphinx pip install sphinxcontrib-napoleon pip install sphinx --upgrade pip install sphinxcontrib-napoleon --upgrade cd C:\Python27\Scripts ls C:\Python27\Scripts python -c "import sphinx; print(sphinx.__version__)" CommandLine: python -m utool.util_setup --exec-autogen_sphinx_apidoc Example: >>> # SCRIPT >>> from utool.util_setup import * # NOQA >>> autogen_sphinx_apidoc() """ # TODO: assert sphinx-apidoc exe is found # TODO: make find_exe work? import utool as ut def build_sphinx_apidoc_cmdstr(): print('') print('if this fails try: sudo pip install sphinx') print('') apidoc = 'sphinx-apidoc' if ut.WIN32: winprefix = 'C:/Python27/Scripts/' sphinx_apidoc_exe = winprefix + apidoc + '.exe' else: sphinx_apidoc_exe = apidoc apidoc_argfmt_list = [ sphinx_apidoc_exe, '--force', '--full', '--maxdepth="{maxdepth}"', '--doc-author="{author}"', '--doc-version="{doc_version}"', '--doc-release="{doc_release}"', '--output-dir="_doc"', #'--separate', # Put documentation for each module on its own page '--private', # Include "_private" modules '{pkgdir}', ] outputdir = '_doc' author = ut.parse_author() packages = ut.find_packages(maxdepth=1) assert len( packages) != 0, 'directory must contain at least one package' if len(packages) > 1: assert len(packages) == 1,\ ('FIXME I dont know what to do with more than one root package: %r' % (packages,)) pkgdir = packages[0] version = ut.parse_package_for_version(pkgdir) modpath = dirname(ut.truepath(pkgdir)) apidoc_fmtdict = { 'author': author, 'maxdepth': '8', 'pkgdir': pkgdir, 'doc_version': version, 'doc_release': version, 'outputdir': outputdir, } ut.assert_exists('setup.py') ut.ensuredir('_doc') apidoc_fmtstr = ' '.join(apidoc_argfmt_list) apidoc_cmdstr = apidoc_fmtstr.format(**apidoc_fmtdict) print('[util_setup] autogenerate sphinx docs for %r' % (pkgdir, )) if ut.VERBOSE: print(ut.dict_str(apidoc_fmtdict)) return apidoc_cmdstr, modpath, outputdir def build_conf_replstr(): # # Make custom edits to conf.py # FIXME: #ext_search_text = ut.unindent( # r''' # extensions = [ # [^\]]* # ] # ''') ext_search_text = r'extensions = \[[^/]*\]' # TODO: http://sphinx-doc.org/ext/math.html#module-sphinx.ext.pngmath #'sphinx.ext.mathjax', exclude_modules = [] # ['ibeis.all_imports'] ext_repl_text = ut.codeblock(''' MOCK_MODULES = {exclude_modules} if len(MOCK_MODULES) > 0: import mock for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', # For LaTeX 'sphinx.ext.pngmath', # For Google Sytle Docstrs # https://pypi.python.org/pypi/sphinxcontrib-napoleon 'sphinxcontrib.napoleon', #'sphinx.ext.napoleon', ] ''').format(exclude_modules=str(exclude_modules)) #theme_search = 'html_theme = \'default\'' theme_search = 'html_theme = \'[a-zA-Z_1-3]*\'' theme_repl = ut.codeblock(''' import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] ''') head_text = ut.codeblock(''' from sphinx.ext.autodoc import between import sphinx_rtd_theme import sys import os # Dont parse IBEIS args os.environ['IBIES_PARSE_ARGS'] = 'OFF' os.environ['UTOOL_AUTOGEN_SPHINX_RUNNING'] = 'ON' sys.path.append('{modpath}') sys.path.append(sys.path.insert(0, os.path.abspath("../"))) autosummary_generate = True modindex_common_prefix = ['_'] ''').format(modpath=ut.truepath(modpath)) tail_text = ut.codeblock(''' def setup(app): # Register a sphinx.ext.autodoc.between listener to ignore everything # between lines that contain the word IGNORE app.connect('autodoc-process-docstring', between('^.*IGNORE.*$', exclude=True)) return app ''') return (ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text) apidoc_cmdstr, modpath, outputdir = build_sphinx_apidoc_cmdstr() ext_search_text, ext_repl_text, theme_search, theme_repl, head_text, tail_text = build_conf_replstr( ) dry = ut.get_argflag('--dry') if not dry: # Execute sphinx-apidoc ut.cmd(apidoc_cmdstr, shell=True) # sphinx-apidoc outputs conf.py to <outputdir>, add custom commands # # Change dir to <outputdir> print('chdir' + outputdir) os.chdir(outputdir) conf_fname = 'conf.py' conf_text = ut.read_from(conf_fname) conf_text = conf_text.replace('import sys', 'import sys # NOQA') conf_text = conf_text.replace('import os', 'import os # NOQA') conf_text = ut.regex_replace(theme_search, theme_repl, conf_text) conf_text = ut.regex_replace(ext_search_text, ext_repl_text, conf_text) conf_text = head_text + '\n' + conf_text + tail_text ut.write_to(conf_fname, conf_text) # Make the documentation #if ut.LINUX: # ut.cmd('make html', shell=True) #if ut.WIN32: #raw_input('waiting') if not ut.get_argflag('--nomake'): ut.cmd('make', 'html', shell=True) else: print(apidoc_cmdstr) print('cd ' + outputdir) print('manual edits of conf.py') print('make html')
def make_application_icon(exe_fpath, dry=True, props={}): r""" CommandLine: python -m utool.util_ubuntu --exec-make_application_icon --exe=cockatrice --icon=/home/joncrall/code/Cockatrice/cockatrice/resources/cockatrice.png python -m utool.util_ubuntu --exec-make_application_icon --exe=cockatrice --icon=/home/joncrall/code/Cockatrice/cockatrice/resources/cockatrice.png python -m utool.util_ubuntu --exec-make_application_icon --exe=/opt/zotero/zotero --icon=/opt/zotero/chrome/icons/default/main-window.ico python -m utool.util_ubuntu --exec-make_application_icon --exe "env WINEPREFIX="/home/joncrall/.wine" wine C:\\\\windows\\\\command\\\\start.exe /Unix /home/joncrall/.wine32-dotnet45/dosdevices/c:/users/Public/Desktop/Hearthstone.lnk" --path "/home/joncrall/.wine/dosdevices/c:/Program Files (x86)/Hearthstone" # Exec=env WINEPREFIX="/home/joncrall/.wine" wine /home/joncrall/.wine/drive_c/Program\ Files\ \(x86\)/Battle.net/Battle.net.exe --icon=/opt/zotero/chrome/icons/default/main-window.ico python -m utool.util_ubuntu --exec-make_application_icon --exe=/home/joncrall/code/build-ArenaTracker-Desktop_Qt_5_6_1_GCC_64bit-Debug update-desktop-database ~/.local/share/applications Example: >>> # DISABLE_DOCTEST >>> from utool.util_ubuntu import * # NOQA >>> import utool as ut >>> exe_fpath = ut.get_argval('--exe', default='cockatrice') >>> icon = ut.get_argval('--icon', default=None) >>> dry = not ut.get_argflag(('--write', '-w')) >>> props = {'terminal': False, 'icon': icon} >>> result = make_application_icon(exe_fpath, dry, props) >>> print(result) """ import utool as ut exe_fname_noext = splitext(basename(exe_fpath))[0] app_name = exe_fname_noext.replace('_', '-') nice_name = ' '.join( [word[0].upper() + word[1:].lower() for word in app_name.replace('-', ' ').split(' ')] ) lines = [ '[Desktop Entry]', 'Name={nice_name}', 'Exec={exe_fpath}', ] if 'mime_name' in props: lines += ['MimeType=application/x-{mime_name}'] if 'icon' in props: lines += ['Icon={icon}'] if props.get('path'): lines += ['Path={path}'] # if props.get('comment'): # lines += ['Path={comment}'] lines += [ 'Terminal={terminal}', 'Type=Application', 'Categories=Utility;Application;', 'Comment=Custom App', ] fmtdict = locals() fmtdict.update(props) prefix = ut.truepath('~/.local/share') app_codeblock = '\n'.join(lines).format(**fmtdict) app_dpath = join(prefix, 'applications') app_fpath = join(app_dpath, '{app_name}.desktop'.format(**locals())) print(app_codeblock) print('---') print(app_fpath) print('L___') if not dry: ut.writeto(app_fpath, app_codeblock, verbose=ut.NOT_QUIET, n=None) ut.cmd('update-desktop-database ~/.local/share/applications')
'simplejson', ] for modname in modlist: try: level = 0 module = __import__(modname, globals(), locals(), fromlist=[], level=level) except ImportError as ex: install_cmd = get_install_cmd(modname) print( 'Please Run follow instruction and then rerun fix_common_issues.py: ' ) print(install_cmd) import utool as ut ut.cmd(install_cmd, shell=True) sys.exit(0) """ References: http://superuser.com/questions/345719/how-to-chmod-and-chown-hidden-files-in-linux cd /opt/ibeis shopt -s dotglob sudo chown -R jasonp:ibeis * sudo chmod -R 775 * shopt -u dotglob """
def add_new_mimetype_association(ext, mime_name, exe_fpath=None, dry=True): """ TODO: move to external manager and generalize Args: ext (str): extension to associate mime_name (str): the name of the mime_name to create (defaults to ext) exe_fpath (str): executable location if this is for one specific file References: https://wiki.archlinux.org/index.php/Default_applications#Custom_file_associations Args: ext (str): extension to associate exe_fpath (str): executable location mime_name (str): the name of the mime_name to create (defaults to ext) CommandLine: python -m utool.util_ubuntu --exec-add_new_mimetype_association # Add ability to open ipython notebooks via double click python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=/usr/local/bin/ipynb python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=ipynb+json --ext=.ipynb --exe-fpath=jupyter-notebook --force python -m utool.util_ubuntu --exec-add_new_mimetype_association --mime-name=sqlite --ext=.sqlite --exe-fpath=sqlitebrowser Example: >>> # SCRIPT >>> from utool.util_ubuntu import * # NOQA >>> import utool as ut >>> ext = ut.get_argval('--ext', type_=str, default=None) >>> mime_name = ut.get_argval('--mime_name', type_=str, default=None) >>> exe_fpath = ut.get_argval('--exe_fpath', type_=str, default=None) >>> dry = not ut.get_argflag('--force') >>> result = add_new_mimetype_association(ext, mime_name, exe_fpath, dry) >>> print(result) """ import utool as ut terminal = True mime_codeblock = ut.codeblock( ''' <?xml version="1.0" encoding="UTF-8"?> <mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info"> <mime-type type="application/x-{mime_name}"> <glob-deleteall/> <glob pattern="*{ext}"/> </mime-type> </mime-info> ''' ).format(**locals()) prefix = ut.truepath('~/.local/share') mime_dpath = join(prefix, 'mime/packages') mime_fpath = join(mime_dpath, 'application-x-{mime_name}.xml'.format(**locals())) print(mime_codeblock) print('---') print(mime_fpath) print('L___') if exe_fpath is not None: exe_fname_noext = splitext(basename(exe_fpath))[0] app_name = exe_fname_noext.replace('_', '-') nice_name = ' '.join( [word[0].upper() + word[1:].lower() for word in app_name.replace('-', ' ').split(' ')] ) app_codeblock = ut.codeblock( ''' [Desktop Entry] Name={nice_name} Exec={exe_fpath} MimeType=application/x-{mime_name} Terminal={terminal} Type=Application Categories=Utility;Application; Comment=Custom App ''' ).format(**locals()) app_dpath = join(prefix, 'applications') app_fpath = join(app_dpath, '{app_name}.desktop'.format(**locals())) print(app_codeblock) print('---') print(app_fpath) print('L___') # WRITE FILES if not dry: ut.ensuredir(mime_dpath) ut.ensuredir(app_dpath) ut.writeto(mime_fpath, mime_codeblock, verbose=ut.NOT_QUIET, n=None) if exe_fpath is not None: ut.writeto(app_fpath, app_codeblock, verbose=ut.NOT_QUIET, n=None) # UPDATE BACKENDS #ut.cmd('update-mime-database /usr/share/mime') #~/.local/share/applications/mimeapps.list print(ut.codeblock( ''' Run these commands: update-desktop-database ~/.local/share/applications update-mime-database ~/.local/share/mime ''' )) if exe_fpath is not None: ut.cmd('update-desktop-database ~/.local/share/applications') ut.cmd('update-mime-database ~/.local/share/mime') else: print('dry_run')