def find_or_download_tomcat(): r""" Returns: str: tomcat_dpath CommandLine: # Reset python -m purge_local_wildbook python -m ibeis --tf purge_local_wildbook python -m ibeis --tf find_or_download_tomcat Example: >>> # SCRIPT >>> from ibeis.control.wildbook_manager import * # NOQA >>> tomcat_dpath = find_or_download_tomcat() >>> result = ('tomcat_dpath = %s' % (str(tomcat_dpath),)) >>> print(result) """ tomcat_dpath = find_tomcat() if tomcat_dpath is None: tomcat_dpath = download_tomcat() else: ut.assertpath(tomcat_dpath) return tomcat_dpath
def find_or_download_tomcat(): r""" Returns: str: tomcat_dpath CommandLine: # Reset python -m ibeis.control.manual_wildbook_funcs --test-reset_local_wildbook python -m ibeis.control.manual_wildbook_funcs --test-find_or_download_tomcat python -m ibeis --tf reset_local_wildbook python -m ibeis --tf find_or_download_tomcat Example: >>> # SCRIPT >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> tomcat_dpath = find_or_download_tomcat() >>> result = ('tomcat_dpath = %s' % (str(tomcat_dpath),)) >>> print(result) """ tomcat_dpath = find_tomcat() if tomcat_dpath is None: tomcat_dpath = download_tomcat() else: ut.assertpath(tomcat_dpath) return tomcat_dpath
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath, )) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath, )) gs_exe = find_ghostscript_exe() cmd_list = (gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath) ut.cmd(*cmd_list) return output_pdf_fpath
def TEST_DELETE_IMAGE_THUMBTUPS(ibs, back): gpath_list = grabdata.get_test_gpaths(ndata=None)[0:4] gid_list = ibs.add_images(gpath_list) bbox_list = [(0, 0, 100, 100)]*len(gid_list) name_list = ['a', 'b', 'a', 'd'] aid_list = ibs.add_annots(gid_list, bbox_list=bbox_list, name_list=name_list) assert len(aid_list) !=0, "No annotations added" thumbpath_list = ibs.get_image_thumbpath(gid_list) gpath_list = ibs.get_image_paths(gid_list) ibs.delete_image_thumbtups(gid_list) assert utool.is_list(thumbpath_list), "thumbpath_list is not a list" assert utool.is_list(gpath_list), "gpath_list is not a list" for path in thumbpath_list: assert not utool.checkpath(path), "Thumbnail not deleted" for path in gpath_list: utool.assertpath(path) return locals()
def find_java_jvm(): r""" CommandLine: python -m ibeis.control.manual_wildbook_funcs --test-find_java_jvm Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> result = find_java_jvm() >>> print(result) """ candidate_path_list = [ #os.environ.get('JAVA_HOME', None), #'/usr/lib/jvm/java-7-openjdk-amd64', ] jvm_fpath = ut.search_candidate_paths(candidate_path_list, verbose=True) ut.assertpath(jvm_fpath, 'IBEIS cannot find Java Runtime Environment') return jvm_fpath
def find_java_jvm(): r""" CommandLine: python -m ibeis find_java_jvm Example: >>> # DISABLE_DOCTEST >>> from ibeis.control.wildbook_manager import * # NOQA >>> jvm_fpath = find_java_jvm() >>> result = ('jvm_fpath = %r' % (jvm_fpath,)) >>> print(result) """ candidate_path_list = [ os.environ.get('JAVA_HOME', None), '/usr/lib/jvm/java-7-openjdk-amd64', ] jvm_fpath = ut.search_candidate_paths(candidate_path_list, verbose=True) ut.assertpath(jvm_fpath, 'IBEIS cannot find Java Runtime Environment') return jvm_fpath
def find_installed_tomcat(check_unpacked=True, strict=True): """ Asserts that tomcat was properly installed Args: check_unpacked (bool): (default = True) Returns: str: tomcat_dpath CommandLine: python -m ibeis --tmod ibeis.control.manual_wildbook_funcs --exec-find_installed_tomcat python -m ibeis.control.manual_wildbook_funcs --exec-find_installed_tomcat python -m ibeis -tm ibeis.control.manual_wildbook_funcs --exec-find_installed_tomcat python -m ibeis --tf find_installed_tomcat Example: >>> # ENABLE_DOCTEST >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> check_unpacked = True >>> strict = False >>> tomcat_dpath = find_installed_tomcat(check_unpacked, strict) >>> result = ('tomcat_dpath = %s' % (str(tomcat_dpath),)) >>> print(result) """ tomcat_dpath = find_tomcat() if tomcat_dpath is None: msg = 'Cannot find tomcat' if strict: raise ImportError(msg) else: print(msg) return None if check_unpacked: import ibeis # Check that webapps was unpacked wb_target = ibeis.const.WILDBOOK_TARGET webapps_dpath = join(tomcat_dpath, 'webapps') unpacked_war_dpath = join(webapps_dpath, wb_target) ut.assertpath(unpacked_war_dpath) return tomcat_dpath
def compress_pdf(pdf_fpath, output_fname=None): """ uses ghostscript to write a pdf """ import utool as ut ut.assertpath(pdf_fpath) suffix = '_' + ut.get_datestamp(False) + '_compressed' print('pdf_fpath = %r' % (pdf_fpath,)) output_pdf_fpath = ut.augpath(pdf_fpath, suffix, newfname=output_fname) print('output_pdf_fpath = %r' % (output_pdf_fpath,)) gs_exe = find_ghostscript_exe() cmd_list = ( gs_exe, '-sDEVICE=pdfwrite', '-dCompatibilityLevel=1.4', '-dNOPAUSE', '-dQUIET', '-dBATCH', '-sOutputFile=' + output_pdf_fpath, pdf_fpath ) ut.cmd(*cmd_list) return output_pdf_fpath
def find_installed_tomcat(check_unpacked=True, strict=True): """ Asserts that tomcat was properly installed Args: check_unpacked (bool): (default = True) Returns: str: tomcat_dpath CommandLine: python -m wbia find_installed_tomcat Example: >>> # ENABLE_DOCTEST >>> from wbia.control.wildbook_manager import * # NOQA >>> check_unpacked = False >>> strict = False >>> tomcat_dpath = find_installed_tomcat(check_unpacked, strict) >>> result = ('tomcat_dpath = %s' % (str(tomcat_dpath),)) >>> print(result) """ tomcat_dpath = find_tomcat() if tomcat_dpath is None: msg = 'Cannot find tomcat' if strict: raise ImportError(msg) else: logger.info(msg) return None if check_unpacked: import wbia # Check that webapps was unpacked wb_target = wbia.const.WILDBOOK_TARGET webapps_dpath = join(tomcat_dpath, 'webapps') unpacked_war_dpath = join(webapps_dpath, wb_target) ut.assertpath(unpacked_war_dpath) return tomcat_dpath
def get_data_list(): r""" CommandLine: python ~/code/ibeis/_installers/ibeis_pyinstaller_data_helper.py --test-get_data_list Example: >>> # ENABLE_DOCTEST >>> from ibeis_pyinstaller_data_helper import * # NOQA >>> result = get_data_list() >>> DATATUP_LIST, BINARYTUP_LIST, iconfile = result >>> print('DATATUP_LIST = ' + ut.list_str(DATATUP_LIST)) >>> print('BINARYTUP_LIST = ' + ut.list_str(BINARYTUP_LIST)) >>> print(len(DATATUP_LIST)) >>> print(len(BINARYTUP_LIST)) >>> print(iconfile) """ # Build data before running analysis for quick debugging DATATUP_LIST = [] BINARYTUP_LIST = [] #import pyhesaff #pyhesaff.HESAFF_CLIB.__LIB_FPATH__ #import pyrf #pyrf.RF_CLIB.__LIB_FPATH__ # Hesaff libhesaff_fname = 'libhesaff' + LIB_EXT libhesaff_src = realpath(join(root_dir, '..', 'hesaff', 'pyhesaff', libhesaff_fname)) libhesaff_dst = join(ibsbuild, 'pyhesaff', 'lib', libhesaff_fname) DATATUP_LIST.append((libhesaff_dst, libhesaff_src)) # PyRF libpyrf_fname = 'libpyrf' + LIB_EXT libpyrf_src = realpath(join(root_dir, '..', 'pyrf', 'pyrf', libpyrf_fname)) libpyrf_dst = join(ibsbuild, 'pyrf', 'lib', libpyrf_fname) DATATUP_LIST.append((libpyrf_dst, libpyrf_src)) # FLANN libflann_fname = 'libflann' + LIB_EXT #try: # #import pyflann # #pyflann.__file__ # #join(dirname(dirname(pyflann.__file__)), 'build') #except ImportError as ex: # print('PYFLANN IS NOT IMPORTABLE') # raise #if WIN32 or LINUX: # FLANN #libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) #libflann_dst = join(ibsbuild, libflann_fname) #elif APPLE: # # libflann_src = '/pyflann/lib/libflann.dylib' # # libflann_dst = join(ibsbuild, libflann_fname) # libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) # libflann_dst = join(ibsbuild, libflann_fname) # This path is when pyflann was built using setup.py develop libflann_src = realpath(join(root_dir, '..', 'flann', 'build', 'lib', libflann_fname)) libflann_dst = join(ibsbuild, 'pyflann', 'lib', libflann_fname) DATATUP_LIST.append((libflann_dst, libflann_src)) # VTool vtool_libs = ['libsver'] for libname in vtool_libs: lib_fname = libname + LIB_EXT vtlib_src = realpath(join(root_dir, '..', 'vtool', 'vtool', lib_fname)) vtlib_dst = join(ibsbuild, 'vtool', lib_fname) DATATUP_LIST.append((vtlib_dst, vtlib_src)) linux_lib_dpaths = [ '/usr/lib/x86_64-linux-gnu', '/usr/lib', '/usr/local/lib' ] # OpenMP if APPLE: # BSDDB, Fix for the modules that PyInstaller needs and (for some reason) # are not being added by PyInstaller libbsddb_src = '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload/_bsddb.so' libbsddb_dst = join(ibsbuild, '_bsddb.so') DATATUP_LIST.append((libbsddb_dst, libbsddb_src)) #libgomp_src = '/opt/local/lib/libgomp.dylib' libgomp_src = '/opt/local/lib/gcc48/libgomp.dylib' BINARYTUP_LIST.append(('libgomp.1.dylib', libgomp_src, 'BINARY')) # very hack libiomp_src = '/Users/bluemellophone/code/libomp_oss/exports/mac_32e/lib.thin/libiomp5.dylib' BINARYTUP_LIST.append(('libiomp5.dylib', libiomp_src, 'BINARY')) if LINUX: libgomp_src = ut.search_in_dirs('libgomp.so.1', linux_lib_dpaths) ut.assertpath(libgomp_src) BINARYTUP_LIST.append(('libgomp.so.1', libgomp_src, 'BINARY')) # MinGW if WIN32: mingw_root = r'C:\MinGW\bin' mingw_dlls = ['libgcc_s_dw2-1.dll', 'libstdc++-6.dll', 'libgomp-1.dll', 'pthreadGC2.dll'] for lib_fname in mingw_dlls: lib_src = join(mingw_root, lib_fname) lib_dst = join(ibsbuild, lib_fname) DATATUP_LIST.append((lib_dst, lib_src)) # We need to add these 4 opencv libraries because pyinstaller does not find them. #OPENCV_EXT = {'win32': '248.dll', # 'darwin': '.2.4.dylib', # 'linux2': '.so.2.4'}[PLATFORM] target_cv_version = '3.0.0' OPENCV_EXT = {'win32': target_cv_version.replace('.', '') + '.dll', 'darwin': '.' + target_cv_version + '.dylib', 'linux2': '.so.' + target_cv_version}[PLATFORM] missing_cv_name_list = [ 'libopencv_videostab', 'libopencv_superres', 'libopencv_stitching', #'libopencv_gpu', 'libopencv_core', 'libopencv_highgui', 'libopencv_imgproc', ] # Hack to find the appropriate opencv libs for name in missing_cv_name_list: fname = name + OPENCV_EXT src = '' dst = '' if APPLE: src = join('/opt/local/lib', fname) elif LINUX: #src = join('/usr/lib', fname) src, tried = ut.search_in_dirs(fname, linux_lib_dpaths, strict=True, return_tried=True) elif WIN32: if ut.get_computer_name() == 'Ooo': src = join(r'C:/Program Files (x86)/OpenCV/x86/mingw/bin', fname) else: src = join(root_dir, '../opencv/build/bin', fname) dst = join(ibsbuild, fname) # ut.assertpath(src) DATATUP_LIST.append((dst, src)) ################################## # QT Gui dependencies ################################## if APPLE: walk_path = '/opt/local/Library/Frameworks/QtGui.framework/Versions/4/Resources/qt_menu.nib' for root, dirs, files in os.walk(walk_path): for lib_fname in files: toc_src = join(walk_path, lib_fname) toc_dst = join('qt_menu.nib', lib_fname) DATATUP_LIST.append((toc_dst, toc_src)) ################################## # Documentation, Icons, and Web Assets ################################## # Documentation #userguide_dst = join('.', '_docs', 'IBEISUserGuide.pdf') #userguide_src = join(root_dir, '_docs', 'IBEISUserGuide.pdf') #DATATUP_LIST.append((userguide_dst, userguide_src)) # Icon File ICON_EXT = {'darwin': '.icns', 'win32': '.ico', 'linux2': '.ico'}[PLATFORM] iconfile = join('_installers', 'ibsicon' + ICON_EXT) icon_src = join(root_dir, iconfile) icon_dst = join(ibsbuild, iconfile) DATATUP_LIST.append((icon_dst, icon_src)) print('[installer] Checking Data (preweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath(src, verbose=True), 'checkpath for src=%r failed' % (src,) except Exception as ex: ut.printex(ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise # Web Assets INSTALL_WEB = True and not ut.get_argflag('--noweb') if INSTALL_WEB: web_root = join('ibeis', 'web/') #walk_path = join(web_root, 'static') #static_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # static_data.append((toc_dst, toc_src)) #ut.get_list_column(static_data, 1) == ut.glob(walk_path, '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_src_list = ut.glob(join(web_root, 'static'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_dst_list = [relpath(src, join(root_dir, 'ibeis')) for src in static_src_list] static_data = zip(static_dst_list, static_src_list) DATATUP_LIST.extend(static_data) #walk_path = join(web_root, 'templates') #template_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # template_data.append((toc_dst, toc_src)) template_src_list = ut.glob(join(web_root, 'templates'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) template_dst_list = [relpath(src, join(root_dir, 'ibeis')) for src in template_src_list] template_data = zip(template_dst_list, template_src_list) DATATUP_LIST.extend(template_data) print('[installer] Checking Data (postweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath(src, verbose=False), 'checkpath for src=%r failed' % (src,) except Exception as ex: ut.printex(ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise return DATATUP_LIST, BINARYTUP_LIST, iconfile
def assert_models(modeldir='default', verbose=True): for algo, algo_modeldir in iter_algo_modeldirs(modeldir): ut.assertpath(algo_modeldir, verbose=verbose)
#!/usr/bin/env python # -*- coding: utf-8 -*- """ script to open directory in current window manager """ import utool as ut if __name__ == '__main__': import sys if len(sys.argv) == 2: path = sys.argv[1] else: path = None ut.assertpath(path) if ut.checkpath(path, verbose=True): ut.view_directory(path) #F:\\data\\work\\PZ_MTEST\\_ibsdb\\
def install_wildbook(verbose=ut.NOT_QUIET): """ Script to setup wildbook on a unix based system (hopefully eventually this will generalize to win32) CommandLine: # Reset python -m ibeis --tf reset_local_wildbook # Setup python -m ibeis --tf install_wildbook # Startup python -m ibeis --tf startup_wildbook_server --show --exec-mode # Reset python -m ibeis.control.manual_wildbook_funcs --test-reset_local_wildbook # Setup python -m ibeis.control.manual_wildbook_funcs --test-install_wildbook # Startup python -m ibeis.control.manual_wildbook_funcs --test-startup_wildbook_server --show --exec-mode Example: >>> # SCRIPT >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> verbose = True >>> result = install_wildbook() >>> print(result) """ # TODO: allow custom specified tomcat directory from os.path import basename, splitext, join import time import re import subprocess try: output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) _java_version = output.split('\n')[0] _java_version = _java_version.replace('java version ', '') java_version = _java_version.replace('"', '') print('java_version = %r' % (java_version,)) if not java_version.startswith('1.7'): print('Warning wildbook is only supported for java 1.7') except OSError: output = None if output is None: raise ImportError( 'Cannot find java on this machine. ' 'Please install java: http://www.java.com/en/download/') tomcat_dpath = find_or_download_tomcat() assert tomcat_dpath is not None, 'Could not find tomcat' war_fpath = find_or_download_wilbook_warfile() war_fname = basename(war_fpath) wb_target = splitext(war_fname)[0] # Ensure environment variables #os.environ['JAVA_HOME'] = find_java_jvm() #os.environ['TOMCAT_HOME'] = tomcat_dpath #os.environ['CATALINA_HOME'] = tomcat_dpath # Move the war file to tomcat webapps if not there webapps_dpath = join(tomcat_dpath, 'webapps') deploy_war_fpath = join(webapps_dpath, war_fname) if not ut.checkpath(deploy_war_fpath, verbose=verbose): ut.copy(war_fpath, deploy_war_fpath) # Ensure that the war file has been unpacked unpacked_war_dpath = join(webapps_dpath, wb_target) if not ut.checkpath(unpacked_war_dpath, verbose=verbose): # Need to make sure you start catalina in the same directory otherwise # the derby databsae gets put in in cwd tomcat_startup_dir = get_tomcat_startup_tmpdir() with ut.ChdirContext(tomcat_startup_dir): # Starting and stoping catalina should be sufficient to unpack the # war startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') ut.cmd(ut.quote_single_command(startup_fpath)) print('It is NOT ok if the startup.sh fails\n') # wait for the war to be unpacked for retry_count in range(0, 6): time.sleep(1) if ut.checkpath(unpacked_war_dpath, verbose=True): break else: print('Retrying') # ensure that the server is ruuning import requests print('Checking if we can ping the server') response = requests.get('http://localhost:8080') if response is None or response.status_code != 200: print('There may be an error starting the server') else: print('Seem able to ping the server') # assert tht the war was unpacked ut.assertpath(unpacked_war_dpath, ( 'Wildbook war might have not unpacked correctly. This may ' 'be ok. Try again. If it fails a second time, then there is a ' 'problem.'), verbose=True) # shutdown the server ut.cmd(ut.quote_single_command(shutdown_fpath)) print('It is ok if the shutdown.sh fails') time.sleep(.5) # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line,) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') print('Wildbook is installed and waiting to be started')
def update_wildbook_install_config(webapps_dpath, unpacked_war_dpath): """ CommandLine: python -m ibeis ensure_local_war python -m ibeis update_wildbook_install_config python -m ibeis update_wildbook_install_config --show Example: >>> from ibeis.control.wildbook_manager import * # NOQA >>> import ibeis >>> tomcat_dpath = find_installed_tomcat() >>> webapps_dpath = join(tomcat_dpath, 'webapps') >>> wb_target = ibeis.const.WILDBOOK_TARGET >>> unpacked_war_dpath = join(webapps_dpath, wb_target) >>> locals_ = ut.exec_func_src(update_wildbook_install_config, globals()) >>> #update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) >>> ut.quit_if_noshow() >>> ut.vd(unpacked_war_dpath) >>> ut.editfile(locals_['permission_fpath']) >>> ut.editfile(locals_['jdoconfig_fpath']) >>> ut.editfile(locals_['asset_store_fpath']) """ mysql_mode = not ut.get_argflag('--nomysql') #if ut.get_argflag('--vd'): # ut.vd(unpacked_war_dpath) #find_installed_tomcat # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ # '/ImageSetSetMarkedIndividual = authc, roles[admin]' '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line,) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') # Make sure we are using a non-process based database jdoconfig_fpath = join(unpacked_war_dpath, 'WEB-INF/classes/bundles/jdoconfig.properties') print('Fixing backend database config') print('jdoconfig_fpath = %r' % (jdoconfig_fpath,)) ut.assertpath(jdoconfig_fpath) jdoconfig_text = ut.readfrom(jdoconfig_fpath) #ut.vd(dirname(jdoconfig_fpath)) #ut.editfile(jdoconfig_fpath) if mysql_mode: jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', False) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', 1) mysql_user = '******' mysql_passwd = 'somepassword' mysql_dbname = 'ibeiswbtestdb' # Use mysql jdoconfig_text = re.sub( 'datanucleus.ConnectionUserName = .*$', 'datanucleus.ConnectionUserName = '******'datanucleus.ConnectionPassword = .*$', 'datanucleus.ConnectionPassword = '******'datanucleus.ConnectionURL *= *jdbc:mysql:.*$', 'datanucleus.ConnectionURL = jdbc:mysql://localhost:3306/' + mysql_dbname, jdoconfig_text, flags=re.MULTILINE) jdoconfig_text = re.sub( '^.*jdbc:mysql://localhost:3306/shepherd.*$', '', jdoconfig_text, flags=re.MULTILINE) else: # Use SQLIIte jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', False) ut.writeto(jdoconfig_fpath, jdoconfig_text) # Need to make sure wildbook can store information in a reasonalbe place #tomcat_data_dir = join(tomcat_startup_dir, 'webapps', 'wildbook_data_dir') tomcat_data_dir = join(webapps_dpath, 'wildbook_data_dir') ut.ensuredir(tomcat_data_dir) ut.writeto(join(tomcat_data_dir, 'test.txt'), 'A hosted test file') asset_store_fpath = join(unpacked_war_dpath, 'createAssetStore.jsp') asset_store_text = ut.read_from(asset_store_fpath) #data_path_pat = ut.named_field('data_path', 'new File(".*?").toPath') new_line = 'LocalAssetStore as = new LocalAssetStore("example Local AssetStore", new File("%s").toPath(), "%s", true);' % ( tomcat_data_dir, 'http://localhost:8080/' + basename(tomcat_data_dir) ) # HACKY asset_store_text2 = re.sub('^LocalAssetStore as = .*$', new_line, asset_store_text, flags=re.MULTILINE) ut.writeto(asset_store_fpath, asset_store_text2)
def install_wildbook(verbose=ut.NOT_QUIET): """ Script to setup wildbook on a unix based system (hopefully eventually this will generalize to win32) CommandLine: # Reset ibeis purge_local_wildbook ibeis ensure_wb_mysql ibeis ensure_local_war # Setup ibeis install_wildbook # ibeis install_wildbook --nomysql # Startup ibeis startup_wildbook_server --show Alternates: ibeis install_wildbook --redownload-war ibeis install_wildbook --assets ibeis startup_wildbook_server --show Example: >>> # SCRIPT >>> from ibeis.control.wildbook_manager import * # NOQA >>> verbose = True >>> result = install_wildbook() >>> print(result) """ import requests # Ensure that the war file has been unpacked tomcat_dpath, webapps_dpath, wb_target = ensure_local_war() unpacked_war_dpath = join(webapps_dpath, wb_target) tomcat_startup_dir = get_tomcat_startup_tmpdir() fresh_install = not ut.checkpath(unpacked_war_dpath, verbose=verbose) if fresh_install: # Need to make sure you start catalina in the same directory otherwise # the derby databsae gets put in in cwd with ut.ChdirContext(tomcat_startup_dir): # Starting and stoping catalina should be sufficient to unpack the # war startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') #shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') ut.cmd(ut.quote_single_command(startup_fpath)) print('It is NOT ok if the startup.sh fails\n') # wait for the war to be unpacked for retry_count in range(0, 6): time.sleep(1) if ut.checkpath(unpacked_war_dpath, verbose=True): break else: print('Retrying') # ensure that the server is ruuning print('Checking if we can ping the server') response = requests.get('http://localhost:8080') if response is None or response.status_code != 200: print('There may be an error starting the server') else: print('Seem able to ping the server') # assert tht the war was unpacked ut.assertpath(unpacked_war_dpath, ( 'Wildbook war might have not unpacked correctly. This may ' 'be ok. Try again. If it fails a second time, then there is a ' 'problem.'), verbose=True) # Don't shutdown just yet. Need to create assets update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) asset_flag_fpath = join(tomcat_startup_dir, 'made_assets.flag') # Pinging the server to create asset store # Ensureing that createAssetStore exists if not ut.checkpath(asset_flag_fpath): if not fresh_install: startup_wildbook_server() #web_url = startup_wildbook_server(verbose=False) print('Creating asset store') wb_url = 'http://localhost:8080/' + wb_target response = requests.get(wb_url + '/createAssetStore.jsp') if response is None or response.status_code != 200: print('There may be an error starting the server') #if response.status_code == 500: print(response.text) assert False, 'response error' else: print('Created asset store') # Create file signaling we did this ut.writeto(asset_flag_fpath, 'True') shutdown_wildbook_server(verbose=False) print('It is ok if the shutdown fails') elif fresh_install: shutdown_wildbook_server(verbose=False) #127.0.0.1:8080/wildbook_data_dir/test.txt print('Wildbook is installed and waiting to be started')
def get_data_list(): r""" CommandLine: python ~/code/ibeis/_installers/ibeis_pyinstaller_data_helper.py --test-get_data_list Example: >>> # ENABLE_DOCTEST >>> from ibeis_pyinstaller_data_helper import * # NOQA >>> result = get_data_list() >>> DATATUP_LIST, BINARYTUP_LIST, iconfile = result >>> print('DATATUP_LIST = ' + ut.list_str(DATATUP_LIST)) >>> print('BINARYTUP_LIST = ' + ut.list_str(BINARYTUP_LIST)) >>> print(len(DATATUP_LIST)) >>> print(len(BINARYTUP_LIST)) >>> print(iconfile) """ # Build data before running analysis for quick debugging DATATUP_LIST = [] BINARYTUP_LIST = [] #import pyhesaff #pyhesaff.HESAFF_CLIB.__LIB_FPATH__ #import pyrf #pyrf.RF_CLIB.__LIB_FPATH__ # Hesaff libhesaff_fname = 'libhesaff' + LIB_EXT libhesaff_src = realpath( join(root_dir, '..', 'hesaff', 'pyhesaff', libhesaff_fname)) libhesaff_dst = join(ibsbuild, 'pyhesaff', 'lib', libhesaff_fname) DATATUP_LIST.append((libhesaff_dst, libhesaff_src)) # PyRF libpyrf_fname = 'libpyrf' + LIB_EXT libpyrf_src = realpath(join(root_dir, '..', 'pyrf', 'pyrf', libpyrf_fname)) libpyrf_dst = join(ibsbuild, 'pyrf', 'lib', libpyrf_fname) DATATUP_LIST.append((libpyrf_dst, libpyrf_src)) # FLANN libflann_fname = 'libflann' + LIB_EXT #try: # #import pyflann # #pyflann.__file__ # #join(dirname(dirname(pyflann.__file__)), 'build') #except ImportError as ex: # print('PYFLANN IS NOT IMPORTABLE') # raise #if WIN32 or LINUX: # FLANN #libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) #libflann_dst = join(ibsbuild, libflann_fname) #elif APPLE: # # libflann_src = '/pyflann/lib/libflann.dylib' # # libflann_dst = join(ibsbuild, libflann_fname) # libflann_src = join_SITE_PACKAGES('pyflann', 'lib', libflann_fname) # libflann_dst = join(ibsbuild, libflann_fname) # This path is when pyflann was built using setup.py develop libflann_src = realpath( join(root_dir, '..', 'flann', 'build', 'lib', libflann_fname)) libflann_dst = join(ibsbuild, 'pyflann', 'lib', libflann_fname) DATATUP_LIST.append((libflann_dst, libflann_src)) # VTool vtool_libs = ['libsver'] for libname in vtool_libs: lib_fname = libname + LIB_EXT vtlib_src = realpath(join(root_dir, '..', 'vtool', 'vtool', lib_fname)) vtlib_dst = join(ibsbuild, 'vtool', lib_fname) DATATUP_LIST.append((vtlib_dst, vtlib_src)) linux_lib_dpaths = [ '/usr/lib/x86_64-linux-gnu', '/usr/lib', '/usr/local/lib' ] # OpenMP if APPLE: # BSDDB, Fix for the modules that PyInstaller needs and (for some reason) # are not being added by PyInstaller libbsddb_src = '/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload/_bsddb.so' libbsddb_dst = join(ibsbuild, '_bsddb.so') DATATUP_LIST.append((libbsddb_dst, libbsddb_src)) #libgomp_src = '/opt/local/lib/libgomp.dylib' libgomp_src = '/opt/local/lib/gcc48/libgomp.dylib' BINARYTUP_LIST.append(('libgomp.1.dylib', libgomp_src, 'BINARY')) # very hack libiomp_src = '/Users/bluemellophone/code/libomp_oss/exports/mac_32e/lib.thin/libiomp5.dylib' BINARYTUP_LIST.append(('libiomp5.dylib', libiomp_src, 'BINARY')) if LINUX: libgomp_src = ut.search_in_dirs('libgomp.so.1', linux_lib_dpaths) ut.assertpath(libgomp_src) BINARYTUP_LIST.append(('libgomp.so.1', libgomp_src, 'BINARY')) # MinGW if WIN32: mingw_root = r'C:\MinGW\bin' mingw_dlls = [ 'libgcc_s_dw2-1.dll', 'libstdc++-6.dll', 'libgomp-1.dll', 'pthreadGC2.dll' ] for lib_fname in mingw_dlls: lib_src = join(mingw_root, lib_fname) lib_dst = join(ibsbuild, lib_fname) DATATUP_LIST.append((lib_dst, lib_src)) # We need to add these 4 opencv libraries because pyinstaller does not find them. #OPENCV_EXT = {'win32': '248.dll', # 'darwin': '.2.4.dylib', # 'linux2': '.so.2.4'}[PLATFORM] target_cv_version = '3.0.0' OPENCV_EXT = { 'win32': target_cv_version.replace('.', '') + '.dll', 'darwin': '.' + target_cv_version + '.dylib', 'linux2': '.so.' + target_cv_version }[PLATFORM] missing_cv_name_list = [ 'libopencv_videostab', 'libopencv_superres', 'libopencv_stitching', #'libopencv_gpu', 'libopencv_core', 'libopencv_highgui', 'libopencv_imgproc', ] # Hack to find the appropriate opencv libs for name in missing_cv_name_list: fname = name + OPENCV_EXT src = '' dst = '' if APPLE: src = join('/opt/local/lib', fname) elif LINUX: #src = join('/usr/lib', fname) src, tried = ut.search_in_dirs(fname, linux_lib_dpaths, strict=True, return_tried=True) elif WIN32: if ut.get_computer_name() == 'Ooo': src = join(r'C:/Program Files (x86)/OpenCV/x86/mingw/bin', fname) else: src = join(root_dir, '../opencv/build/bin', fname) dst = join(ibsbuild, fname) # ut.assertpath(src) DATATUP_LIST.append((dst, src)) ################################## # QT Gui dependencies ################################## if APPLE: walk_path = '/opt/local/Library/Frameworks/QtGui.framework/Versions/4/Resources/qt_menu.nib' for root, dirs, files in os.walk(walk_path): for lib_fname in files: toc_src = join(walk_path, lib_fname) toc_dst = join('qt_menu.nib', lib_fname) DATATUP_LIST.append((toc_dst, toc_src)) ################################## # Documentation, Icons, and Web Assets ################################## # Documentation #userguide_dst = join('.', '_docs', 'IBEISUserGuide.pdf') #userguide_src = join(root_dir, '_docs', 'IBEISUserGuide.pdf') #DATATUP_LIST.append((userguide_dst, userguide_src)) # Icon File ICON_EXT = {'darwin': '.icns', 'win32': '.ico', 'linux2': '.ico'}[PLATFORM] iconfile = join('_installers', 'ibsicon' + ICON_EXT) icon_src = join(root_dir, iconfile) icon_dst = join(ibsbuild, iconfile) DATATUP_LIST.append((icon_dst, icon_src)) print('[installer] Checking Data (preweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath( src, verbose=True), 'checkpath for src=%r failed' % (src, ) except Exception as ex: ut.printex( ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise # Web Assets INSTALL_WEB = True and not ut.get_argflag('--noweb') if INSTALL_WEB: web_root = join('ibeis', 'web/') #walk_path = join(web_root, 'static') #static_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # static_data.append((toc_dst, toc_src)) #ut.get_list_column(static_data, 1) == ut.glob(walk_path, '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_src_list = ut.glob(join(web_root, 'static'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) static_dst_list = [ relpath(src, join(root_dir, 'ibeis')) for src in static_src_list ] static_data = zip(static_dst_list, static_src_list) DATATUP_LIST.extend(static_data) #walk_path = join(web_root, 'templates') #template_data = [] #for root, dirs, files in os.walk(walk_path): # root2 = root.replace(web_root, '') # for icon_fname in files: # if '.DS_Store' not in icon_fname: # toc_src = join(abspath(root), icon_fname) # toc_dst = join(root2, icon_fname) # template_data.append((toc_dst, toc_src)) template_src_list = ut.glob(join(web_root, 'templates'), '*', recursive=True, with_dirs=False, exclude_dirs=['.DS_Store']) template_dst_list = [ relpath(src, join(root_dir, 'ibeis')) for src in template_src_list ] template_data = zip(template_dst_list, template_src_list) DATATUP_LIST.extend(template_data) print('[installer] Checking Data (postweb)') try: for (dst, src) in DATATUP_LIST: assert ut.checkpath( src, verbose=False), 'checkpath for src=%r failed' % (src, ) except Exception as ex: ut.printex( ex, 'Checking data failed DATATUP_LIST=' + ut.list_str(DATATUP_LIST)) raise return DATATUP_LIST, BINARYTUP_LIST, iconfile
def update_wildbook_install_config(webapps_dpath, unpacked_war_dpath): """ CommandLine: python -m ibeis ensure_local_war python -m ibeis update_wildbook_install_config python -m ibeis update_wildbook_install_config --show Example: >>> from ibeis.control.wildbook_manager import * # NOQA >>> import ibeis >>> tomcat_dpath = find_installed_tomcat() >>> webapps_dpath = join(tomcat_dpath, 'webapps') >>> wb_target = ibeis.const.WILDBOOK_TARGET >>> unpacked_war_dpath = join(webapps_dpath, wb_target) >>> locals_ = ut.exec_func_src(update_wildbook_install_config, globals()) >>> #update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) >>> ut.quit_if_noshow() >>> ut.vd(unpacked_war_dpath) >>> ut.editfile(locals_['permission_fpath']) >>> ut.editfile(locals_['jdoconfig_fpath']) >>> ut.editfile(locals_['asset_store_fpath']) """ mysql_mode = not ut.get_argflag('--nomysql') #if ut.get_argflag('--vd'): # ut.vd(unpacked_war_dpath) #find_installed_tomcat # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ # '/ImageSetSetMarkedIndividual = authc, roles[admin]' '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line, ) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') # Make sure we are using a non-process based database jdoconfig_fpath = join(unpacked_war_dpath, 'WEB-INF/classes/bundles/jdoconfig.properties') print('Fixing backend database config') print('jdoconfig_fpath = %r' % (jdoconfig_fpath, )) ut.assertpath(jdoconfig_fpath) jdoconfig_text = ut.readfrom(jdoconfig_fpath) #ut.vd(dirname(jdoconfig_fpath)) #ut.editfile(jdoconfig_fpath) if mysql_mode: jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', False) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', 1) mysql_user = '******' mysql_passwd = 'somepassword' mysql_dbname = 'ibeiswbtestdb' # Use mysql jdoconfig_text = re.sub('datanucleus.ConnectionUserName = .*$', 'datanucleus.ConnectionUserName = '******'datanucleus.ConnectionPassword = .*$', 'datanucleus.ConnectionPassword = '******'datanucleus.ConnectionURL *= *jdbc:mysql:.*$', 'datanucleus.ConnectionURL = jdbc:mysql://localhost:3306/' + mysql_dbname, jdoconfig_text, flags=re.MULTILINE) jdoconfig_text = re.sub('^.*jdbc:mysql://localhost:3306/shepherd.*$', '', jdoconfig_text, flags=re.MULTILINE) else: # Use SQLIIte jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', False) ut.writeto(jdoconfig_fpath, jdoconfig_text) # Need to make sure wildbook can store information in a reasonalbe place #tomcat_data_dir = join(tomcat_startup_dir, 'webapps', 'wildbook_data_dir') tomcat_data_dir = join(webapps_dpath, 'wildbook_data_dir') ut.ensuredir(tomcat_data_dir) ut.writeto(join(tomcat_data_dir, 'test.txt'), 'A hosted test file') asset_store_fpath = join(unpacked_war_dpath, 'createAssetStore.jsp') asset_store_text = ut.read_from(asset_store_fpath) #data_path_pat = ut.named_field('data_path', 'new File(".*?").toPath') new_line = 'LocalAssetStore as = new LocalAssetStore("example Local AssetStore", new File("%s").toPath(), "%s", true);' % ( tomcat_data_dir, 'http://localhost:8080/' + basename(tomcat_data_dir)) # HACKY asset_store_text2 = re.sub('^LocalAssetStore as = .*$', new_line, asset_store_text, flags=re.MULTILINE) ut.writeto(asset_store_fpath, asset_store_text2)