def ensure_explicit_namespace(fpath, namespace, varname_list): import re import utool as ut text = ut.read_from(fpath) orig_text = text new_text = text for varname in varname_list: regex = ''.join(( ut.named_field('prefix', '[^.]'), ut.named_field('var', ut.whole_word(varname)), )) repl = ''.join(( ut.bref_field('prefix'), namespace, '.', ut.bref_field('var') )) new_text = re.sub(regex, repl, new_text) textdiff = ut.get_textdiff(orig_text, new_text) print(textdiff) if ut.user_cmdline_prompt('Does the text look good?'): # if diff looks good write ut.write_to(fpath, new_text)
def preprocess_research(input_str): """ test of an em --- dash test of an em — dash """ import utool as ut inside = ut.named_field('ref', '.*?') input_str = re.sub(r'\\emph{' + inside + '}', ut.bref_field('ref'), input_str) # input_str = input_str.decode('utf-8') input_str = ut.ensure_unicode(input_str) pause = re.escape(' <break time="300ms"/> ') # pause = ', ' emdash = u'\u2014' # # print('input_str = %r' % (input_str,)) # print('emdash = %r' % (emdash,)) # print('emdash = %s' % (emdash,)) input_str = re.sub('\s?' + re.escape('---') + '\s?', pause, input_str) input_str = re.sub('\s?' + emdash + '\s?', pause, input_str) # print('input_str = %r' % (input_str,)) input_str = re.sub('\\\\cite{[^}]*}', '', input_str) input_str = re.sub('et al.', 'et all', input_str) # Let rob say et al. input_str = re.sub(' i\.e\.', ' i e ' + pause, input_str) # Let rob say et al. input_str = re.sub(r'\\r', '', input_str) # input_str = re.sub(r'\\n', '', input_str) # input_str = re.sub('\\\\', '', input_str) # #input_str = re.sub('[a-z]?[a-z]', 'et all', input_str) # Let rob say et al. input_str = re.sub('\\.[^a-zA-Z0-1]+', '.\n', input_str) # Split the document at periods input_str = re.sub('\r\n', '\n', input_str) input_str = re.sub('^ *$\n', '', input_str) input_str = re.sub('\n\n*', '\n', input_str) return input_str
def update_wildbook_ia_config(ibs, wildbook_tomcat_path, dryrun=False): """ #if use_config_file and wildbook_tomcat_path: # # Update the Wildbook configuration to see *THIS* wbia database # with lockfile.LockFile(lock_fpath): # update_wildbook_ia_config(ibs, wildbook_tomcat_path, dryrun) """ wildbook_properteis_dpath = join(wildbook_tomcat_path, 'WEB-INF/classes/bundles/') logger.info('[ibs.update_wildbook_ia_config()] Wildbook properties=%r' % (wildbook_properteis_dpath, )) # The src file is non-standard. It should be remove here as well wildbook_config_fpath_dst = join(wildbook_properteis_dpath, 'commonConfiguration.properties') ut.assert_exists(wildbook_properteis_dpath) # for come reason the .default file is not there, that should be ok though orig_content = ut.read_from(wildbook_config_fpath_dst) content = orig_content # Make sure wildbook knows where to find us if False: # Old way of telling WB where to find IA content = re.sub('IBEIS_DB_path = .*', 'IBEIS_DB_path = ' + ibs.get_db_core_path(), content) content = re.sub('IBEIS_image_path = .*', 'IBEIS_image_path = ' + ibs.get_imgdir(), content) web_port = ibs.get_web_port_via_scan() if web_port is None: raise ValueError('IA web server is not running on any expected port') ia_hostport = 'http://localhost:%s' % (web_port, ) ia_rest_prefix = ut.named_field('prefix', 'IBEISIARestUrl.*') host_port = ut.named_field('host_port', 'http://.*?:[0-9]+') content = re.sub(ia_rest_prefix + host_port, ut.bref_field('prefix') + ia_hostport, content) # Write to the configuration if it is different if orig_content != content: need_sudo = not ut.is_file_writable(wildbook_config_fpath_dst) if need_sudo: quoted_content = '"%s"' % (content, ) logger.info( 'Attempting to gain sudo access to update wildbook config') command = [ 'sudo', 'sh', '-c', "'", 'echo', quoted_content, '>', wildbook_config_fpath_dst, "'", ] # ut.cmd(command, sudo=True) command = ' '.join(command) if not dryrun: os.system(command) else: ut.write_to(wildbook_config_fpath_dst, content)
def ensure_explicit_namespace(fpath, namespace, varname_list): import re import utool as ut text = ut.read_from(fpath) orig_text = text new_text = text for varname in varname_list: regex = ''.join(( ut.named_field('prefix', '[^.]'), ut.named_field('var', ut.whole_word(varname)), )) repl = ''.join( (ut.bref_field('prefix'), namespace, '.', ut.bref_field('var'))) new_text = re.sub(regex, repl, new_text) textdiff = ut.get_textdiff(orig_text, new_text) print(textdiff) if ut.user_cmdline_prompt('Does the text look good?'): # if diff looks good write ut.write_to(fpath, new_text)
def _rectify_text(self, text): text = text.replace('±', '\\pm') # Put all numbers in math mode pat = ( # ut.negative_lookbehind('[A-Za-z]') ut.named_field('num', '[0-9.]+(\\\\pm)?[0-9.]*') # + ut.negative_lookahead('[A-Za-z]') + '') text2 = re.sub(pat, '$' + ut.bref_field('num') + '$', text) # if True: # # def _boldface_best(): # # pass # # text2 = re.sub(pat, '$' + ut.bref_field('num') + '$', text2) # latex_str = re.sub(' -0.00 ', ' 0.00 ', latex_str) return text2
def update_wildbook_ia_config(ibs, wildbook_tomcat_path, dryrun=False): """ #if use_config_file and wildbook_tomcat_path: # # Update the Wildbook configuration to see *THIS* ibeis database # with lockfile.LockFile(lock_fpath): # update_wildbook_ia_config(ibs, wildbook_tomcat_path, dryrun) """ wildbook_properteis_dpath = join(wildbook_tomcat_path, 'WEB-INF/classes/bundles/') print('[ibs.update_wildbook_ia_config()] Wildbook properties=%r' % ( wildbook_properteis_dpath, )) # The src file is non-standard. It should be remove here as well wildbook_config_fpath_dst = join(wildbook_properteis_dpath, 'commonConfiguration.properties') ut.assert_exists(wildbook_properteis_dpath) # for come reason the .default file is not there, that should be ok though orig_content = ut.read_from(wildbook_config_fpath_dst) content = orig_content # Make sure wildbook knows where to find us if False: # Old way of telling WB where to find IA content = re.sub('IBEIS_DB_path = .*', 'IBEIS_DB_path = ' + ibs.get_db_core_path(), content) content = re.sub('IBEIS_image_path = .*', 'IBEIS_image_path = ' + ibs.get_imgdir(), content) ia_hostport = 'http://localhost:5000' ia_rest_prefix = ut.named_field('prefix', 'IBEISIARestUrl.*') host_port = ut.named_field('host_port', 'http://.*?:[0-9]+') content = re.sub(ia_rest_prefix + host_port, ut.bref_field('prefix') + ia_hostport, content) # Write to the configuration if it is different if orig_content != content: need_sudo = not ut.is_file_writable(wildbook_config_fpath_dst) if need_sudo: quoted_content = '"%s"' % (content, ) print('Attempting to gain sudo access to update wildbook config') command = ['sudo', 'sh', '-c', '\'', 'echo', quoted_content, '>', wildbook_config_fpath_dst, '\''] # ut.cmd(command, sudo=True) command = ' '.join(command) if not dryrun: os.system(command) else: ut.write_to(wildbook_config_fpath_dst, content)
def update_wildbook_install_config(webapps_dpath, unpacked_war_dpath): """ CommandLine: python -m ibeis ensure_local_war python -m ibeis update_wildbook_install_config python -m ibeis update_wildbook_install_config --show Example: >>> from ibeis.control.wildbook_manager import * # NOQA >>> import ibeis >>> tomcat_dpath = find_installed_tomcat() >>> webapps_dpath = join(tomcat_dpath, 'webapps') >>> wb_target = ibeis.const.WILDBOOK_TARGET >>> unpacked_war_dpath = join(webapps_dpath, wb_target) >>> locals_ = ut.exec_func_src(update_wildbook_install_config, globals()) >>> #update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) >>> ut.quit_if_noshow() >>> ut.vd(unpacked_war_dpath) >>> ut.editfile(locals_['permission_fpath']) >>> ut.editfile(locals_['jdoconfig_fpath']) >>> ut.editfile(locals_['asset_store_fpath']) """ mysql_mode = not ut.get_argflag('--nomysql') #if ut.get_argflag('--vd'): # ut.vd(unpacked_war_dpath) #find_installed_tomcat # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ # '/ImageSetSetMarkedIndividual = authc, roles[admin]' '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line, ) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') # Make sure we are using a non-process based database jdoconfig_fpath = join(unpacked_war_dpath, 'WEB-INF/classes/bundles/jdoconfig.properties') print('Fixing backend database config') print('jdoconfig_fpath = %r' % (jdoconfig_fpath, )) ut.assertpath(jdoconfig_fpath) jdoconfig_text = ut.readfrom(jdoconfig_fpath) #ut.vd(dirname(jdoconfig_fpath)) #ut.editfile(jdoconfig_fpath) if mysql_mode: jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', False) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', 1) mysql_user = '******' mysql_passwd = 'somepassword' mysql_dbname = 'ibeiswbtestdb' # Use mysql jdoconfig_text = re.sub('datanucleus.ConnectionUserName = .*$', 'datanucleus.ConnectionUserName = '******'datanucleus.ConnectionPassword = .*$', 'datanucleus.ConnectionPassword = '******'datanucleus.ConnectionURL *= *jdbc:mysql:.*$', 'datanucleus.ConnectionURL = jdbc:mysql://localhost:3306/' + mysql_dbname, jdoconfig_text, flags=re.MULTILINE) jdoconfig_text = re.sub('^.*jdbc:mysql://localhost:3306/shepherd.*$', '', jdoconfig_text, flags=re.MULTILINE) else: # Use SQLIIte jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', False) ut.writeto(jdoconfig_fpath, jdoconfig_text) # Need to make sure wildbook can store information in a reasonalbe place #tomcat_data_dir = join(tomcat_startup_dir, 'webapps', 'wildbook_data_dir') tomcat_data_dir = join(webapps_dpath, 'wildbook_data_dir') ut.ensuredir(tomcat_data_dir) ut.writeto(join(tomcat_data_dir, 'test.txt'), 'A hosted test file') asset_store_fpath = join(unpacked_war_dpath, 'createAssetStore.jsp') asset_store_text = ut.read_from(asset_store_fpath) #data_path_pat = ut.named_field('data_path', 'new File(".*?").toPath') new_line = 'LocalAssetStore as = new LocalAssetStore("example Local AssetStore", new File("%s").toPath(), "%s", true);' % ( tomcat_data_dir, 'http://localhost:8080/' + basename(tomcat_data_dir)) # HACKY asset_store_text2 = re.sub('^LocalAssetStore as = .*$', new_line, asset_store_text, flags=re.MULTILINE) ut.writeto(asset_store_fpath, asset_store_text2)
def install_wildbook(verbose=ut.NOT_QUIET): """ Script to setup wildbook on a unix based system (hopefully eventually this will generalize to win32) CommandLine: # Reset python -m ibeis --tf reset_local_wildbook # Setup python -m ibeis --tf install_wildbook # Startup python -m ibeis --tf startup_wildbook_server --show --exec-mode # Reset python -m ibeis.control.manual_wildbook_funcs --test-reset_local_wildbook # Setup python -m ibeis.control.manual_wildbook_funcs --test-install_wildbook # Startup python -m ibeis.control.manual_wildbook_funcs --test-startup_wildbook_server --show --exec-mode Example: >>> # SCRIPT >>> from ibeis.control.manual_wildbook_funcs import * # NOQA >>> verbose = True >>> result = install_wildbook() >>> print(result) """ # TODO: allow custom specified tomcat directory from os.path import basename, splitext, join import time import re import subprocess try: output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) _java_version = output.split('\n')[0] _java_version = _java_version.replace('java version ', '') java_version = _java_version.replace('"', '') print('java_version = %r' % (java_version,)) if not java_version.startswith('1.7'): print('Warning wildbook is only supported for java 1.7') except OSError: output = None if output is None: raise ImportError( 'Cannot find java on this machine. ' 'Please install java: http://www.java.com/en/download/') tomcat_dpath = find_or_download_tomcat() assert tomcat_dpath is not None, 'Could not find tomcat' war_fpath = find_or_download_wilbook_warfile() war_fname = basename(war_fpath) wb_target = splitext(war_fname)[0] # Ensure environment variables #os.environ['JAVA_HOME'] = find_java_jvm() #os.environ['TOMCAT_HOME'] = tomcat_dpath #os.environ['CATALINA_HOME'] = tomcat_dpath # Move the war file to tomcat webapps if not there webapps_dpath = join(tomcat_dpath, 'webapps') deploy_war_fpath = join(webapps_dpath, war_fname) if not ut.checkpath(deploy_war_fpath, verbose=verbose): ut.copy(war_fpath, deploy_war_fpath) # Ensure that the war file has been unpacked unpacked_war_dpath = join(webapps_dpath, wb_target) if not ut.checkpath(unpacked_war_dpath, verbose=verbose): # Need to make sure you start catalina in the same directory otherwise # the derby databsae gets put in in cwd tomcat_startup_dir = get_tomcat_startup_tmpdir() with ut.ChdirContext(tomcat_startup_dir): # Starting and stoping catalina should be sufficient to unpack the # war startup_fpath = join(tomcat_dpath, 'bin', 'startup.sh') shutdown_fpath = join(tomcat_dpath, 'bin', 'shutdown.sh') ut.cmd(ut.quote_single_command(startup_fpath)) print('It is NOT ok if the startup.sh fails\n') # wait for the war to be unpacked for retry_count in range(0, 6): time.sleep(1) if ut.checkpath(unpacked_war_dpath, verbose=True): break else: print('Retrying') # ensure that the server is ruuning import requests print('Checking if we can ping the server') response = requests.get('http://localhost:8080') if response is None or response.status_code != 200: print('There may be an error starting the server') else: print('Seem able to ping the server') # assert tht the war was unpacked ut.assertpath(unpacked_war_dpath, ( 'Wildbook war might have not unpacked correctly. This may ' 'be ok. Try again. If it fails a second time, then there is a ' 'problem.'), verbose=True) # shutdown the server ut.cmd(ut.quote_single_command(shutdown_fpath)) print('It is ok if the shutdown.sh fails') time.sleep(.5) # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line,) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') print('Wildbook is installed and waiting to be started')
def update_wildbook_install_config(webapps_dpath, unpacked_war_dpath): """ CommandLine: python -m ibeis ensure_local_war python -m ibeis update_wildbook_install_config python -m ibeis update_wildbook_install_config --show Example: >>> from ibeis.control.wildbook_manager import * # NOQA >>> import ibeis >>> tomcat_dpath = find_installed_tomcat() >>> webapps_dpath = join(tomcat_dpath, 'webapps') >>> wb_target = ibeis.const.WILDBOOK_TARGET >>> unpacked_war_dpath = join(webapps_dpath, wb_target) >>> locals_ = ut.exec_func_src(update_wildbook_install_config, globals()) >>> #update_wildbook_install_config(webapps_dpath, unpacked_war_dpath) >>> ut.quit_if_noshow() >>> ut.vd(unpacked_war_dpath) >>> ut.editfile(locals_['permission_fpath']) >>> ut.editfile(locals_['jdoconfig_fpath']) >>> ut.editfile(locals_['asset_store_fpath']) """ mysql_mode = not ut.get_argflag('--nomysql') #if ut.get_argflag('--vd'): # ut.vd(unpacked_war_dpath) #find_installed_tomcat # Make sure permissions are correctly set in wildbook # Comment out the line that requires authentication permission_fpath = join(unpacked_war_dpath, 'WEB-INF/web.xml') ut.assertpath(permission_fpath) permission_text = ut.readfrom(permission_fpath) lines_to_remove = [ # '/ImageSetSetMarkedIndividual = authc, roles[admin]' '/EncounterSetMarkedIndividual = authc, roles[admin]' ] new_permission_text = permission_text[:] for line in lines_to_remove: re.search(re.escape(line), permission_text) prefix = ut.named_field('prefix', '\\s*') suffix = ut.named_field('suffix', '\\s*\n') pattern = ('^' + prefix + re.escape(line) + suffix) match = re.search(pattern, permission_text, flags=re.MULTILINE | re.DOTALL) if match is None: continue newline = '<!--%s -->' % (line,) repl = ut.bref_field('prefix') + newline + ut.bref_field('suffix') new_permission_text = re.sub(pattern, repl, permission_text, flags=re.MULTILINE | re.DOTALL) assert new_permission_text != permission_text, ( 'text should have changed') if new_permission_text != permission_text: print('Need to write new permission texts') ut.writeto(permission_fpath, new_permission_text) else: print('Permission file seems to be ok') # Make sure we are using a non-process based database jdoconfig_fpath = join(unpacked_war_dpath, 'WEB-INF/classes/bundles/jdoconfig.properties') print('Fixing backend database config') print('jdoconfig_fpath = %r' % (jdoconfig_fpath,)) ut.assertpath(jdoconfig_fpath) jdoconfig_text = ut.readfrom(jdoconfig_fpath) #ut.vd(dirname(jdoconfig_fpath)) #ut.editfile(jdoconfig_fpath) if mysql_mode: jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', False) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', 1) mysql_user = '******' mysql_passwd = 'somepassword' mysql_dbname = 'ibeiswbtestdb' # Use mysql jdoconfig_text = re.sub( 'datanucleus.ConnectionUserName = .*$', 'datanucleus.ConnectionUserName = '******'datanucleus.ConnectionPassword = .*$', 'datanucleus.ConnectionPassword = '******'datanucleus.ConnectionURL *= *jdbc:mysql:.*$', 'datanucleus.ConnectionURL = jdbc:mysql://localhost:3306/' + mysql_dbname, jdoconfig_text, flags=re.MULTILINE) jdoconfig_text = re.sub( '^.*jdbc:mysql://localhost:3306/shepherd.*$', '', jdoconfig_text, flags=re.MULTILINE) else: # Use SQLIIte jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'derby', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'mysql', 1) jdoconfig_text = ut.toggle_comment_lines(jdoconfig_text, 'sqlite', False) ut.writeto(jdoconfig_fpath, jdoconfig_text) # Need to make sure wildbook can store information in a reasonalbe place #tomcat_data_dir = join(tomcat_startup_dir, 'webapps', 'wildbook_data_dir') tomcat_data_dir = join(webapps_dpath, 'wildbook_data_dir') ut.ensuredir(tomcat_data_dir) ut.writeto(join(tomcat_data_dir, 'test.txt'), 'A hosted test file') asset_store_fpath = join(unpacked_war_dpath, 'createAssetStore.jsp') asset_store_text = ut.read_from(asset_store_fpath) #data_path_pat = ut.named_field('data_path', 'new File(".*?").toPath') new_line = 'LocalAssetStore as = new LocalAssetStore("example Local AssetStore", new File("%s").toPath(), "%s", true);' % ( tomcat_data_dir, 'http://localhost:8080/' + basename(tomcat_data_dir) ) # HACKY asset_store_text2 = re.sub('^LocalAssetStore as = .*$', new_line, asset_store_text, flags=re.MULTILINE) ut.writeto(asset_store_fpath, asset_store_text2)