def non_shed_tool_panel_configs( self ): return common_util.get_non_shed_tool_panel_configs( self.app )
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): """ Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' raise Exception( message ) # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. tree = util.parse_xml( migrated_tools_config ) root = tree.getroot() self.tool_path = root.get( 'tool_path' ) print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) # Parse tool_shed_install_config to check each of the tools. self.tool_shed_install_config = tool_shed_install_config tree = util.parse_xml( tool_shed_install_config ) root = tree.getroot() self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) ) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config ) # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. # The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: if len( self.proprietary_tool_confs ) == 1: plural = '' file_names = self.proprietary_tool_confs[ 0 ] else: plural = 's' file_names = ', '.join( self.proprietary_tool_confs ) if missing_tool_configs_dict: for repository_elem in root: self.install_repository( repository_elem, install_dependencies ) else: message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number ) message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names ) print message else: message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number ) message += "Try again later.\n" print message
def verify_tools(app, url, galaxy_config_file, engine_options={}): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. dialect = (url.split(':', 1))[0] try: egg = dialect_to_egg[dialect] try: pkg_resources.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) meta = MetaData(bind=engine) # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. version_table = Table("migrate_tools", meta, autoload=True) # Verify that the code and the database are in sync. db_schema = schema.ControlledSchema(engine, migrate_repository) latest_tool_migration_script_number = migrate_repository.versions.latest if latest_tool_migration_script_number != db_schema.version: # The default behavior is that the tool shed is down. tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. missing_tool_configs_dict = odict() else: tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): if v: have_tool_dependencies = True break config_arg = '' if os.path.abspath(os.path.join( os.getcwd(), 'universe_wsgi.ini')) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath(os.getcwd()), '.') if not app.config.running_functional_tests: if tool_shed_accessible: # Automatically update the value of the migrate_tools.version database table column. cmd = 'sh manage_tools.sh%s upgrade' % config_arg proc = subprocess.Popen(args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return_code = proc.wait() output = proc.stdout.read(32768) if return_code != 0: raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output) elif missing_tool_configs_dict: if len(tool_panel_configs) == 1: plural = '' tool_panel_config_file_names = tool_panel_configs[0] else: plural = 's' tool_panel_config_file_names = ', '.join( tool_panel_configs) msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names) msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" if have_tool_dependencies: msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" processed_tool_dependencies = [] for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items( ): for tool_dependencies_tup in tool_dependencies: if tool_dependencies_tup not in processed_tool_dependencies: msg += "------------------------------------\n" msg += "Tool Dependency\n" msg += "------------------------------------\n" msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[0], tool_dependencies_tup[1], tool_dependencies_tup[2]) if tool_dependencies_tup[3]: msg += "Requirements and installation information:\n" msg += "%s\n" % tool_dependencies_tup[3] else: msg += "\n" msg += "------------------------------------\n" processed_tool_dependencies.append( tool_dependencies_tup) msg += "\n" msg += "%s" % output.replace('done', '') msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" if have_tool_dependencies: msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" msg += "option to the above command like this:\n\n" msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items( ): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" raise Exception(msg) else: log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version) else: log.info("At migrate_tools version %d" % db_schema.version)
def verify_tools(app, url, galaxy_config_file, engine_options={}): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. dialect = (url.split(":", 1))[0] try: egg = dialect_to_egg[dialect] try: pkg_resources.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning("%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error("database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) meta = MetaData(bind=engine) # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. version_table = Table("migrate_tools", meta, autoload=True) # Verify that the code and the database are in sync. db_schema = schema.ControlledSchema(engine, migrate_repository) latest_tool_migration_script_number = migrate_repository.versions.latest if latest_tool_migration_script_number != db_schema.version: # The default behavior is that the tool shed is down. tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. missing_tool_configs_dict = odict() else: tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): if v: have_tool_dependencies = True break config_arg = "" if os.path.abspath(os.path.join(os.getcwd(), "universe_wsgi.ini")) != galaxy_config_file: config_arg = " -c %s" % galaxy_config_file.replace(os.path.abspath(os.getcwd()), ".") if not app.config.running_functional_tests: if tool_shed_accessible: # Automatically update the value of the migrate_tools.version database table column. cmd = "sh manage_tools.sh%s upgrade" % config_arg proc = subprocess.Popen(args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return_code = proc.wait() output = proc.stdout.read(32768) if return_code != 0: raise Exception("Error attempting to update the value of migrate_tools.version: %s" % output) elif missing_tool_configs_dict: if len(tool_panel_configs) == 1: plural = "" tool_panel_config_file_names = tool_panel_configs[0] else: plural = "s" tool_panel_config_file_names = ", ".join(tool_panel_configs) msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" msg += "be removed from your file%s named %s.\n\n" % (plural, tool_panel_config_file_names) msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" if have_tool_dependencies: msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" processed_tool_dependencies = [] for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): for tool_dependencies_tup in tool_dependencies: if tool_dependencies_tup not in processed_tool_dependencies: msg += "------------------------------------\n" msg += "Tool Dependency\n" msg += "------------------------------------\n" msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[0], tool_dependencies_tup[1], tool_dependencies_tup[2], ) if tool_dependencies_tup[3]: msg += "Requirements and installation information:\n" msg += "%s\n" % tool_dependencies_tup[3] else: msg += "\n" msg += "------------------------------------\n" processed_tool_dependencies.append(tool_dependencies_tup) msg += "\n" msg += "%s" % output.replace("done", "") msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" if have_tool_dependencies: msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" msg += "option to the above command like this:\n\n" msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += ( "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number ) msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" msg += ( "Tool dependencies can be installed after the repositories have been installed as well.\n\n" ) msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" msg += ( "removed from your file%s named %s, but only after the installation process finishes.\n\n" % (plural, tool_panel_config_file_names) ) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" raise Exception(msg) else: log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version ) else: log.info("At migrate_tools version %d" % db_schema.version)
def non_shed_tool_panel_configs(self): return common_util.get_non_shed_tool_panel_configs(self.app)
def __init__(self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies): """ Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app self.toolbox = self.app.toolbox self.migrated_tools_config = migrated_tools_config # If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message. if install_dependencies and app.config.tool_dependency_dir is None: message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini ' message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.' raise Exception(message) # Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml). self.proprietary_tool_confs = self.non_shed_tool_panel_configs self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number) # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. tree = util.parse_xml(migrated_tools_config) root = tree.getroot() self.tool_path = root.get('tool_path') print "Repositories will be installed into configured tool_path location ", str( self.tool_path) # Parse tool_shed_install_config to check each of the tools. self.tool_shed_install_config = tool_shed_install_config tree = util.parse_xml(tool_shed_install_config) root = tree.getroot() self.tool_shed = suc.clean_tool_shed_url(root.get('name')) self.repository_owner = common_util.REPOSITORY_OWNER index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config) # Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in # tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk. # The default behavior is that the tool shed is down. tool_shed_accessible = False tool_panel_configs = common_util.get_non_shed_tool_panel_configs(app) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() if tool_shed_accessible: if len(self.proprietary_tool_confs) == 1: plural = '' file_names = self.proprietary_tool_confs[0] else: plural = 's' file_names = ', '.join(self.proprietary_tool_confs) if missing_tool_configs_dict: for repository_elem in root: self.install_repository(repository_elem, install_dependencies) else: message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number) message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names) print message else: message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number) message += "Try again later.\n" print message