def _get_new_toolbox(app, save_integrated_tool_panel=True):
"""
Generate a new toolbox, by constructing a toolbox from the config files,
and then adding pre-existing data managers from the old toolbox to the new toolbox.
"""
from galaxy import tools
from galaxy.tools.special_tools import load_lib_tools
if hasattr(app, 'tool_shed_repository_cache'):
app.tool_shed_repository_cache.rebuild()
tool_configs = app.config.tool_configs
new_toolbox = tools.ToolBox(
tool_configs,
app.config.tool_path,
app,
save_integrated_tool_panel=save_integrated_tool_panel)
new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
app.datatypes_registry.load_datatype_converters(new_toolbox,
use_cached=True)
app.datatypes_registry.load_external_metadata_tool(new_toolbox)
load_lib_tools(new_toolbox)
[
new_toolbox.register_tool(tool)
for tool in new_toolbox.data_manager_tools.values()
]
app.toolbox = new_toolbox
app.toolbox.persist_cache()
def __init__(self, **kwd):
log.debug("python path is: %s", ", ".join(sys.path))
self.name = "tool_shed"
# Read the tool_shed.ini configuration file and check for errors.
self.config = config.Configuration(**kwd)
self.config.check()
configure_logging(self.config)
self.application_stack = application_stack_instance()
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root,
self.config.datatypes_config)
# Initialize the Tool Shed repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry(
)
# Initialize the RepositoryGridFilterManager.
self.repository_grid_filter_manager = RepositoryGridFilterManager()
# Determine the Tool Shed database connection string.
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize the Tool Shed database and check for appropriate schema version.
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Set up the Tool Shed database engine and ORM.
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init(self.config.file_path, db_url,
self.config.database_engine_options)
self.security = idencoding.IdEncodingHelper(
id_secret=self.config.id_secret)
# initialize the Tool Shed tag handler.
self.tag_handler = CommunityTagHandler(self)
# Initialize the Tool Shed tool data tables. Never pass a configuration file here
# because the Tool Shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(
self.config.tool_data_path)
self.genome_builds = GenomeBuilds(self)
from galaxy import auth
self.auth_manager = auth.AuthManager(self)
# Citation manager needed to load tools.
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager(self)
# The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Initialize the Tool Shed security agent.
self.security_agent = self.model.security_agent
# The Tool Shed makes no use of a quota, but this attribute is still required.
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# Initialize the baseline Tool Shed statistics component.
self.shed_counter = self.model.shed_counter
# Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry(self)
# used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base.
self.server_starttime = int(time.time())
log.debug("Tool shed hgweb.config file is: %s",
self.hgweb_config_manager.hgweb_config)
def setup_shed_tools_for_test(app, tmpdir, testing_migrated_tools, testing_installed_tools):
"""Modify Galaxy app's toolbox for migrated or installed tool tests."""
# Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
galaxy_tool_shed_test_file = os.path.join(tmpdir, 'shed_tools_dict')
shed_tools_dict = {}
if testing_migrated_tools:
has_test_data, shed_tools_dict = parse_tool_panel_config(MIGRATED_TOOL_PANEL_CONFIG, shed_tools_dict)
elif testing_installed_tools:
for shed_tool_config in INSTALLED_TOOL_PANEL_CONFIGS:
has_test_data, shed_tools_dict = parse_tool_panel_config(shed_tool_config, shed_tools_dict)
# Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
with open(galaxy_tool_shed_test_file, 'w') as shed_tools_file:
shed_tools_file.write(json.dumps(shed_tools_dict))
if not os.path.isabs(galaxy_tool_shed_test_file):
galaxy_tool_shed_test_file = os.path.join(galaxy_root, galaxy_tool_shed_test_file)
os.environ['GALAXY_TOOL_SHED_TEST_FILE'] = galaxy_tool_shed_test_file
if testing_installed_tools:
# TODO: Do this without modifying app - that is a pretty violation
# of Galaxy's abstraction - we shouldn't require app at all let alone
# be modifying it.
tool_configs = app.config.tool_configs
# Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
# and reload the app's toolbox.
relative_migrated_tool_panel_config = os.path.join(app.config.root, MIGRATED_TOOL_PANEL_CONFIG)
if relative_migrated_tool_panel_config in tool_configs:
tool_configs.remove(relative_migrated_tool_panel_config)
for installed_tool_panel_config in INSTALLED_TOOL_PANEL_CONFIGS:
tool_configs.append(installed_tool_panel_config)
from galaxy import tools # delay import because this brings in so many modules for small tests # noqa: E402
app.toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
def _get_new_toolbox(app):
"""
Generate a new toolbox, by constructing a toolbox from the config files,
and then adding pre-existing data managers from the old toolbox to the new toolbox.
"""
from galaxy import tools
from galaxy.tools.special_tools import load_lib_tools
from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
app.tool_version_cache = ToolVersionCache(
app) # Load new tools into version cache
tool_configs = app.config.tool_configs
if app.config.migrated_tools_config not in tool_configs:
tool_configs.append(app.config.migrated_tools_config)
start = time.time()
new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app,
app.toolbox._tool_conf_watcher)
new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
load_lib_tools(new_toolbox)
new_toolbox.load_hidden_lib_tool("galaxy/datatypes/set_metadata_tool.xml")
[
new_toolbox.register_tool(tool)
for tool in new_toolbox.data_manager_tools.values()
]
end = time.time() - start
log.debug("Toolbox reload took %d seconds", end)
app.reindex_tool_search(new_toolbox)
return new_toolbox
def _get_new_toolbox(app):
"""
Generate a new toolbox, by constructing a toolbox from the config files,
and then adding pre-existing data managers from the old toolbox to the new toolbox.
"""
from galaxy import tools
from galaxy.tools.special_tools import load_lib_tools
from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
if hasattr(app, 'tool_shed_repository_cache'):
app.tool_shed_repository_cache.rebuild()
app.tool_version_cache = ToolVersionCache(
app) # Load new tools into version cache
tool_configs = app.config.tool_configs
if app.config.migrated_tools_config not in tool_configs:
tool_configs.append(app.config.migrated_tools_config)
new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
app.datatypes_registry.load_datatype_converters(new_toolbox,
use_cached=True)
app.datatypes_registry.load_external_metadata_tool(new_toolbox)
load_lib_tools(new_toolbox)
[
new_toolbox.register_tool(tool)
for tool in new_toolbox.data_manager_tools.values()
]
app.toolbox = new_toolbox
def __init__( self, **kwargs ):
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
# Connect up the object model
if self.config.database_connection:
self.model = galaxy.model.mapping.init( self.config.file_path,
self.config.database_connection,
create_tables = True )
else:
self.model = galaxy.model.mapping.init( self.config.file_path,
"sqlite://%s?isolation_level=IMMEDIATE" % self.config.database,
create_tables = True )
# Initialize the tools
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path )
# Start the job queue
self.job_queue = jobs.JobQueue( self.config.job_queue_workers, self )
self.heartbeat = None
# Start the heartbeat process if configured and available
if self.config.use_heartbeat:
from galaxy import heartbeat
if heartbeat.Heartbeat:
self.heartbeat = heartbeat.Heartbeat()
self.heartbeat.start()
def __init__(self, **kwd):
print >> sys.stderr, "python path is: " + ", ".join(sys.path)
self.name = "tool_shed"
# Read the tool_shed_wsgi.ini configuration file and check for errors.
self.config = config.Configuration(**kwd)
self.config.check()
config.configure_logging(self.config)
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root,
self.config.datatypes_config)
# Initialize the Tool Shed repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry(
)
# Initialize the RepositoryGridFilterManager.
self.repository_grid_filter_manager = RepositoryGridFilterManager()
# Determine the Tool Shed database connection string.
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize the Tool Shed database and check for appropriate schema version.
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Set up the Tool Shed database engine and ORM.
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init(self.config.file_path, db_url,
self.config.database_engine_options)
# Initialize the Tool SHed security helper.
self.security = security.SecurityHelper(
id_secret=self.config.id_secret)
# initialize the Tool Shed tag handler.
self.tag_handler = CommunityTagHandler()
# Initialize the Tool Shed tool data tables. Never pass a configuration file here
# because the Tool Shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(
self.config.tool_data_path)
self.genome_builds = GenomeBuilds(self)
# Citation manager needed to load tools.
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager(self)
# The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Initialize the Tool Shed security agent.
self.security_agent = self.model.security_agent
# The Tool Shed makes no use of a quota, but this attribute is still required.
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
# Initialize the baseline Tool Shed statistics component.
self.shed_counter = self.model.shed_counter
# Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry(self)
print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def __init__( self, tools_migration_config ):
galaxy_config_file = 'universe_wsgi.ini'
if '-c' in sys.argv:
pos = sys.argv.index( '-c' )
sys.argv.pop( pos )
galaxy_config_file = sys.argv.pop( pos )
if not os.path.exists( galaxy_config_file ):
print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file
sys.exit( 1 )
config_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } )
config_parser.read( galaxy_config_file )
galaxy_config_dict = {}
for key, value in config_parser.items( "app:main" ):
galaxy_config_dict[ key ] = value
self.config = galaxy.config.Configuration( **galaxy_config_dict )
if self.config.database_connection is None:
self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
self.config.update_integrated_tool_panel = True
self.object_store = build_object_store_from_config( self.config )
# Setup the database engine and ORM
self.model = galaxy.model.mapping.init( self.config.file_path,
self.config.database_connection,
engine_options={},
create_tables=False,
object_store=self.object_store )
# Create an empty datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
# Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
tool_configs = self.config.tool_configs
if self.config.migrated_tools_config not in tool_configs:
tool_configs.append( self.config.migrated_tools_config )
self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
# Search support for tools
self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
# Set up the tool sheds registry.
if os.path.isfile( self.config.tool_sheds_config ):
self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
else:
self.tool_shed_registry = None
# Get the latest tool migration script number to send to the Install manager.
latest_migration_script_number = int( tools_migration_config.split( '_' )[ 0 ] )
# The value of migrated_tools_config is migrated_tools_conf.xml, and is reserved for containing only those tools that have been
# eliminated from the distribution and moved to the tool shed. A side-effect of instantiating the InstallManager is the automatic
# installation of all appropriate tool shed repositories.
self.install_manager = install_manager.InstallManager( app=self,
latest_migration_script_number=latest_migration_script_number,
tool_shed_install_config=os.path.join( self.config.root,
'scripts',
'migrate_tools',
tools_migration_config ),
migrated_tools_config=self.config.migrated_tools_config )
def _configure_toolbox(self):
if not isinstance(self, BasicSharedApp):
raise Exception("Must inherit from BasicSharedApp")
self.citations_manager = CitationsManager(self)
self.biotools_metadata_source = get_galaxy_biotools_metadata_source(
self.config)
self.dynamic_tools_manager = DynamicToolManager(self)
self._toolbox_lock = threading.RLock()
self.toolbox = tools.ToolBox(self.config.tool_configs,
self.config.tool_path, self)
galaxy_root_dir = os.path.abspath(self.config.root)
file_path = os.path.abspath(self.config.file_path)
app_info = AppInfo(
galaxy_root_dir=galaxy_root_dir,
default_file_path=file_path,
tool_data_path=self.config.tool_data_path,
shed_tool_data_path=self.config.shed_tool_data_path,
outputs_to_working_directory=self.config.
outputs_to_working_directory,
container_image_cache_path=self.config.container_image_cache_path,
library_import_dir=self.config.library_import_dir,
enable_mulled_containers=self.config.enable_mulled_containers,
container_resolvers_config_file=self.config.
container_resolvers_config_file,
container_resolvers_config_dict=self.config.container_resolvers,
involucro_path=self.config.involucro_path,
involucro_auto_init=self.config.involucro_auto_init,
mulled_channels=self.config.mulled_channels,
)
mulled_resolution_cache = None
if self.config.mulled_resolution_cache_type:
cache_opts = {
"cache.type": self.config.mulled_resolution_cache_type,
"cache.data_dir": self.config.mulled_resolution_cache_data_dir,
"cache.lock_dir": self.config.mulled_resolution_cache_lock_dir,
"cache.expire": self.config.mulled_resolution_cache_expire,
}
mulled_resolution_cache = CacheManager(
**parse_cache_config_options(cache_opts)).get_cache(
'mulled_resolution')
self.container_finder = containers.ContainerFinder(
app_info, mulled_resolution_cache=mulled_resolution_cache)
self._set_enabled_container_types()
index_help = getattr(self.config, "index_tool_help", True)
self.toolbox_search = ToolBoxSearch(
self.toolbox,
index_dir=self.config.tool_search_index_dir,
index_help=index_help)
def setup_shed_tools_for_test(app, tmpdir, testing_migrated_tools, testing_installed_tools):
"""Modify Galaxy app's toolbox for migrated or installed tool tests."""
if testing_installed_tools:
# TODO: Do this without modifying app - that is a pretty violation
# of Galaxy's abstraction - we shouldn't require app at all let alone
# be modifying it.
tool_configs = app.config.tool_configs
# Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
# and reload the app's toolbox.
relative_migrated_tool_panel_config = os.path.join(app.config.root, MIGRATED_TOOL_PANEL_CONFIG)
if relative_migrated_tool_panel_config in tool_configs:
tool_configs.remove(relative_migrated_tool_panel_config)
for installed_tool_panel_config in INSTALLED_TOOL_PANEL_CONFIGS:
tool_configs.append(installed_tool_panel_config)
from galaxy import tools # delay import because this brings in so many modules for small tests # noqa: E402
app.toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
def __init__(self, **kwargs):
print >> sys.stderr, "python path is: " + ", ".join(sys.path)
self.name = "community"
# Read config file and check for errors
self.config = config.Configuration(**kwargs)
self.config.check()
config.configure_logging(self.config)
# Set up datatypes registry
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root,
self.config.datatypes_config)
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize database / check for appropriate schema version
from galaxy.webapps.community.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Setup the database engine and ORM
from galaxy.webapps.community.model import mapping
self.model = mapping.init(self.config.file_path, db_url,
self.config.database_engine_options)
# Security helper
self.security = security.SecurityHelper(
id_secret=self.config.id_secret)
# Tag handler
self.tag_handler = CommunityTagHandler()
# Tool data tables - never pass a config file here because the tool shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(
self.config.tool_data_path)
# The tool shed has no toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Load security policy
self.security_agent = self.model.security_agent
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
self.shed_counter = self.model.shed_counter
# Let the HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def _configure_toolbox( self ):
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
tool_configs = self.config.tool_configs
if self.config.migrated_tools_config not in tool_configs:
tool_configs.append( self.config.migrated_tools_config )
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager( self )
from galaxy import tools
self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
self.reindex_tool_search()
from galaxy.tools.deps import containers
galaxy_root_dir = os.path.abspath(self.config.root)
file_path = os.path.abspath(getattr(self.config, "file_path"))
app_info = containers.AppInfo(
galaxy_root_dir,
default_file_path=file_path,
outputs_to_working_directory=self.config.outputs_to_working_directory,
container_image_cache_path=self.config.container_image_cache_path,
)
self.container_finder = containers.ContainerFinder(app_info)
# Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
shed_tools_file = open( galaxy_tool_shed_test_file, 'w' )
shed_tools_file.write( dumps( shed_tools_dict ) )
shed_tools_file.close()
if not os.path.isabs( galaxy_tool_shed_test_file ):
galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file )
os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file
if testing_installed_tools:
# Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
# and reload the app's toolbox.
relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config )
if relative_migrated_tool_panel_config in tool_configs:
tool_configs.remove( relative_migrated_tool_panel_config )
for installed_tool_panel_config in installed_tool_panel_configs:
tool_configs.append( installed_tool_panel_config )
app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app )
success = _run_functional_test( testing_shed_tools=True )
try:
os.unlink( tmp_tool_panel_conf )
except:
log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf )
try:
os.unlink( galaxy_tool_shed_test_file )
except:
log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file )
else:
if galaxy_test_file_dir:
os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir
success = _run_functional_test( )
except:
log.exception( "Failure running tests" )
def __init__( self, **kwargs ):
print >> sys.stderr, "python path is: " + ", ".join( sys.path )
self.new_installation = False
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize database / check for appropriate schema version. # If this
# is a new installation, we'll restrict the tool migration messaging.
from galaxy.model.migrate.check import create_or_verify_database
create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self )
# Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed.
from galaxy.tool_shed.migrate.check import verify_tools
verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options )
# Object store manager
self.object_store = build_object_store_from_config(self.config)
# Setup the database engine and ORM
from galaxy.model import mapping
self.model = mapping.init( self.config.file_path,
db_url,
self.config.database_engine_options,
database_query_profiling_proxy = self.config.database_query_profiling_proxy,
object_store = self.object_store )
# Set up the tool sheds registry
if os.path.isfile( self.config.tool_sheds_config ):
self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
else:
self.tool_shed_registry = None
# Manage installed tool shed repositories.
self.installed_repository_manager = galaxy.tool_shed.InstalledRepositoryManager( self )
# Create an empty datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
# Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories. We
# load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some
# generic sniffers (eg text,xml) which catch anything, so it's impossible for proprietary sniffers to be used.
# However, if there is a conflict (2 datatypes with the same extension) between a proprietary datatype and a datatype
# in the Galaxy distribution, the datatype in the Galaxy distribution will take precedence. If there is a conflict
# between 2 proprietary datatypes, the datatype from the repository that was installed earliest will take precedence.
# This will also load proprietary datatype converters and display applications.
self.installed_repository_manager.load_proprietary_datatypes()
# Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
galaxy.model.set_datatypes_registry( self.datatypes_registry )
# Security helper
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
# Tag handler
self.tag_handler = GalaxyTagHandler()
# Tool data tables
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path )
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
tool_configs = self.config.tool_configs
if self.config.migrated_tools_config not in tool_configs:
tool_configs.append( self.config.migrated_tools_config )
self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
# Search support for tools
self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox )
# If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories.
if self.config.get_bool( 'enable_tool_shed_check', False ):
from tool_shed import update_manager
self.update_manager = update_manager.UpdateManager( self )
# Load datatype display applications defined in local datatypes_conf.xml
self.datatypes_registry.load_display_applications()
# Load datatype converters defined in local datatypes_conf.xml
self.datatypes_registry.load_datatype_converters( self.toolbox )
# Load external metadata tool
self.datatypes_registry.load_external_metadata_tool( self.toolbox )
# Load history import/export tools.
load_history_imp_exp_tools( self.toolbox )
# Load genome indexer tool.
load_genome_index_tools( self.toolbox )
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions )
# Load quota management.
if self.config.enable_quotas:
self.quota_agent = galaxy.quota.QuotaAgent( self.model )
else:
self.quota_agent = galaxy.quota.NoQuotaAgent( self.model )
# Heartbeat and memdump for thread / heap profiling
self.heartbeat = None
self.memdump = None
self.memory_usage = None
# Container for OpenID authentication routines
if self.config.enable_openid:
from galaxy.web.framework import openid_manager
self.openid_manager = openid_manager.OpenIDManager( self.config.openid_consumer_cache_path )
self.openid_providers = OpenIDProviders.from_file( self.config.openid_config )
else:
self.openid_providers = OpenIDProviders()
# Start the heartbeat process if configured and available
if self.config.use_heartbeat:
from galaxy.util import heartbeat
if heartbeat.Heartbeat:
self.heartbeat = heartbeat.Heartbeat( fname=self.config.heartbeat_log )
self.heartbeat.start()
# Enable the memdump signal catcher if configured and available
if self.config.use_memdump:
from galaxy.util import memdump
if memdump.Memdump:
self.memdump = memdump.Memdump()
# Transfer manager client
if self.config.get_bool( 'enable_beta_job_managers', False ):
from jobs import transfer_manager
self.transfer_manager = transfer_manager.TransferManager( self )
# Start the job manager
from jobs import manager
self.job_manager = manager.JobManager( self )
# FIXME: These are exposed directly for backward compatibility
self.job_queue = self.job_manager.job_queue
self.job_stop_queue = self.job_manager.job_stop_queue
# Initialize the external service types
self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self )
def __init__( self, **kwargs ):
print >> sys.stderr, "python path is: " + ", ".join( sys.path )
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
# Set up datatypes registry
self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config )
galaxy.model.set_datatypes_registry( self.datatypes_registry )
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize database / check for appropriate schema version
from galaxy.model.migrate.check import create_or_verify_database
create_or_verify_database( db_url, self.config.database_engine_options )
# Setup the database engine and ORM
from galaxy.model import mapping
self.model = mapping.init( self.config.file_path,
db_url,
self.config.database_engine_options )
# Security helper
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
# Initialize the tools
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
# Load datatype converters
self.datatypes_registry.load_datatype_converters( self.toolbox )
#load external metadata tool
self.datatypes_registry.load_external_metadata_tool( self.toolbox )
# Load datatype indexers
self.datatypes_registry.load_datatype_indexers( self.toolbox )
#Load security policy
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions )
# Heartbeat and memdump for thread / heap profiling
self.heartbeat = None
self.memdump = None
self.memory_usage = None
# Start the heartbeat process if configured and available
if self.config.use_heartbeat:
from galaxy.util import heartbeat
if heartbeat.Heartbeat:
self.heartbeat = heartbeat.Heartbeat()
self.heartbeat.start()
# Enable the memdump signal catcher if configured and available
if self.config.use_memdump:
from galaxy.util import memdump
if memdump.Memdump:
self.memdump = memdump.Memdump()
# Enable memory_usage logging if configured
if self.config.log_memory_usage:
from galaxy.util import memory_usage
self.memory_usage = memory_usage
# Start the job queue
self.job_manager = jobs.JobManager( self )
# FIXME: These are exposed directly for backward compatibility
self.job_queue = self.job_manager.job_queue
self.job_stop_queue = self.job_manager.job_stop_queue
# Start the cloud manager
self.cloud_manager = cloud.CloudManager( self )
def setup():
"""Start the web server for the tests"""
global server, app
galaxy_test_host = os.environ.get('GALAXY_TEST_HOST',
default_galaxy_test_host)
galaxy_test_port = os.environ.get('GALAXY_TEST_PORT',
default_galaxy_test_port)
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
if start_server:
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'database', 'files')
os.makedirs(file_path)
if 'GALAXY_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_TEST_DBURI']
else:
database_connection = 'sqlite:///' + os.path.join(
tempdir, 'database', 'universe.sqlite')
app = UniverseApplication(job_queue_workers=5,
template_path="templates",
database_connection=database_connection,
file_path=file_path,
tool_config_file="tool_conf.xml",
tool_path="tools",
test_conf="test.conf",
log_destination="stdout",
use_heartbeat=True)
log.info("Embedded Universe application started")
webapp = universe_wsgi.app_factory(dict(),
use_translogger=False,
app=app)
server = galaxy.web.server.serve(webapp,
dict(),
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
atexit.register(teardown)
import threading
t = threading.Thread(target=server.serve_forever)
t.start()
time.sleep(2)
log.info("Embedded web server started")
if app:
# TODO: provisions for loading toolbox from file when using external server
import test_toolbox
test_toolbox.toolbox = app.toolbox
else:
from galaxy import tools
import test_toolbox
test_toolbox.toolbox = tools.ToolBox('tool_conf.xml', 'tools')
# Test if the server is up
import httplib
conn = httplib.HTTPConnection(galaxy_test_host, galaxy_test_port)
conn.request("GET", "/")
assert conn.getresponse(
).status == 200, "Test HTTP server did not return '200 OK'"
os.environ['GALAXY_TEST_HOST'] = galaxy_test_host
os.environ['GALAXY_TEST_PORT'] = galaxy_test_port
os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir
new_path = [os.path.join(cwd, "test")]
new_path.extend(sys.path[1:])
sys.path = new_path
import functional.test_toolbox
if app:
# TODO: provisions for loading toolbox from file when using external server
functional.test_toolbox.toolbox = app.toolbox
functional.test_toolbox.build_tests()
else:
# FIXME: This doesn't work at all now that toolbox requires an 'app' instance
# (to get at datatypes, might just pass a datatype registry directly)
my_app = bunch.Bunch(
datatypes_registry=galaxy.datatypes.registry.Registry())
test_toolbox.toolbox = tools.ToolBox('tool_conf.xml.test', 'tools',
my_app)
# ---- Find tests ---------------------------------------------------------
log.info("Functional tests will be run against %s:%s" %
(galaxy_test_host, galaxy_test_port))
success = False
try:
import nose.core
import nose.config
import nose.loader
import nose.plugins.manager