def check_db(config_parser):
dburi = None
if config_parser.has_option("app:main", "database_connection"):
dburi = config_parser.get("app:main", "database_connection")
elif config_parser.has_option("app:main", "database_file"):
db_file = config_parser.get("app:main", "database_file")
dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
else:
sys.exit(
"The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap."
)
sa_session = None
database_exists_message = (
"The database configured for this Tool Shed is not new, so bootstrapping is not allowed. "
)
database_exists_message += "Create a new database that has not been migrated before attempting to bootstrap."
try:
model = tool_shed_model.init(
config_parser.get("app:main", "file_path"), dburi, engine_options={}, create_tables=False
)
sa_session = model.context.current
sys.exit(database_exists_message)
except ProgrammingError:
pass
except OperationalError:
pass
try:
if sa_session is not None:
result = sa_session.execute("SELECT version FROM migrate_version").first()
if result[0] >= 2:
sys.exit(database_exists_message)
else:
pass
except ProgrammingError:
pass
if config_parser.has_option("app:main", "hgweb_config_dir"):
hgweb_config_parser = ConfigParser.ConfigParser()
hgweb_dir = config_parser.get("app:main", "hgweb_config_dir")
hgweb_config_file = os.path.join(hgweb_dir, "hgweb.config")
if not os.path.exists(hgweb_config_file):
sys.exit(0)
hgweb_config_parser.read(hgweb_config_file)
configured_repos = hgweb_config_parser.items("paths")
if len(configured_repos) >= 1:
message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete"
message += " the current hgweb.config file along with all associated repositories in the configured "
message += "location before attempting to boostrap."
sys.exit(message)
else:
sys.exit(0)
else:
sys.exit(0)
sys.exit(0)
def check_db(config_parser):
dburi = None
if config_parser.has_option('app:main', 'database_connection'):
dburi = config_parser.get('app:main', 'database_connection')
elif config_parser.has_option('app:main', 'database_file'):
db_file = config_parser.get('app:main', 'database_file')
dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
else:
print 'The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.'
exit(1)
sa_session = None
database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. '
database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.'
try:
model = tool_shed_model.init(config_parser.get('app:main',
'file_path'),
dburi,
engine_options={},
create_tables=False)
sa_session = model.context.current
print database_exists_message
exit(1)
except ProgrammingError, e:
pass
def __init__(self, **kwd):
log.debug("python path is: %s", ", ".join(sys.path))
self.name = "tool_shed"
# Read the tool_shed.ini configuration file and check for errors.
self.config = config.Configuration(**kwd)
self.config.check()
configure_logging(self.config)
self.application_stack = application_stack_instance()
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root,
self.config.datatypes_config)
# Initialize the Tool Shed repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry(
)
# Initialize the RepositoryGridFilterManager.
self.repository_grid_filter_manager = RepositoryGridFilterManager()
# Determine the Tool Shed database connection string.
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize the Tool Shed database and check for appropriate schema version.
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Set up the Tool Shed database engine and ORM.
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init(self.config.file_path, db_url,
self.config.database_engine_options)
self.security = idencoding.IdEncodingHelper(
id_secret=self.config.id_secret)
# initialize the Tool Shed tag handler.
self.tag_handler = CommunityTagHandler(self)
# Initialize the Tool Shed tool data tables. Never pass a configuration file here
# because the Tool Shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(
self.config.tool_data_path)
self.genome_builds = GenomeBuilds(self)
from galaxy import auth
self.auth_manager = auth.AuthManager(self)
# Citation manager needed to load tools.
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager(self)
# The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Initialize the Tool Shed security agent.
self.security_agent = self.model.security_agent
# The Tool Shed makes no use of a quota, but this attribute is still required.
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# Initialize the baseline Tool Shed statistics component.
self.shed_counter = self.model.shed_counter
# Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry(self)
# used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base.
self.server_starttime = int(time.time())
log.debug("Tool shed hgweb.config file is: %s",
self.hgweb_config_manager.hgweb_config)
def check_db(config_parser):
dburi = None
if config_parser.has_option("app:main", "database_connection"):
dburi = config_parser.get("app:main", "database_connection")
elif config_parser.has_option("app:main", "database_file"):
db_file = config_parser.get("app:main", "database_file")
dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
else:
print "The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap."
exit(1)
sa_session = None
database_exists_message = (
"The database configured for this Tool Shed is not new, so bootstrapping is not allowed. "
)
database_exists_message += "Create a new database that has not been migrated before attempting to bootstrap."
try:
model = tool_shed_model.init(
config_parser.get("app:main", "file_path"), dburi, engine_options={}, create_tables=False
)
sa_session = model.context.current
print database_exists_message
exit(1)
except ProgrammingError, e:
pass
def __init__(self, **kwd):
log.debug("python path is: %s", ", ".join(sys.path))
self.name = "tool_shed"
# Read the tool_shed.ini configuration file and check for errors.
self.config = config.Configuration(**kwd)
self.config.check()
configure_logging(self.config)
self.application_stack = application_stack_instance()
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root, self.config.datatypes_config)
# Initialize the Tool Shed repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry()
# Initialize the RepositoryGridFilterManager.
self.repository_grid_filter_manager = RepositoryGridFilterManager()
# Determine the Tool Shed database connection string.
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize the Tool Shed database and check for appropriate schema version.
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Set up the Tool Shed database engine and ORM.
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init(self.config.file_path,
db_url,
self.config.database_engine_options)
# Initialize the Tool Shed security helper.
self.security = security.SecurityHelper(id_secret=self.config.id_secret)
# initialize the Tool Shed tag handler.
self.tag_handler = CommunityTagManager(self)
# Initialize the Tool Shed tool data tables. Never pass a configuration file here
# because the Tool Shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path)
self.genome_builds = GenomeBuilds(self)
from galaxy import auth
self.auth_manager = auth.AuthManager(self)
# Citation manager needed to load tools.
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager(self)
# The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Initialize the Tool Shed security agent.
self.security_agent = self.model.security_agent
# The Tool Shed makes no use of a quota, but this attribute is still required.
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
# Initialize the baseline Tool Shed statistics component.
self.shed_counter = self.model.shed_counter
# Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry(self)
# used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base.
self.server_starttime = int(time.time())
log.debug("Tool shed hgweb.config file is: %s", self.hgweb_config_manager.hgweb_config)
def __init__(self, **kwd):
print >> sys.stderr, "python path is: " + ", ".join(sys.path)
self.name = "tool_shed"
# Read the tool_shed_wsgi.ini configuration file and check for errors.
self.config = config.Configuration(**kwd)
self.config.check()
config.configure_logging(self.config)
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes(self.config.root,
self.config.datatypes_config)
# Initialize the Tool Shed repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry(
)
# Initialize the RepositoryGridFilterManager.
self.repository_grid_filter_manager = RepositoryGridFilterManager()
# Determine the Tool Shed database connection string.
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize the Tool Shed database and check for appropriate schema version.
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database(db_url, self.config.database_engine_options)
# Set up the Tool Shed database engine and ORM.
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init(self.config.file_path, db_url,
self.config.database_engine_options)
# Initialize the Tool SHed security helper.
self.security = security.SecurityHelper(
id_secret=self.config.id_secret)
# initialize the Tool Shed tag handler.
self.tag_handler = CommunityTagHandler()
# Initialize the Tool Shed tool data tables. Never pass a configuration file here
# because the Tool Shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(
self.config.tool_data_path)
self.genome_builds = GenomeBuilds(self)
# Citation manager needed to load tools.
from galaxy.managers.citations import CitationsManager
self.citations_manager = CitationsManager(self)
# The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox([], self.config.tool_path, self)
# Initialize the Tool Shed security agent.
self.security_agent = self.model.security_agent
# The Tool Shed makes no use of a quota, but this attribute is still required.
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
# Initialize the baseline Tool Shed statistics component.
self.shed_counter = self.model.shed_counter
# Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry(self)
print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def check_db(config_parser):
dburi = None
if config_parser.has_option('app:main', 'database_connection'):
dburi = config_parser.get('app:main', 'database_connection')
elif config_parser.has_option('app:main', 'database_file'):
db_file = config_parser.get('app:main', 'database_file')
dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file
else:
sys.exit('The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.')
sa_session = None
database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. '
database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.'
try:
model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False)
sa_session = model.context.current
sys.exit(database_exists_message)
except ProgrammingError:
pass
except OperationalError:
pass
try:
if sa_session is not None:
result = sa_session.execute('SELECT version FROM migrate_version').first()
if result[0] >= 2:
sys.exit(database_exists_message)
else:
pass
except ProgrammingError:
pass
if config_parser.has_option('app:main', 'hgweb_config_dir'):
hgweb_config_parser = ConfigParser()
hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir')
hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config')
if not os.path.exists(hgweb_config_file):
sys.exit(0)
hgweb_config_parser.read(hgweb_config_file)
configured_repos = hgweb_config_parser.items('paths')
if len(configured_repos) >= 1:
message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete"
message += " the current hgweb.config file along with all associated repositories in the configured "
message += "location before attempting to boostrap."
sys.exit(message)
else:
sys.exit(0)
else:
sys.exit(0)
sys.exit(0)
def __init__(self, config):
self.config = config
if not self.config.database_connection:
self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str(config.database)
print 'Using database connection: ', self.config.database_connection
# Setup the database engine and ORM
self.model = mapping.init(self.config.file_path,
self.config.database_connection,
engine_options={},
create_tables=False)
self.security = security.SecurityHelper(id_secret=self.config.id_secret)
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
print 'Using hgweb.config file: ', self.hgweb_config_manager.hgweb_config
def __init__( self, **kwargs ):
print >> sys.stderr, "python path is: " + ", ".join( sys.path )
self.name = "tool_shed"
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
# Set up datatypes registry
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
# Set up the repository_types registry.
self.repository_types_registry = tool_shed.repository_types.registry.Registry()
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
else:
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
# Initialize database / check for appropriate schema version
from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database
create_or_verify_database( db_url, self.config.database_engine_options )
# Setup the database engine and ORM
from galaxy.webapps.tool_shed.model import mapping
self.model = mapping.init( self.config.file_path,
db_url,
self.config.database_engine_options )
# Security helper
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
# Tag handler
self.tag_handler = CommunityTagHandler()
# Tool data tables - never pass a config file here because the tool shed should always have an empty dictionary!
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path )
# The tool shed has no toolbox, but this attribute is still required.
self.toolbox = tools.ToolBox( [], self.config.tool_path, self )
# Load security policy
self.security_agent = self.model.security_agent
self.quota_agent = galaxy.quota.NoQuotaAgent( self.model )
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
self.shed_counter = self.model.shed_counter
# Let the HgwebConfigManager know where the hgweb.config file is located.
self.hgweb_config_manager = self.model.hgweb_config_manager
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi,
**kwargs):
"""
Build two search indexes simultaneously
One is for repositories and the other for tools.
"""
model = ts_mapping.init(file_path,
dburi,
engine_options={},
create_tables=False)
sa_session = model.context.current
# Rare race condition exists here and below
tool_index_dir = os.path.join(whoosh_index_dir, 'tools')
if not os.path.exists(whoosh_index_dir):
os.makedirs(whoosh_index_dir)
os.makedirs(tool_index_dir)
work_repo_dir = whoosh_index_dir
work_tool_dir = tool_index_dir
else:
# Index exists, prevent in-place index regeneration
work_repo_dir = tempfile.mkdtemp(prefix="tmp-whoosh-repo")
work_tool_dir = tempfile.mkdtemp(prefix="tmp-whoosh-tool")
repo_index_storage = FileStorage(work_repo_dir)
tool_index_storage = FileStorage(work_tool_dir)
repo_index = repo_index_storage.create_index(repo_schema)
tool_index = tool_index_storage.create_index(tool_schema)
repo_index_writer = repo_index.writer()
tool_index_writer = tool_index.writer()
repos_indexed = 0
tools_indexed = 0
execution_timer = ExecutionTimer()
for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):
repo_index_writer.add_document(
id=repo.get('id'),
name=unicodify(repo.get('name')),
description=unicodify(repo.get('description')),
long_description=unicodify(repo.get('long_description')),
homepage_url=unicodify(repo.get('homepage_url')),
remote_repository_url=unicodify(repo.get('remote_repository_url')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
categories=unicodify(repo.get('categories')),
times_downloaded=repo.get('times_downloaded'),
approved=repo.get('approved'),
last_updated=repo.get('last_updated'),
full_last_updated=repo.get('full_last_updated'),
repo_lineage=unicodify(repo.get('repo_lineage')))
# Tools get their own index
for tool in repo.get('tools_list'):
tool_index_writer.add_document(
id=unicodify(tool.get('id')),
name=unicodify(tool.get('name')),
version=unicodify(tool.get('version')),
description=unicodify(tool.get('description')),
help=unicodify(tool.get('help')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
repo_name=unicodify(repo.get('name')),
repo_id=repo.get('id'))
tools_indexed += 1
repos_indexed += 1
tool_index_writer.commit()
repo_index_writer.commit()
log.info("Indexed repos: %s, tools: %s", repos_indexed, tools_indexed)
log.info("Toolbox index finished %s", execution_timer)
# Copy the built indexes if we were working in a tmp folder.
if work_repo_dir is not whoosh_index_dir:
shutil.rmtree(whoosh_index_dir)
os.makedirs(whoosh_index_dir)
os.makedirs(tool_index_dir)
copy_tree(work_repo_dir, whoosh_index_dir)
copy_tree(work_tool_dir, tool_index_dir)
shutil.rmtree(work_repo_dir)
