def __init__(self, **kwd): log.debug("python path is: %s", ", ".join(sys.path)) self.name = "tool_shed" # will be overwritten when building WSGI app self.is_webapp = False # Read the tool_shed.ini configuration file and check for errors. self.config = config.Configuration(**kwd) self.config.check() configure_logging(self.config) self.application_stack = application_stack_instance() # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes(self.config.root, self.config.datatypes_config) # Initialize the Tool Shed repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry() # Initialize the RepositoryGridFilterManager. self.repository_grid_filter_manager = RepositoryGridFilterManager() # Determine the Tool Shed database connection string. if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize the Tool Shed database and check for appropriate schema version. from tool_shed.webapp.model.migrate.check import create_or_verify_database create_or_verify_database(db_url, self.config.database_engine_options) # Set up the Tool Shed database engine and ORM. from tool_shed.webapp.model import mapping self.model = mapping.init(self.config.file_path, db_url, self.config.database_engine_options) self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagHandler(self) # Initialize the Tool Shed tool data tables. Never pass a configuration file here # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path) self.genome_builds = GenomeBuilds(self) from galaxy import auth self.auth_manager = auth.AuthManager(self) # Citation manager needed to load tools. from galaxy.managers.citations import CitationsManager self.citations_manager = CitationsManager(self) self.use_tool_dependency_resolution = False # Initialize the Tool Shed security agent. self.security_agent = self.model.security_agent # The Tool Shed makes no use of a quota, but this attribute is still required. self.quota_agent = galaxy.quota.NoQuotaAgent(self.model) # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. self.server_starttime = int(time.time()) log.debug("Tool shed hgweb.config file is: %s", self.hgweb_config_manager.hgweb_config)
def __init__(self, config): self.config = config if not self.config.database_connection: self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str( config.database) print('Using database connection: ', self.config.database_connection) # Setup the database engine and ORM self.model = mapping.init(self.config.file_path, self.config.database_connection, engine_options={}, create_tables=False) self.security = IdEncodingHelper(id_secret=self.config.id_secret)
def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. Returns a tuple with number of repos and tools that were indexed. """ model = ts_mapping.init(file_path, dburi, engine_options={}, create_tables=False) sa_session = model.context.current repo_index, tool_index = _get_or_create_index(whoosh_index_dir) repo_index_writer = AsyncWriter(repo_index) tool_index_writer = AsyncWriter(tool_index) repos_indexed = 0 tools_indexed = 0 execution_timer = ExecutionTimer() with repo_index.searcher() as searcher: for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): tools_list = repo.pop('tools_list') repo_id = repo['id'] indexed_document = searcher.document(id=repo_id) if indexed_document: if indexed_document['full_last_updated'] == repo.get( 'full_last_updated'): # We're done, since we sorted repos by update time break else: # Got an update, delete the previous document repo_index_writer.delete_by_term('id', repo_id) repo_index_writer.add_document(**repo) # Tools get their own index for tool in tools_list: tool_index_writer.add_document(**tool) tools_indexed += 1 repos_indexed += 1 tool_index_writer.commit() repo_index_writer.commit() log.info("Indexed repos: %s, tools: %s", repos_indexed, tools_indexed) log.info("Toolbox index finished %s", execution_timer) return repos_indexed, tools_indexed
def check_db(config: ToolShedAppConfiguration): dburi = config.database_connection sa_session = None database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. ' database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.' try: model = tool_shed_model.init(config.file_path, dburi, engine_options={}, create_tables=False) sa_session = model.context.current sys.exit(database_exists_message) except ProgrammingError: pass except OperationalError: pass try: if sa_session is not None: result = sa_session.execute( 'SELECT version FROM migrate_version').first() if result[0] >= 2: sys.exit(database_exists_message) else: pass except ProgrammingError: pass if config.hgweb_config_dir: hgweb_config_parser = ConfigParser() hgweb_dir = config.hgweb_config_dir hgweb_config_file = os.path.join(hgweb_dir, "hgweb.config") if not os.path.exists(hgweb_config_file): sys.exit(0) hgweb_config_parser.read(hgweb_config_file) configured_repos = hgweb_config_parser.items('paths') if len(configured_repos) >= 1: message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete" message += " the current hgweb.config file along with all associated repositories in the configured " message += "location before attempting to boostrap." sys.exit(message) else: sys.exit(0) sys.exit(0)
def check_db(config_parser): dburi = None if config_parser.has_option('app:main', 'database_connection'): dburi = config_parser.get('app:main', 'database_connection') elif config_parser.has_option('app:main', 'database_file'): db_file = config_parser.get('app:main', 'database_file') dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: sys.exit( 'The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.' ) sa_session = None database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. ' database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.' try: model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False) sa_session = model.context.current sys.exit(database_exists_message) except ProgrammingError: pass except OperationalError: pass try: if sa_session is not None: result = sa_session.execute( 'SELECT version FROM migrate_version').first() if result[0] >= 2: sys.exit(database_exists_message) else: pass except ProgrammingError: pass if config_parser.has_option('app:main', 'hgweb_config_dir'): hgweb_config_parser = ConfigParser() hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir') hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config') if not os.path.exists(hgweb_config_file): sys.exit(0) hgweb_config_parser.read(hgweb_config_file) configured_repos = hgweb_config_parser.items('paths') if len(configured_repos) >= 1: message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete" message += " the current hgweb.config file along with all associated repositories in the configured " message += "location before attempting to boostrap." sys.exit(message) else: sys.exit(0) else: sys.exit(0) sys.exit(0)
def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. """ model = ts_mapping.init(file_path, dburi, engine_options={}, create_tables=False) sa_session = model.context.current # Rare race condition exists here and below tool_index_dir = os.path.join(whoosh_index_dir, 'tools') if not os.path.exists(whoosh_index_dir): os.makedirs(whoosh_index_dir) os.makedirs(tool_index_dir) work_repo_dir = whoosh_index_dir work_tool_dir = tool_index_dir else: # Index exists, prevent in-place index regeneration work_repo_dir = tempfile.mkdtemp(prefix="tmp-whoosh-repo") work_tool_dir = tempfile.mkdtemp(prefix="tmp-whoosh-tool") repo_index_storage = FileStorage(work_repo_dir) tool_index_storage = FileStorage(work_tool_dir) repo_index = repo_index_storage.create_index(repo_schema) tool_index = tool_index_storage.create_index(tool_schema) repo_index_writer = repo_index.writer() tool_index_writer = tool_index.writer() repos_indexed = 0 tools_indexed = 0 execution_timer = ExecutionTimer() for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): repo_index_writer.add_document( id=repo.get('id'), name=unicodify(repo.get('name')), description=unicodify(repo.get('description')), long_description=unicodify(repo.get('long_description')), homepage_url=unicodify(repo.get('homepage_url')), remote_repository_url=unicodify(repo.get('remote_repository_url')), repo_owner_username=unicodify(repo.get('repo_owner_username')), categories=unicodify(repo.get('categories')), times_downloaded=repo.get('times_downloaded'), approved=repo.get('approved'), last_updated=repo.get('last_updated'), full_last_updated=repo.get('full_last_updated'), repo_lineage=unicodify(repo.get('repo_lineage'))) # Tools get their own index for tool in repo.get('tools_list'): tool_index_writer.add_document( id=unicodify(tool.get('id')), name=unicodify(tool.get('name')), version=unicodify(tool.get('version')), description=unicodify(tool.get('description')), help=unicodify(tool.get('help')), repo_owner_username=unicodify(repo.get('repo_owner_username')), repo_name=unicodify(repo.get('name')), repo_id=repo.get('id')) tools_indexed += 1 repos_indexed += 1 tool_index_writer.commit() repo_index_writer.commit() log.info("Indexed repos: %s, tools: %s", repos_indexed, tools_indexed) log.info("Toolbox index finished %s", execution_timer) # Copy the built indexes if we were working in a tmp folder. if work_repo_dir is not whoosh_index_dir: shutil.rmtree(whoosh_index_dir) os.makedirs(whoosh_index_dir) os.makedirs(tool_index_dir) copy_tree(work_repo_dir, whoosh_index_dir) copy_tree(work_tool_dir, tool_index_dir) shutil.rmtree(work_repo_dir)
def model(): db_uri = 'sqlite:///:memory:' return mapping.init('/tmp', db_uri, create_tables=True)