def check_db(config_parser): dburi = None if config_parser.has_option("app:main", "database_connection"): dburi = config_parser.get("app:main", "database_connection") elif config_parser.has_option("app:main", "database_file"): db_file = config_parser.get("app:main", "database_file") dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: sys.exit( "The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap." ) sa_session = None database_exists_message = ( "The database configured for this Tool Shed is not new, so bootstrapping is not allowed. " ) database_exists_message += "Create a new database that has not been migrated before attempting to bootstrap." try: model = tool_shed_model.init( config_parser.get("app:main", "file_path"), dburi, engine_options={}, create_tables=False ) sa_session = model.context.current sys.exit(database_exists_message) except ProgrammingError: pass except OperationalError: pass try: if sa_session is not None: result = sa_session.execute("SELECT version FROM migrate_version").first() if result[0] >= 2: sys.exit(database_exists_message) else: pass except ProgrammingError: pass if config_parser.has_option("app:main", "hgweb_config_dir"): hgweb_config_parser = ConfigParser.ConfigParser() hgweb_dir = config_parser.get("app:main", "hgweb_config_dir") hgweb_config_file = os.path.join(hgweb_dir, "hgweb.config") if not os.path.exists(hgweb_config_file): sys.exit(0) hgweb_config_parser.read(hgweb_config_file) configured_repos = hgweb_config_parser.items("paths") if len(configured_repos) >= 1: message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete" message += " the current hgweb.config file along with all associated repositories in the configured " message += "location before attempting to boostrap." sys.exit(message) else: sys.exit(0) else: sys.exit(0) sys.exit(0)
def check_db(config_parser): dburi = None if config_parser.has_option('app:main', 'database_connection'): dburi = config_parser.get('app:main', 'database_connection') elif config_parser.has_option('app:main', 'database_file'): db_file = config_parser.get('app:main', 'database_file') dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: print 'The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.' exit(1) sa_session = None database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. ' database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.' try: model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False) sa_session = model.context.current print database_exists_message exit(1) except ProgrammingError, e: pass
def __init__(self, **kwd): log.debug("python path is: %s", ", ".join(sys.path)) self.name = "tool_shed" # Read the tool_shed.ini configuration file and check for errors. self.config = config.Configuration(**kwd) self.config.check() configure_logging(self.config) self.application_stack = application_stack_instance() # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes(self.config.root, self.config.datatypes_config) # Initialize the Tool Shed repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry( ) # Initialize the RepositoryGridFilterManager. self.repository_grid_filter_manager = RepositoryGridFilterManager() # Determine the Tool Shed database connection string. if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize the Tool Shed database and check for appropriate schema version. from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database create_or_verify_database(db_url, self.config.database_engine_options) # Set up the Tool Shed database engine and ORM. from galaxy.webapps.tool_shed.model import mapping self.model = mapping.init(self.config.file_path, db_url, self.config.database_engine_options) self.security = idencoding.IdEncodingHelper( id_secret=self.config.id_secret) # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagHandler(self) # Initialize the Tool Shed tool data tables. Never pass a configuration file here # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path) self.genome_builds = GenomeBuilds(self) from galaxy import auth self.auth_manager = auth.AuthManager(self) # Citation manager needed to load tools. from galaxy.managers.citations import CitationsManager self.citations_manager = CitationsManager(self) # The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required. self.toolbox = tools.ToolBox([], self.config.tool_path, self) # Initialize the Tool Shed security agent. self.security_agent = self.model.security_agent # The Tool Shed makes no use of a quota, but this attribute is still required. self.quota_agent = galaxy.quota.NoQuotaAgent(self.model) # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. self.server_starttime = int(time.time()) log.debug("Tool shed hgweb.config file is: %s", self.hgweb_config_manager.hgweb_config)
def check_db(config_parser): dburi = None if config_parser.has_option("app:main", "database_connection"): dburi = config_parser.get("app:main", "database_connection") elif config_parser.has_option("app:main", "database_file"): db_file = config_parser.get("app:main", "database_file") dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: print "The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap." exit(1) sa_session = None database_exists_message = ( "The database configured for this Tool Shed is not new, so bootstrapping is not allowed. " ) database_exists_message += "Create a new database that has not been migrated before attempting to bootstrap." try: model = tool_shed_model.init( config_parser.get("app:main", "file_path"), dburi, engine_options={}, create_tables=False ) sa_session = model.context.current print database_exists_message exit(1) except ProgrammingError, e: pass
def __init__(self, **kwd): log.debug("python path is: %s", ", ".join(sys.path)) self.name = "tool_shed" # Read the tool_shed.ini configuration file and check for errors. self.config = config.Configuration(**kwd) self.config.check() configure_logging(self.config) self.application_stack = application_stack_instance() # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes(self.config.root, self.config.datatypes_config) # Initialize the Tool Shed repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry() # Initialize the RepositoryGridFilterManager. self.repository_grid_filter_manager = RepositoryGridFilterManager() # Determine the Tool Shed database connection string. if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize the Tool Shed database and check for appropriate schema version. from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database create_or_verify_database(db_url, self.config.database_engine_options) # Set up the Tool Shed database engine and ORM. from galaxy.webapps.tool_shed.model import mapping self.model = mapping.init(self.config.file_path, db_url, self.config.database_engine_options) # Initialize the Tool Shed security helper. self.security = security.SecurityHelper(id_secret=self.config.id_secret) # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagManager(self) # Initialize the Tool Shed tool data tables. Never pass a configuration file here # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path) self.genome_builds = GenomeBuilds(self) from galaxy import auth self.auth_manager = auth.AuthManager(self) # Citation manager needed to load tools. from galaxy.managers.citations import CitationsManager self.citations_manager = CitationsManager(self) # The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required. self.toolbox = tools.ToolBox([], self.config.tool_path, self) # Initialize the Tool Shed security agent. self.security_agent = self.model.security_agent # The Tool Shed makes no use of a quota, but this attribute is still required. self.quota_agent = galaxy.quota.NoQuotaAgent(self.model) # TODO: Add OpenID support self.openid_providers = OpenIDProviders() # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. self.server_starttime = int(time.time()) log.debug("Tool shed hgweb.config file is: %s", self.hgweb_config_manager.hgweb_config)
def __init__(self, **kwd): print >> sys.stderr, "python path is: " + ", ".join(sys.path) self.name = "tool_shed" # Read the tool_shed_wsgi.ini configuration file and check for errors. self.config = config.Configuration(**kwd) self.config.check() config.configure_logging(self.config) # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes(self.config.root, self.config.datatypes_config) # Initialize the Tool Shed repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry( ) # Initialize the RepositoryGridFilterManager. self.repository_grid_filter_manager = RepositoryGridFilterManager() # Determine the Tool Shed database connection string. if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize the Tool Shed database and check for appropriate schema version. from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database create_or_verify_database(db_url, self.config.database_engine_options) # Set up the Tool Shed database engine and ORM. from galaxy.webapps.tool_shed.model import mapping self.model = mapping.init(self.config.file_path, db_url, self.config.database_engine_options) # Initialize the Tool SHed security helper. self.security = security.SecurityHelper( id_secret=self.config.id_secret) # initialize the Tool Shed tag handler. self.tag_handler = CommunityTagHandler() # Initialize the Tool Shed tool data tables. Never pass a configuration file here # because the Tool Shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path) self.genome_builds = GenomeBuilds(self) # Citation manager needed to load tools. from galaxy.managers.citations import CitationsManager self.citations_manager = CitationsManager(self) # The Tool Shed makes no use of a Galaxy toolbox, but this attribute is still required. self.toolbox = tools.ToolBox([], self.config.tool_path, self) # Initialize the Tool Shed security agent. self.security_agent = self.model.security_agent # The Tool Shed makes no use of a quota, but this attribute is still required. self.quota_agent = galaxy.quota.NoQuotaAgent(self.model) # TODO: Add OpenID support self.openid_providers = OpenIDProviders() # Initialize the baseline Tool Shed statistics component. self.shed_counter = self.model.shed_counter # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def check_db(config_parser): dburi = None if config_parser.has_option('app:main', 'database_connection'): dburi = config_parser.get('app:main', 'database_connection') elif config_parser.has_option('app:main', 'database_file'): db_file = config_parser.get('app:main', 'database_file') dburi = "sqlite:///%s?isolation_level=IMMEDIATE" % db_file else: sys.exit('The database configuration setting is missing from the tool_shed.ini file. Add this setting before attempting to bootstrap.') sa_session = None database_exists_message = 'The database configured for this Tool Shed is not new, so bootstrapping is not allowed. ' database_exists_message += 'Create a new database that has not been migrated before attempting to bootstrap.' try: model = tool_shed_model.init(config_parser.get('app:main', 'file_path'), dburi, engine_options={}, create_tables=False) sa_session = model.context.current sys.exit(database_exists_message) except ProgrammingError: pass except OperationalError: pass try: if sa_session is not None: result = sa_session.execute('SELECT version FROM migrate_version').first() if result[0] >= 2: sys.exit(database_exists_message) else: pass except ProgrammingError: pass if config_parser.has_option('app:main', 'hgweb_config_dir'): hgweb_config_parser = ConfigParser() hgweb_dir = config_parser.get('app:main', 'hgweb_config_dir') hgweb_config_file = os.path.join(hgweb_dir, 'hgweb.config') if not os.path.exists(hgweb_config_file): sys.exit(0) hgweb_config_parser.read(hgweb_config_file) configured_repos = hgweb_config_parser.items('paths') if len(configured_repos) >= 1: message = "This Tool Shed's hgweb.config file contains entries, so bootstrapping is not allowed. Delete" message += " the current hgweb.config file along with all associated repositories in the configured " message += "location before attempting to boostrap." sys.exit(message) else: sys.exit(0) else: sys.exit(0) sys.exit(0)
def __init__(self, config): self.config = config if not self.config.database_connection: self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % str(config.database) print 'Using database connection: ', self.config.database_connection # Setup the database engine and ORM self.model = mapping.init(self.config.file_path, self.config.database_connection, engine_options={}, create_tables=False) self.security = security.SecurityHelper(id_secret=self.config.id_secret) self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir print 'Using hgweb.config file: ', self.hgweb_config_manager.hgweb_config
def __init__( self, **kwargs ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) self.name = "tool_shed" # Read config file and check for errors self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) # Set up datatypes registry self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) # Set up the repository_types registry. self.repository_types_registry = tool_shed.repository_types.registry.Registry() # Determine the database url if self.config.database_connection: db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database # Initialize database / check for appropriate schema version from galaxy.webapps.tool_shed.model.migrate.check import create_or_verify_database create_or_verify_database( db_url, self.config.database_engine_options ) # Setup the database engine and ORM from galaxy.webapps.tool_shed.model import mapping self.model = mapping.init( self.config.file_path, db_url, self.config.database_engine_options ) # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # Tag handler self.tag_handler = CommunityTagHandler() # Tool data tables - never pass a config file here because the tool shed should always have an empty dictionary! self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path ) # The tool shed has no toolbox, but this attribute is still required. self.toolbox = tools.ToolBox( [], self.config.tool_path, self ) # Load security policy self.security_agent = self.model.security_agent self.quota_agent = galaxy.quota.NoQuotaAgent( self.model ) # TODO: Add OpenID support self.openid_providers = OpenIDProviders() self.shed_counter = self.model.shed_counter # Let the HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = self.model.hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config
def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. """ model = ts_mapping.init(file_path, dburi, engine_options={}, create_tables=False) sa_session = model.context.current # Rare race condition exists here and below tool_index_dir = os.path.join(whoosh_index_dir, 'tools') if not os.path.exists(whoosh_index_dir): os.makedirs(whoosh_index_dir) os.makedirs(tool_index_dir) work_repo_dir = whoosh_index_dir work_tool_dir = tool_index_dir else: # Index exists, prevent in-place index regeneration work_repo_dir = tempfile.mkdtemp(prefix="tmp-whoosh-repo") work_tool_dir = tempfile.mkdtemp(prefix="tmp-whoosh-tool") repo_index_storage = FileStorage(work_repo_dir) tool_index_storage = FileStorage(work_tool_dir) repo_index = repo_index_storage.create_index(repo_schema) tool_index = tool_index_storage.create_index(tool_schema) repo_index_writer = repo_index.writer() tool_index_writer = tool_index.writer() repos_indexed = 0 tools_indexed = 0 execution_timer = ExecutionTimer() for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): repo_index_writer.add_document( id=repo.get('id'), name=unicodify(repo.get('name')), description=unicodify(repo.get('description')), long_description=unicodify(repo.get('long_description')), homepage_url=unicodify(repo.get('homepage_url')), remote_repository_url=unicodify(repo.get('remote_repository_url')), repo_owner_username=unicodify(repo.get('repo_owner_username')), categories=unicodify(repo.get('categories')), times_downloaded=repo.get('times_downloaded'), approved=repo.get('approved'), last_updated=repo.get('last_updated'), full_last_updated=repo.get('full_last_updated'), repo_lineage=unicodify(repo.get('repo_lineage'))) # Tools get their own index for tool in repo.get('tools_list'): tool_index_writer.add_document( id=unicodify(tool.get('id')), name=unicodify(tool.get('name')), version=unicodify(tool.get('version')), description=unicodify(tool.get('description')), help=unicodify(tool.get('help')), repo_owner_username=unicodify(repo.get('repo_owner_username')), repo_name=unicodify(repo.get('name')), repo_id=repo.get('id')) tools_indexed += 1 repos_indexed += 1 tool_index_writer.commit() repo_index_writer.commit() log.info("Indexed repos: %s, tools: %s", repos_indexed, tools_indexed) log.info("Toolbox index finished %s", execution_timer) # Copy the built indexes if we were working in a tmp folder. if work_repo_dir is not whoosh_index_dir: shutil.rmtree(whoosh_index_dir) os.makedirs(whoosh_index_dir) os.makedirs(tool_index_dir) copy_tree(work_repo_dir, whoosh_index_dir) copy_tree(work_tool_dir, tool_index_dir) shutil.rmtree(work_repo_dir)