def test_create_exists_sqlite_database(database_name):
with tempfile.TemporaryDirectory() as tmp_dir:
url = make_sqlite_url(tmp_dir, database_name)
assert not database_exists(url, database_name)
create_database(url, database_name)
assert database_exists(url, database_name)
drop_database(url, database_name)
assert not database_exists(url, database_name)
def test_create_exists_postgres_database__pass_as_url(database_name, postgres_url):
# the database in the url is the one to create/check
url = replace_database_in_url(postgres_url, database_name)
assert not database_exists(url)
create_database(url)
assert database_exists(url)
drop_database(postgres_url, database_name)
assert not database_exists(url)
def test_create_exists_sqlite_database__pass_as_url(database_name):
# the database in the url is the one to create/check
with tempfile.TemporaryDirectory() as tmp_dir:
url = make_sqlite_url(tmp_dir, database_name)
assert not database_exists(url)
create_database(url)
assert database_exists(url)
drop_database(url, database_name)
assert not database_exists(url)
def _wait_for_database(self, url):
attempts = self.config.database_wait_attempts
pause = self.config.database_wait_sleep
for i in range(1, attempts):
try:
database_exists(url)
break
except Exception:
log.info("Waiting for database: attempt %d of %d" %
(i, attempts))
time.sleep(pause)
def database_conf(db_path, prefix="GALAXY", prefer_template_database=False):
"""Find (and populate if needed) Galaxy database connection."""
database_auto_migrate = False
check_migrate_databases = True
dburi_var = "%s_TEST_DBURI" % prefix
template_name = None
if dburi_var in os.environ:
database_connection = os.environ[dburi_var]
# only template if postgres - not mysql or sqlite
do_template = prefer_template_database and database_connection.startswith(
"p")
if do_template:
database_template_parsed = urlparse(database_connection)
template_name = database_template_parsed.path[
1:] # drop / from /galaxy
actual_db = "gxtest" + ''.join(
random.choice(string.ascii_uppercase) for _ in range(10))
actual_database_parsed = database_template_parsed._replace(
path="/%s" % actual_db)
database_connection = actual_database_parsed.geturl()
if not database_exists(database_connection):
# We pass by migrations and instantiate the current table
create_database(database_connection)
mapping.init('/tmp',
database_connection,
create_tables=True,
map_install_models=True)
toolshed_mapping.init(database_connection, create_tables=True)
check_migrate_databases = False
else:
default_db_filename = "%s.sqlite" % prefix.lower()
template_var = "%s_TEST_DB_TEMPLATE" % prefix
db_path = os.path.join(db_path, default_db_filename)
if template_var in os.environ:
# Middle ground between recreating a completely new
# database and pointing at existing database with
# GALAXY_TEST_DBURI. The former requires a lot of setup
# time, the latter results in test failures in certain
# cases (namely tool shed tests expecting clean database).
copy_database_template(os.environ[template_var], db_path)
database_auto_migrate = True
database_connection = 'sqlite:///%s' % db_path
config = {
"check_migrate_databases": check_migrate_databases,
"database_connection": database_connection,
"database_auto_migrate": database_auto_migrate
}
if not database_connection.startswith("sqlite://"):
config["database_engine_option_max_overflow"] = "20"
config["database_engine_option_pool_size"] = "10"
if template_name:
config["database_template"] = template_name
return config
def create_or_verify_database(url, engine_options=None):
"""
Check that the database is use-able, possibly creating it if empty (this is
the only time we automatically create tables, otherwise we force the
user to do it using the management script so they can create backups).
1) Empty database --> initialize with latest version and return
2) Database older than migration support --> fail and require manual update
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
"""
engine_options = engine_options or {}
# Create engine and metadata
if not database_exists(url):
message = f"Creating database for URI [{url}]"
log.info(message)
create_database(url)
engine = create_engine(url, **engine_options)
meta = MetaData(bind=engine)
# Try to load dataset table
try:
Table("galaxy_user", meta, autoload=True)
except NoSuchTableError:
# No 'galaxy_user' table means a completely uninitialized database, which
# is fine, init the database in a versioned state
log.info("No database, initializing")
# Database might or might not be versioned
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except Exception:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
return
try:
Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
exception_msg = "Your database has version '%d' but this code expects version '%d'. " % (
db_schema.version, migrate_repository.versions.latest)
exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:"
exception_msg += "\n\nsh manage_db.sh upgrade tool_shed\n"
raise Exception(exception_msg)
else:
log.info("At database version %d" % db_schema.version)
def create_or_verify_database(url, engine_options=None, app=None):
"""
"""
# Create engine and metadata
engine_options = engine_options or {}
if not database_exists(url):
message = "Creating database for URI [%s]" % url
log.info(message)
create_database(url)
engine = create_engine(url, **engine_options)
def migrate():
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except Exception:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
meta = MetaData(bind=engine)
if app and getattr(app.config, 'database_auto_migrate', False):
migrate()
return
# Try to load tool_shed_repository table
try:
Table("tool_shed_repository", meta, autoload=True)
except NoSuchTableError:
# No table means a completely uninitialized database. If we
# have an app, we'll set its new_installation setting to True
# so the tool migration process will be skipped.
log.info("Creating install database from scratch, skipping migrations")
mapping.init(url=url, create_tables=True)
current_version = migrate_repository.version().version
schema.ControlledSchema.create(engine,
migrate_repository,
version=current_version)
db_schema = schema.ControlledSchema(engine, migrate_repository)
assert db_schema.version == current_version
migrate()
return
try:
Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
exception_msg = "Your database has version '%d' but this code expects version '%d'. " % (
db_schema.version, migrate_repository.versions.latest)
exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:"
exception_msg += "\n\nsh manage_db.sh upgrade install\n"
else:
log.info("At database version %d" % db_schema.version)
def create_or_verify_database(url,
galaxy_config_file,
engine_options=None,
app=None,
map_install_models=False):
"""
Check that the database is use-able, possibly creating it if empty (this is
the only time we automatically create tables, otherwise we force the
user to do it using the management script so they can create backups).
1) Empty database --> initialize with latest version and return
2) Database older than migration support --> fail and require manual update
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
"""
# Create the base database if it doesn't yet exist.
engine_options = engine_options or {}
new_database = not database_exists(url)
if new_database:
template = app and getattr(app.config, "database_template", None)
encoding = app and getattr(app.config, "database_encoding", None)
create_kwds = {}
message = f"Creating database for URI [{url}]"
if template:
message += f" from template [{template}]"
create_kwds["template"] = template
if encoding:
message += f" with encoding [{encoding}]"
create_kwds["encoding"] = encoding
log.info(message)
create_database(url, **create_kwds)
# Create engine and metadata
engine = create_engine(url, **engine_options)
def migrate():
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except Exception:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
def migrate_from_scratch():
if not os.environ.get("GALAXY_TEST_FORCE_DATABASE_MIGRATION"):
log.info("Creating new database from scratch, skipping migrations")
current_version = migrate_repository.version().version
mapping.init(file_path='/tmp',
url=url,
map_install_models=map_install_models,
create_tables=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=current_version)
db_schema = schema.ControlledSchema(engine, migrate_repository)
assert db_schema.version == current_version
migrate()
if app:
# skips the tool migration process.
app.new_installation = True
meta = MetaData(bind=engine)
if new_database:
migrate_from_scratch()
return
elif app and getattr(app.config, 'database_auto_migrate', False):
migrate()
return
# Try to load dataset table
try:
Table("dataset", meta, autoload=True)
except NoSuchTableError:
# No 'dataset' table means a completely uninitialized database.
log.info("No database, initializing")
migrate_from_scratch()
return
try:
hda_table = Table("history_dataset_association", meta, autoload=True)
except NoSuchTableError:
raise Exception(
"Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually"
)
# There is a 'history_dataset_association' table, so we (hopefully) have
# version 1 of the database, but without the migrate_version table. This
# happens if the user has a build from right before migration was added.
# Verify that this is true, if it is any older they'll have to update
# manually
if 'copied_from_history_dataset_association_id' not in hda_table.c:
# The 'copied_from_history_dataset_association_id' column was added in
# rev 1464:c7acaa1bb88f. This is the oldest revision we currently do
# automated versioning for, so stop here
raise Exception(
"Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually"
)
# At revision 1464:c7acaa1bb88f or greater (database version 1), make sure
# that the db has version information. This is the trickiest case -- we
# have a database but no version control, and are assuming it is a certain
# version. If the user has postion version 1 changes this could cause
# problems
try:
Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
config_arg = ''
if galaxy_config_file and os.path.abspath(
os.path.join(os.getcwd(), 'config',
'galaxy.ini')) != galaxy_config_file:
config_arg = f" -c {galaxy_config_file.replace(os.path.abspath(os.getcwd()), '.')}"
expect_msg = "Your database has version '%d' but this code expects version '%d'" % (
db_schema.version, migrate_repository.versions.latest)
instructions = ""
if db_schema.version > migrate_repository.versions.latest:
instructions = "To downgrade the database schema you have to checkout the Galaxy version that you were running previously. "
cmd_msg = "sh manage_db.sh%s downgrade %d" % (
config_arg, migrate_repository.versions.latest)
else:
cmd_msg = f"sh manage_db.sh{config_arg} upgrade"
backup_msg = f"Please backup your database and then migrate the database schema by running '{cmd_msg}'."
allow_future_database = os.environ.get("GALAXY_ALLOW_FUTURE_DATABASE",
False)
if db_schema.version > migrate_repository.versions.latest and allow_future_database:
log.warning(
"WARNING: Database is from the future, but GALAXY_ALLOW_FUTURE_DATABASE is set, so Galaxy will continue to start."
)
else:
raise Exception(f"{expect_msg}. {instructions}{backup_msg}")
else:
log.info("At database version %d" % db_schema.version)
def test_create_exists_mysql_database(database_name, mysql_url):
assert not database_exists(mysql_url, database_name)
create_database(mysql_url, database_name)
assert database_exists(mysql_url, database_name)
drop_database(mysql_url, database_name)
assert not database_exists(mysql_url, database_name)
def test_exists_sqlite_in_memory_database(database_name, sqlite_memory_url):
assert database_exists(sqlite_memory_url)
def test_create_exists_postgres_database(database_name, postgres_url):
assert not database_exists(postgres_url, database_name)
create_database(postgres_url, database_name)
assert database_exists(postgres_url, database_name)
drop_database(postgres_url, database_name)
assert not database_exists(postgres_url, database_name)
