def schema_is_up_to_date(self) -> bool:
"""Checks if the database schema is up-to-date"""
# Check if the database is empty
db_vars_table_exists = len(self.query("""
select 1 from pg_tables where tablename = 'database_variables' and schemaname = 'public'
""").flat()) > 0
if not db_vars_table_exists:
log.info(
"Database table 'database_variables' does not exist, probably the database is empty at this point.")
return True
# Current schema version
(current_schema_version,) = self.query("""
SELECT value AS schema_version
FROM database_variables
WHERE name = 'database-schema-version'
LIMIT 1
""").flat()
current_schema_version = int(current_schema_version)
if current_schema_version == 0:
raise McSchemaIsUpToDateException("Current schema version is 0")
# Target schema version
sql = open(mc_sql_schema_path(), 'r').read()
target_schema_version = schema_version_from_lines(sql)
if not target_schema_version:
raise McSchemaIsUpToDateException("Invalid target schema version.")
# Check if the current schema is up-to-date
if current_schema_version != target_schema_version:
return DatabaseHandler.__should_continue_with_outdated_schema(current_schema_version, target_schema_version)
else:
# Things are fine at this point.
return True
def schema_is_up_to_date(self) -> bool:
"""Checks if the database schema is up-to-date"""
# Check if the database is empty
db_vars_table_exists = len(self.query("""
select 1 from pg_tables where tablename = 'database_variables' and schemaname = 'public'
""").flat()) > 0
if not db_vars_table_exists:
log.info(
"Database table 'database_variables' does not exist, probably the database is empty at this point.")
return True
# Current schema version
(current_schema_version,) = self.query("""
SELECT value AS schema_version
FROM database_variables
WHERE name = 'database-schema-version'
LIMIT 1
""").flat()
current_schema_version = int(current_schema_version)
if current_schema_version == 0:
raise McSchemaIsUpToDateException("Current schema version is 0")
# Target schema version
sql = open(mc_sql_schema_path(), 'r').read()
target_schema_version = schema_version_from_lines(sql)
if not target_schema_version:
raise McSchemaIsUpToDateException("Invalid target schema version.")
# Check if the current schema is up-to-date
if current_schema_version != target_schema_version:
return DatabaseHandler.__should_continue_with_outdated_schema(current_schema_version, target_schema_version)
else:
# Things are fine at this point.
return True
def recreate_db(label: typing.Optional[str] = None,
is_template: bool = False) -> None:
"""(Re)create database schema.
This function drops all objects in all schemas and reruns the schema/mediawords.sql to recreate the schema
(and erase all data!) for the given database.
This function will refuse to run if there are more than 10 million stories in the database, under the assumption
that the database might be a production database in that case.
"""
def reset_all_schemas(db_: DatabaseHandler) -> None:
"""Recreate all schemas."""
schemas = db_.query(
"""
SELECT schema_name
FROM information_schema.schemata
WHERE schema_name NOT LIKE %(schema_pattern)s
AND schema_name != 'information_schema'
ORDER BY schema_name
""", {
'schema_pattern': 'pg_%'
}).flat()
# When dropping schemas, PostgreSQL spits out a lot of notices which break "no warnings" unit test
db_.query('SET client_min_messages=WARNING')
for schema in schemas:
db_.query('DROP SCHEMA IF EXISTS %s CASCADE' % schema)
db_.query('SET client_min_messages=NOTICE')
# ---
label = decode_str_from_bytes_if_needed(label)
db = connect_to_db(label=label,
do_not_check_schema_version=True,
is_template=is_template)
log.info("Resetting all schemas...")
reset_all_schemas(db_=db)
db.set_show_error_statement(True)
mediawords_sql_path = mc_sql_schema_path()
log.info("Importing from %s..." % mediawords_sql_path)
with open(mediawords_sql_path, 'r') as mediawords_sql_f:
mediawords_sql = mediawords_sql_f.read()
db.query(mediawords_sql)
log.info("Done.")
def recreate_db(label: typing.Optional[str] = None, is_template: bool = False) -> None:
"""(Re)create database schema.
This function drops all objects in all schemas and reruns the schema/mediawords.sql to recreate the schema
(and erase all data!) for the given database.
This function will refuse to run if there are more than 10 million stories in the database, under the assumption
that the database might be a production database in that case.
"""
def reset_all_schemas(db_: DatabaseHandler) -> None:
"""Recreate all schemas."""
schemas = db_.query("""
SELECT schema_name
FROM information_schema.schemata
WHERE schema_name NOT LIKE %(schema_pattern)s
AND schema_name != 'information_schema'
ORDER BY schema_name
""", {'schema_pattern': 'pg_%'}).flat()
# When dropping schemas, PostgreSQL spits out a lot of notices which break "no warnings" unit test
db_.query('SET client_min_messages=WARNING')
for schema in schemas:
db_.query('DROP SCHEMA IF EXISTS %s CASCADE' % schema)
db_.query('SET client_min_messages=NOTICE')
# ---
label = decode_str_from_bytes_if_needed(label)
db = connect_to_db(label=label, do_not_check_schema_version=True, is_template=is_template)
log.info("Resetting all schemas...")
reset_all_schemas(db_=db)
db.set_show_error_statement(True)
mediawords_sql_path = mc_sql_schema_path()
log.info("Importing from %s..." % mediawords_sql_path)
with open(mediawords_sql_path, 'r') as mediawords_sql_f:
mediawords_sql = mediawords_sql_f.read()
db.query(mediawords_sql)
log.info("Done.")
def recreate_db(label: str = None) -> None:
"""(Re)create database schema."""
def reset_all_schemas(db_: DatabaseHandler) -> None:
"""Recreate all schemas."""
schemas = db_.query(
"""
SELECT schema_name
FROM information_schema.schemata
WHERE schema_name NOT LIKE %(schema_pattern)s
AND schema_name != 'information_schema'
ORDER BY schema_name
""", {
'schema_pattern': 'pg_%'
}).flat()
# When dropping schemas, PostgreSQL spits out a lot of notices which break "no warnings" unit test
db_.query('SET client_min_messages=WARNING')
for schema in schemas:
db_.query('DROP SCHEMA IF EXISTS %s CASCADE' % schema)
db_.query('SET client_min_messages=NOTICE')
# ---
label = decode_object_from_bytes_if_needed(label)
db = connect_to_db(label=label, do_not_check_schema_version=True)
log.info("Resetting all schemas...")
reset_all_schemas(db_=db)
db.set_show_error_statement(True)
mediawords_sql_path = mc_sql_schema_path()
log.info("Importing from %s..." % mediawords_sql_path)
with open(mediawords_sql_path, 'r') as mediawords_sql_f:
mediawords_sql = mediawords_sql_f.read()
db.query(mediawords_sql)
log.info("Done.")
def test_mc_sql_schema_path():
sql_schema_path = mc_paths.mc_sql_schema_path()
assert os.path.exists(sql_schema_path)
assert os.path.isfile(sql_schema_path)