def migrate_3_0_0_0(self):
"""Create a conversion suggested label if db is not untouched."""
if not self.is_db_untouched():
config_entry = model.Config(Key='utf8_conversion',
Value='suggested')
model.db.session.add(config_entry)
log.warning("Database conversion step suggested!\n"
"Please run command:\n"
" linotp admin fix-db-encoding")
return
def set_config(key, value, typ, description=None):
'''
create an intial config entry, if it does not exist
:param key: the key
:param value: the value
:param description: the description of the key
:return: nothing
'''
count = Session.query(model.Config).filter(
model.Config.Key == "linotp." + key).count()
if count == 0:
config_entry = model.Config(key, value, Type=typ, Description=description)
Session.add(config_entry)
return
def set_version(self, version: str):
"""Set the new db model version number.
- on update: update the entry
- on new: create new db entry
:param version: set the new db model version
"""
if version == self.current_version:
return
config_entry = Migration._query_db_model_version()
if config_entry:
config_entry.Value = version
else:
config_entry = model.Config(Key=self.db_model_key, Value=version)
model.db.session.add(config_entry) # pylint: disable=E1101
def test_fix_db_encoding(app, runner, freezer):
"""Verify the fix-db-encoding will be triggerd.
1.
a: insert iso8859 encode data and
b: set flag that conversion is suggested
2. run conversion by cli command
3.
a: verify that the iso8859 data is now in utf format
b: the conversion flag is removed
"""
from linotp import model
from linotp.model import db
# --------------------------------------------------------------------- --
#1. add db entries
db.init_app(app)
# 1.a add iso8859 encoded data
utf8_str = 'äöü߀'
iso8859_15_str = bytes(utf8_str, encoding='utf-8').decode('iso-8859-15')
assert utf8_str != iso8859_15_str
config_entry = model.Config(Key='test_encode', Value=iso8859_15_str)
model.db.session.add(config_entry)
# 1.b add conversion suggested flag
config_entry = model.Config(Key='utf8_conversion', Value='suggested')
model.db.session.add(config_entry)
model.db.session.commit()
# --------------------------------------------------------------------- --
# 2. run cli command for conversion
result = runner.invoke(cli_main, ['admin', 'fix-db-encoding'])
assert result.exit_code == 0
# --------------------------------------------------------------------- --
# 3.a check that the data is converted
converted = model.Config.query.filter(
model.Config.Key == 'linotp.test_encode').first()
assert converted.Value == utf8_str
# 3.b and the flag is removed
conversion_flag = model.Config.query.filter(
model.Config.Key == "linotp.utf8_conversion").first()
assert conversion_flag is None
def test_migration(app, runner, freezer):
"""Verify the that the conversion hint is set.
1.
a: add Comfig timestamp and
b: set db schema version to pre 3.0
2. run linotp init database to set the conversion hint
3.
a: the conversion flag is set and
b: the db schema version is updated
"""
from linotp import model
from linotp.model import db
# --------------------------------------------------------------------- --
#1. add db entries
db.init_app(app)
# 1.a add Config timestamp if not there - for is_untouched
config_timestamp = model.Config.query.filter(
model.Config.Key == "linotp.Config").first()
if config_timestamp is None:
config_entry = model.Config(Key='Config', Value='2020-12-02 10:42:07')
model.db.session.add(config_entry)
# 1.b set db schema version to pre 3.0
db_schema_version_pre = '2.12.0.0'
db_schema = model.Config.query.filter(
model.Config.Key == "linotp.sql_data_model_version").first()
if db_schema is not None:
update_data = {
model.Config.Value: db_schema_version_pre,
}
# replace by the primary key: Key
model.Config.query.filter(model.Config.Key == db_schema.Key).update(
update_data, synchronize_session=False)
else:
config_entry = model.Config(Key='sql_data_model_version',
Value=db_schema_version_pre)
model.db.session.add(config_entry)
model.db.session.commit()
# --------------------------------------------------------------------- --
# 2. run cli command for mogration
result = runner.invoke(cli_main, ['init', 'database'])
assert result.exit_code == 0
# --------------------------------------------------------------------- --
# 3.a cobversion flag is set
conversion_flag = model.Config.query.filter(
model.Config.Key == "linotp.utf8_conversion").first()
assert conversion_flag and conversion_flag.Value == 'suggested'
# 3.b check that the db schema version is updated
converted = model.Config.query.filter(
model.Config.Key == 'linotp.sql_data_model_version').first()
assert converted.Value != db_schema_version_pre
def migrate_3_0_0_0(self) -> None:
"""Migrate to linotp3 - to python3+mysql.
The major challenge for the linotp3 migration is the migration from
python2+mysql where the mysql driver was using latin1 encoded data,
though the database might already support utf8.
Thus we can exclude all fresh created and non-mysql databases
"""
if self.is_db_untouched():
log.info("Fresh database - no migration required!")
return
# ----------------------------------------------------------------- --
# with linotp3 we drop all previous audit entries to fix audit signing
from linotp.lib.audit.base import getAudit
audit_obj = getAudit()
audit_obj.delete_all_entries()
log.info("All limotp2 audit entries deleted.")
# ----------------------------------------------------------------- --
if not self.engine.url.drivername.startswith("mysql"):
log.info(
"Non mysql databases %r - no migration required.",
self.engine.url.drivername,
)
return
# ----------------------------------------------------------------- --
# MYSQL Schema and Data Migration:
#
# In case of pre buster db (or restored by a backup), the tabel schema
# is defined with charset latin1. Thus the schema must be updated first
# othewise the data migration will fail.
#
# Which tables are converted is reported by the schema migration.
# If no table was converted, we have to assume that the mysql db was
# created newly (e.g. on buster) where this is done with charset=utf8.
#
# In case of a newly (buster) e.g. linotp2 created db, we cannot imply
# that the data migration has to be done, we only can suggest this.
log.info("Starting mysql migration")
# before we adjust the schema, we have to close former sessions,
# otherwise we would run into a database lock
model.db.session.close()
# now we can run the schema migration
mysql_mig = MYSQL_Migration(self.engine)
migrated_tables = mysql_mig.migrate_schema()
mysql_mig.migrate_data(migrated_tables)
if not migrated_tables:
config_entry = model.Config(
Key="utf8_conversion", Value="suggested"
)
model.db.session.add(config_entry)
log.warning(
"Database conversion step suggested!\n"
"Please run command:\n"
" linotp admin fix-db-encoding"
)