def _test(self, path, version):
f = 'glbackend-%d.db' % version
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
self.start_db_file = os.path.abspath(
os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version))
self.final_db_file = os.path.abspath(
os.path.join(GLSettings.db_path,
'glbackend-%d.db' % DATABASE_VERSION))
self.start_db_uri = GLSettings.make_db_uri(self.start_db_file)
GLSettings.db_uri = GLSettings.make_db_uri(self.final_db_file)
shutil.rmtree(GLSettings.db_path, True)
os.mkdir(GLSettings.db_path)
dbpath = os.path.join(path, f)
dbfile = os.path.join(GLSettings.db_path, f)
shutil.copyfile(dbpath, dbfile)
# TESTS PRECONDITIONS
preconditions = getattr(self, 'preconditions_%d' % version, None)
if preconditions is not None:
preconditions()
ret = update_db()
# TESTS POSTCONDITIONS
postconditions = getattr(self, 'postconditions_%d' % version, None)
if postconditions is not None:
postconditions()
shutil.rmtree(GLSettings.db_path)
self.assertNotEqual(ret, -1)
def perform_data_update(dbfile):
store = Store(create_database(GLSettings.make_db_uri(dbfile)))
enabled_languages = [
lang.name for lang in store.find(l10n.EnabledLanguage)
]
removed_languages = list(
set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES))
if len(removed_languages):
removed_languages.sort()
removed_languages = ', '.join(removed_languages)
raise Exception(
"FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n"
"Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop"
% removed_languages)
try:
db_perform_data_update(store)
store.commit()
except:
store.rollback()
raise
finally:
store.close()
def setUp(self):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
os.mkdir(GLSettings.db_path)
db_name = 'glbackend-%d.db' % DATABASE_VERSION
db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name)
shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name))
self.db_file = os.path.join(GLSettings.db_path, db_name)
GLSettings.db_uri = GLSettings.make_db_uri(self.db_file)
# place a dummy version in the current db
store = Store(create_database(GLSettings.db_uri))
prv = config.PrivateFactory(store)
self.dummy_ver = '2.XX.XX'
prv.set_val('version', self.dummy_ver)
self.assertEqual(prv.get_val('version'), self.dummy_ver)
store.commit()
store.close()
# backup various mocks that we will use
self._bck_f = config.is_cfg_valid
GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop('smtp_password')
self.dp = u'yes_you_really_should_change_me'
def test_mig_37_valid_tor_hs_key(self):
self._initStartDB(36)
from globaleaks.db.migrations import update_37
t = update_37.TOR_DIR
update_37.TOR_DIR = GLSettings.db_path
pk_path = os.path.join(update_37.TOR_DIR, 'private_key')
hn_path = os.path.join(update_37.TOR_DIR, 'hostname')
shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/private_key'), pk_path)
shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/hostname'), hn_path)
ret = update_db()
self.assertEqual(ret, None)
new_uri = GLSettings.make_db_uri(
os.path.join(GLSettings.db_path, GLSettings.db_file_name))
store = Store(create_database(new_uri))
hs = config.NodeFactory(store).get_val('onionservice')
pk = config.PrivateFactory(store).get_val('tor_onion_key')
self.assertEqual('lftx7dbyvlc5txtl.onion', hs)
with open(os.path.join(helpers.DATA_DIR,
'tor/ephemeral_service_key')) as f:
saved_key = f.read().strip()
self.assertEqual(saved_key, pk)
store.close()
shutil.rmtree(GLSettings.db_path)
update_37.TOR_DIR = t
def setUp(self):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
shutil.rmtree(GLSettings.db_path, True)
os.mkdir(GLSettings.db_path)
db_name = 'glbackend-%d.db' % DATABASE_VERSION
db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'db', 'populated', db_name)
shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name))
self.db_file = os.path.join(GLSettings.db_path, db_name)
GLSettings.db_uri = GLSettings.make_db_uri(self.db_file)
# place a dummy version in the current db
store = Store(create_database(GLSettings.db_uri))
prv = config.PrivateFactory(store)
self.dummy_ver = '2.XX.XX'
prv.set_val(u'version', self.dummy_ver)
self.assertEqual(prv.get_val(u'version'), self.dummy_ver)
store.commit()
store.close()
# backup various mocks that we will use
self._bck_f = config.is_cfg_valid
GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop(
'smtp_password')
self.dp = u'yes_you_really_should_change_me'
def perform_data_update(dbfile):
new_tmp_store = Store(create_database(GLSettings.make_db_uri(dbfile)))
try:
db_perform_data_update(new_tmp_store)
new_tmp_store.commit()
except:
new_tmp_store.rollback()
raise
finally:
new_tmp_store.close()
def _test(self, path, f):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))
GLSettings.db_uri = GLSettings.make_db_uri(final_db_file)
os.mkdir(GLSettings.db_path)
dbpath = os.path.join(path, f)
dbfile = os.path.join(GLSettings.db_path, f)
shutil.copyfile(dbpath, dbfile)
ret = perform_system_update()
shutil.rmtree(GLSettings.db_path)
self.assertNotEqual(ret, -1)
def _initStartDB(self, target_ver):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
os.mkdir(GLSettings.db_path)
db_name = 'glbackend-%d.db' % target_ver
db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name)
shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name))
self.db_file = os.path.join(GLSettings.db_path, db_name)
GLSettings.db_uri = GLSettings.make_db_uri(self.db_file)
self.store = Store(create_database(GLSettings.db_uri))
def postconditions_36(self):
new_uri = GLSettings.make_db_uri(
os.path.join(GLSettings.db_path, GLSettings.db_file_name))
store = Store(create_database(new_uri))
hs = config.NodeFactory(store).get_val(u'onionservice')
pk = config.PrivateFactory(store).get_val(u'tor_onion_key')
self.assertEqual('lftx7dbyvlc5txtl.onion', hs)
with open(os.path.join(helpers.DATA_DIR,
'tor/ephemeral_service_key')) as f:
saved_key = f.read().strip()
self.assertEqual(saved_key, pk)
store.close()
def _initStartDB(self, target_ver):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
os.mkdir(GLSettings.db_path)
db_name = 'glbackend-%d.db' % target_ver
db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'db', 'populated', db_name)
shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name))
self.db_file = os.path.join(GLSettings.db_path, db_name)
GLSettings.db_uri = GLSettings.make_db_uri(self.db_file)
self.store = Store(create_database(GLSettings.db_uri))
def _test(self, path, f):
helpers.init_glsettings_for_unit_tests()
GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test')
final_db_file = os.path.abspath(
os.path.join(GLSettings.db_path,
'glbackend-%d.db' % DATABASE_VERSION))
GLSettings.db_uri = GLSettings.make_db_uri(final_db_file)
os.mkdir(GLSettings.db_path)
dbpath = os.path.join(path, f)
dbfile = os.path.join(GLSettings.db_path, f)
shutil.copyfile(dbpath, dbfile)
ret = perform_system_update()
shutil.rmtree(GLSettings.db_path)
self.assertNotEqual(ret, -1)
def perform_data_update(dbfile):
store = Store(create_database(GLSettings.make_db_uri(dbfile)))
enabled_languages = [lang.name for lang in store.find(l10n.EnabledLanguage)]
removed_languages = list(set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES))
if len(removed_languages):
removed_languages.sort()
removed_languages = ', '.join(removed_languages)
raise Exception("FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n"
"Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop" % removed_languages)
try:
db_perform_data_update(store)
store.commit()
except:
store.rollback()
raise
finally:
store.close()
def perform_schema_migration(version):
"""
@param version:
@return:
"""
to_delete_on_fail = []
to_delete_on_success = []
if version < FIRST_DATABASE_VERSION_SUPPORTED:
GLSettings.print_msg(
"Migrations from DB version lower than %d are no longer supported!"
% FIRST_DATABASE_VERSION_SUPPORTED)
quit()
tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp'))
orig_db_file = os.path.abspath(
os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version))
final_db_file = os.path.abspath(
os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))
shutil.rmtree(tmpdir, True)
os.mkdir(tmpdir)
shutil.copy2(orig_db_file, tmpdir)
new_db_file = None
try:
while version < DATABASE_VERSION:
old_db_file = os.path.abspath(
os.path.join(tmpdir, 'glbackend-%d.db' % version))
new_db_file = os.path.abspath(
os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1)))
GLSettings.db_file = new_db_file
GLSettings.enable_input_length_checks = False
to_delete_on_fail.append(new_db_file)
to_delete_on_success.append(old_db_file)
GLSettings.print_msg("Updating DB from version %d to version %d" %
(version, version + 1))
store_old = Store(create_database('sqlite:' + old_db_file))
store_new = Store(create_database('sqlite:' + new_db_file))
# Here is instanced the migration script
MigrationModule = importlib.import_module(
"globaleaks.db.migrations.update_%d" % (version + 1))
migration_script = MigrationModule.MigrationScript(
migration_mapping, version, store_old, store_new)
GLSettings.print_msg("Migrating table:")
try:
try:
migration_script.prologue()
except Exception as exception:
GLSettings.print_msg(
"Failure while executing migration prologue: %s" %
exception)
raise exception
for model_name, _ in migration_mapping.iteritems():
if migration_script.model_from[
model_name] is not None and migration_script.model_to[
model_name] is not None:
try:
migration_script.migrate_model(model_name)
# Commit at every table migration in order to be able to detect
# the precise migration that may fail.
migration_script.commit()
except Exception as exception:
GLSettings.print_msg(
"Failure while migrating table %s: %s " %
(model_name, exception))
raise exception
try:
migration_script.epilogue()
migration_script.commit()
except Exception as exception:
GLSettings.print_msg(
"Failure while executing migration epilogue: %s " %
exception)
raise exception
finally:
# the database should be always closed before leaving the application
# in order to not keep leaking journal files.
migration_script.close()
GLSettings.print_msg("Migration stats:")
# we open a new db in order to verify integrity of the generated file
store_verify = Store(
create_database(GLSettings.make_db_uri(new_db_file)))
for model_name, _ in migration_mapping.iteritems():
if model_name == 'ApplicationData':
continue
if migration_script.model_from[
model_name] is not None and migration_script.model_to[
model_name] is not None:
count = store_verify.find(
migration_script.model_to[model_name]).count()
if migration_script.entries_count[model_name] != count:
if migration_script.fail_on_count_mismatch[model_name]:
raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \
(model_name, count, migration_script.entries_count[model_name]))
else:
GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \
(model_name, migration_script.entries_count[model_name], count))
else:
GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \
(model_name, migration_script.entries_count[model_name]))
version += 1
store_verify.close()
perform_data_update(new_db_file)
except Exception as exception:
# simply propagate the exception
raise exception
else:
# in case of success first copy the new migrated db, then as last action delete the original db file
shutil.copy(new_db_file, final_db_file)
security.overwrite_and_remove(orig_db_file)
finally:
# Always cleanup the temporary directory used for the migration
for f in os.listdir(tmpdir):
tmp_db_file = os.path.join(tmpdir, f)
security.overwrite_and_remove(tmp_db_file)
shutil.rmtree(tmpdir)
def perform_schema_migration(version):
"""
@param version:
@return:
"""
to_delete_on_fail = []
to_delete_on_success = []
if version < FIRST_DATABASE_VERSION_SUPPORTED:
GLSettings.print_msg("Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED)
quit()
tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp'))
orig_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version))
final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION))
shutil.rmtree(tmpdir, True)
os.mkdir(tmpdir)
shutil.copy2(orig_db_file, tmpdir)
new_db_file = None
try:
while version < DATABASE_VERSION:
old_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % version))
new_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1)))
GLSettings.db_file = new_db_file
GLSettings.enable_input_length_checks = False
to_delete_on_fail.append(new_db_file)
to_delete_on_success.append(old_db_file)
GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1))
store_old = Store(create_database('sqlite:' + old_db_file))
store_new = Store(create_database('sqlite:' + new_db_file))
# Here is instanced the migration script
MigrationModule = importlib.import_module("globaleaks.db.migrations.update_%d" % (version + 1))
migration_script = MigrationModule.MigrationScript(migration_mapping, version, store_old, store_new)
GLSettings.print_msg("Migrating table:")
try:
try:
migration_script.prologue()
except Exception as exception:
GLSettings.print_msg("Failure while executing migration prologue: %s" % exception)
raise exception
for model_name, _ in migration_mapping.iteritems():
if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None:
try:
migration_script.migrate_model(model_name)
# Commit at every table migration in order to be able to detect
# the precise migration that may fail.
migration_script.commit()
except Exception as exception:
GLSettings.print_msg("Failure while migrating table %s: %s " % (model_name, exception))
raise exception
try:
migration_script.epilogue()
migration_script.commit()
except Exception as exception:
GLSettings.print_msg("Failure while executing migration epilogue: %s " % exception)
raise exception
finally:
# the database should be always closed before leaving the application
# in order to not keep leaking journal files.
migration_script.close()
GLSettings.print_msg("Migration stats:")
# we open a new db in order to verify integrity of the generated file
store_verify = Store(create_database(GLSettings.make_db_uri(new_db_file)))
for model_name, _ in migration_mapping.iteritems():
if model_name == 'ApplicationData':
continue
if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None:
count = store_verify.find(migration_script.model_to[model_name]).count()
if migration_script.entries_count[model_name] != count:
if migration_script.fail_on_count_mismatch[model_name]:
raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \
(model_name, count, migration_script.entries_count[model_name]))
else:
GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \
(model_name, migration_script.entries_count[model_name], count))
else:
GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \
(model_name, migration_script.entries_count[model_name]))
version += 1
store_verify.close()
perform_data_update(new_db_file)
except Exception as exception:
GLSettings.print_msg("[FATAL]: ", exception)
raise exception
else:
# in case of success first copy the new migrated db, then as last action delete the original db file
shutil.copy(new_db_file, final_db_file)
security.overwrite_and_remove(orig_db_file)
finally:
# Always cleanup the temporary directory used for the migration
for f in os.listdir(tmpdir):
tmp_db_file = os.path.join(tmpdir, f)
security.overwrite_and_remove(tmp_db_file)
shutil.rmtree(tmpdir)
