def _test(self, path, version): f = 'glbackend-%d.db' % version helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') self.start_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) self.final_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) self.start_db_uri = GLSettings.make_db_uri(self.start_db_file) GLSettings.db_uri = GLSettings.make_db_uri(self.final_db_file) shutil.rmtree(GLSettings.db_path, True) os.mkdir(GLSettings.db_path) dbpath = os.path.join(path, f) dbfile = os.path.join(GLSettings.db_path, f) shutil.copyfile(dbpath, dbfile) # TESTS PRECONDITIONS preconditions = getattr(self, 'preconditions_%d' % version, None) if preconditions is not None: preconditions() ret = update_db() # TESTS POSTCONDITIONS postconditions = getattr(self, 'postconditions_%d' % version, None) if postconditions is not None: postconditions() shutil.rmtree(GLSettings.db_path) self.assertNotEqual(ret, -1)
def perform_data_update(dbfile): store = Store(create_database(GLSettings.make_db_uri(dbfile))) enabled_languages = [ lang.name for lang in store.find(l10n.EnabledLanguage) ] removed_languages = list( set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES)) if len(removed_languages): removed_languages.sort() removed_languages = ', '.join(removed_languages) raise Exception( "FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n" "Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop" % removed_languages) try: db_perform_data_update(store) store.commit() except: store.rollback() raise finally: store.close()
def setUp(self): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % DATABASE_VERSION db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val('version', self.dummy_ver) self.assertEqual(prv.get_val('version'), self.dummy_ver) store.commit() store.close() # backup various mocks that we will use self._bck_f = config.is_cfg_valid GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop('smtp_password') self.dp = u'yes_you_really_should_change_me'
def test_mig_37_valid_tor_hs_key(self): self._initStartDB(36) from globaleaks.db.migrations import update_37 t = update_37.TOR_DIR update_37.TOR_DIR = GLSettings.db_path pk_path = os.path.join(update_37.TOR_DIR, 'private_key') hn_path = os.path.join(update_37.TOR_DIR, 'hostname') shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/private_key'), pk_path) shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/hostname'), hn_path) ret = update_db() self.assertEqual(ret, None) new_uri = GLSettings.make_db_uri( os.path.join(GLSettings.db_path, GLSettings.db_file_name)) store = Store(create_database(new_uri)) hs = config.NodeFactory(store).get_val('onionservice') pk = config.PrivateFactory(store).get_val('tor_onion_key') self.assertEqual('lftx7dbyvlc5txtl.onion', hs) with open(os.path.join(helpers.DATA_DIR, 'tor/ephemeral_service_key')) as f: saved_key = f.read().strip() self.assertEqual(saved_key, pk) store.close() shutil.rmtree(GLSettings.db_path) update_37.TOR_DIR = t
def setUp(self): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') shutil.rmtree(GLSettings.db_path, True) os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % DATABASE_VERSION db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val(u'version', self.dummy_ver) self.assertEqual(prv.get_val(u'version'), self.dummy_ver) store.commit() store.close() # backup various mocks that we will use self._bck_f = config.is_cfg_valid GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop( 'smtp_password') self.dp = u'yes_you_really_should_change_me'
def perform_data_update(dbfile): new_tmp_store = Store(create_database(GLSettings.make_db_uri(dbfile))) try: db_perform_data_update(new_tmp_store) new_tmp_store.commit() except: new_tmp_store.rollback() raise finally: new_tmp_store.close()
def _test(self, path, f): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) GLSettings.db_uri = GLSettings.make_db_uri(final_db_file) os.mkdir(GLSettings.db_path) dbpath = os.path.join(path, f) dbfile = os.path.join(GLSettings.db_path, f) shutil.copyfile(dbpath, dbfile) ret = perform_system_update() shutil.rmtree(GLSettings.db_path) self.assertNotEqual(ret, -1)
def _initStartDB(self, target_ver): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % target_ver db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) self.store = Store(create_database(GLSettings.db_uri))
def postconditions_36(self): new_uri = GLSettings.make_db_uri( os.path.join(GLSettings.db_path, GLSettings.db_file_name)) store = Store(create_database(new_uri)) hs = config.NodeFactory(store).get_val(u'onionservice') pk = config.PrivateFactory(store).get_val(u'tor_onion_key') self.assertEqual('lftx7dbyvlc5txtl.onion', hs) with open(os.path.join(helpers.DATA_DIR, 'tor/ephemeral_service_key')) as f: saved_key = f.read().strip() self.assertEqual(saved_key, pk) store.close()
def _test(self, path, f): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') final_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) GLSettings.db_uri = GLSettings.make_db_uri(final_db_file) os.mkdir(GLSettings.db_path) dbpath = os.path.join(path, f) dbfile = os.path.join(GLSettings.db_path, f) shutil.copyfile(dbpath, dbfile) ret = perform_system_update() shutil.rmtree(GLSettings.db_path) self.assertNotEqual(ret, -1)
def perform_data_update(dbfile): store = Store(create_database(GLSettings.make_db_uri(dbfile))) enabled_languages = [lang.name for lang in store.find(l10n.EnabledLanguage)] removed_languages = list(set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES)) if len(removed_languages): removed_languages.sort() removed_languages = ', '.join(removed_languages) raise Exception("FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n" "Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop" % removed_languages) try: db_perform_data_update(store) store.commit() except: store.rollback() raise finally: store.close()
def perform_schema_migration(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: GLSettings.print_msg( "Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED) quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module( "globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript( migration_mapping, version, store_old, store_new) GLSettings.print_msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: GLSettings.print_msg( "Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: GLSettings.print_msg( "Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: GLSettings.print_msg( "Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should be always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() GLSettings.print_msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store( create_database(GLSettings.make_db_uri(new_db_file))) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: count = store_verify.find( migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() perform_data_update(new_db_file) except Exception as exception: # simply propagate the exception raise exception else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) security.overwrite_and_remove(orig_db_file) finally: # Always cleanup the temporary directory used for the migration for f in os.listdir(tmpdir): tmp_db_file = os.path.join(tmpdir, f) security.overwrite_and_remove(tmp_db_file) shutil.rmtree(tmpdir)
def perform_schema_migration(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: GLSettings.print_msg("Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED) quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module("globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript(migration_mapping, version, store_old, store_new) GLSettings.print_msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: GLSettings.print_msg("Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should be always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() GLSettings.print_msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store(create_database(GLSettings.make_db_uri(new_db_file))) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: count = store_verify.find(migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() perform_data_update(new_db_file) except Exception as exception: GLSettings.print_msg("[FATAL]: ", exception) raise exception else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) security.overwrite_and_remove(orig_db_file) finally: # Always cleanup the temporary directory used for the migration for f in os.listdir(tmpdir): tmp_db_file = os.path.join(tmpdir, f) security.overwrite_and_remove(tmp_db_file) shutil.rmtree(tmpdir)