def _wrap(self, function, *args, **kwargs): """ Wrap provided function calling it inside a thread and passing the store to it. """ with transact_lock: start_time = datetime.now() store = Store(create_database(GLSettings.db_uri)) try: if self.instance: result = function(self.instance, store, *args, **kwargs) else: result = function(store, *args, **kwargs) store.commit() except: store.rollback() raise else: return result finally: store.reset() store.close() duration = timedelta_to_milliseconds(datetime.now() - start_time) msg = "Query [%s] executed in %.1fms" % (self.method.__name__, duration) if duration > self.timelimit: log.err(msg) schedule_exception_email(msg) else: log.debug(msg)
def perform_data_update(dbfile): store = Store(create_database(GLSettings.make_db_uri(dbfile))) enabled_languages = [ lang.name for lang in store.find(l10n.EnabledLanguage) ] removed_languages = list( set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES)) if len(removed_languages): removed_languages.sort() removed_languages = ', '.join(removed_languages) raise Exception( "FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n" "Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop" % removed_languages) try: db_perform_data_update(store) store.commit() except: store.rollback() raise finally: store.close()
def createUserDB(connString): '''Creates user database and default admin account with password admin''' from tsload.user.localauth import LocalAuth database = create_database(connString) store = Store(database) TableSchema(database, User).create(store) TableSchema(database, Role).create(store) localAuth = LocalAuth() admin = User() admin.name = 'admin' admin.gecosName = u'TSLoad Administrator' admin.authService = 'local' localAuth.changePassword(admin, 'admin') store.add(admin) adminRole = Role() adminRole.user = admin adminRole.role = 'admin' store.add(adminRole) store.commit() store.close()
def createExpsvcDB(connString): database = create_database(connString) store = Store(database) TableSchema(database, Agent).create(store) TableSchema(database, AgentResource).create(store) TableSchema(database, AgentResourceChild).create(store) TableSchema(database, WorkloadType).create(store) TableSchema(database, WorkloadParam).create(store) TableSchema(database, ExperimentProfile).create(store) TableSchema(database, ExperimentThreadPool).create(store) TableSchema(database, ExperimentWorkload).create(store) TableSchema(database, ExperimentWorkloadResource).create(store) store.commit() store.close()
def test_json_property(self): """The JSON property is encoded as JSON""" class TestModel(object): __storm_table__ = "json_test" id = Int(primary=True) json = JSON() connection = self.database.connect() value = {"a": 3, "b": "foo", "c": None} db_value = json.dumps(value).decode("utf-8") connection.execute("INSERT INTO json_test (json) VALUES (?)", (db_value, )) connection.commit() store = Store(self.database) obj = store.find(TestModel).one() store.close() # The JSON object is decoded to python self.assertEqual(value, obj.json)
def perform_data_update(dbfile): store = Store(create_database(GLSettings.make_db_uri(dbfile))) enabled_languages = [lang.name for lang in store.find(l10n.EnabledLanguage)] removed_languages = list(set(enabled_languages) - set(LANGUAGES_SUPPORTED_CODES)) if len(removed_languages): removed_languages.sort() removed_languages = ', '.join(removed_languages) raise Exception("FATAL: cannot complete the upgrade because the support for some of the enabled languages is currently incomplete (%s)\n" "Read about how to handle this condition at: https://github.com/globaleaks/GlobaLeaks/wiki/Upgrade-Guide#lang-drop" % removed_languages) try: db_perform_data_update(store) store.commit() except: store.rollback() raise finally: store.close()
def _wrap(self, function, *args, **kwargs): """ Wrap provided function calling it inside a thread and passing the store to it. """ with transact_lock: store = Store(create_database(GLSettings.db_uri)) try: if self.instance: result = function(self.instance, store, *args, **kwargs) else: result = function(store, *args, **kwargs) store.commit() except: store.rollback() raise else: return result finally: store.reset() store.close()
def perform_schema_migration(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: GLSettings.print_msg( "Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED) quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath( os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath( os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module( "globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript( migration_mapping, version, store_old, store_new) GLSettings.print_msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: GLSettings.print_msg( "Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: GLSettings.print_msg( "Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: GLSettings.print_msg( "Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should be always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() GLSettings.print_msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store( create_database(GLSettings.make_db_uri(new_db_file))) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[ model_name] is not None and migration_script.model_to[ model_name] is not None: count = store_verify.find( migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() perform_data_update(new_db_file) except Exception as exception: # simply propagate the exception raise exception else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) security.overwrite_and_remove(orig_db_file) finally: # Always cleanup the temporary directory used for the migration for f in os.listdir(tmpdir): tmp_db_file = os.path.join(tmpdir, f) security.overwrite_and_remove(tmp_db_file) shutil.rmtree(tmpdir)
def perform_schema_migration(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: GLSettings.print_msg("Migrations from DB version lower than %d are no longer supported!" % FIRST_DATABASE_VERSION_SUPPORTED) quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module("globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript(migration_mapping, version, store_old, store_new) GLSettings.print_msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: GLSettings.print_msg("Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should be always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() GLSettings.print_msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store(create_database(GLSettings.make_db_uri(new_db_file))) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: count = store_verify.find(migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() perform_data_update(new_db_file) except Exception as exception: GLSettings.print_msg("[FATAL]: ", exception) raise exception else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) security.overwrite_and_remove(orig_db_file) finally: # Always cleanup the temporary directory used for the migration for f in os.listdir(tmpdir): tmp_db_file = os.path.join(tmpdir, f) security.overwrite_and_remove(tmp_db_file) shutil.rmtree(tmpdir)
class TSUserAgent(TSLocalAgent): agentId = userAgentId uuid = userAgentUUID agentType = userAgentType def __init__(self, server, connString): TSLocalAgent.__init__(self, server) self.client.getId() self.logger = logging.getLogger('UserAgent') self.rootAgent = server.localAgents[0] self.agentUsers = {} self.authServices = {'local': LocalAuth()} self.database = create_database(connString) self.dbStore = Store(self.database) self.server.listenerFlows.append( Flow(dstAgentId=userAgentId, command='authUser')) @TSMethodImpl(UserAgent.authUser) def authUser(self, context, **kw): @inlineCallbacks def implementation(context, userName, userPassword): userSet = yield self.dbStore.find(User, User.name == str(userName)) user = yield userSet.one() self.logger.info('Authorizing user %s', userName) if user is None: self.logger.warning('Error authorizing user: no such user: %s', userName) raise UserAuthError('No such user: %s' % userName) authMethod = self.authServices[user.authService] if authMethod.authentificate(user, userPassword): agentId = context.client.getId() self.agentUsers[agentId] = user.id roles = yield user.roles role = self._setupRoles(context.client, roles) userDescr = TSUserDescriptor() userDescr.name = user.gecosName userDescr.role = role returnValue(userDescr) return implementation(context, **kw) def _setupRoles(self, client, roles): # First pass - identify maximum role maxRole = TSServerClient.AUTH_NONE for role in roles: if role.role == 'admin': maxRole = TSServerClient.AUTH_ADMIN elif role.role == 'operator' and maxRole != TSServerClient.AUTH_ADMIN: maxRole = TSServerClient.AUTH_OPERATOR else: maxRole = TSServerClient.AUTH_USER client.authorize(maxRole) if maxRole != TSServerClient.AUTH_ADMIN: # TODO: For user/operator need to set ACLs pass return maxRole def onDisconnect(self): self.dbStore.close()
class TSUserAgent(TSLocalAgent): agentId = userAgentId uuid = userAgentUUID agentType = userAgentType def __init__(self, server, connString): TSLocalAgent.__init__(self, server) self.client.getId() self.logger = logging.getLogger('UserAgent') self.rootAgent = server.localAgents[0] self.agentUsers = {} self.authServices = {'local': LocalAuth()} self.database = create_database(connString) self.dbStore = Store(self.database) self.server.listenerFlows.append(Flow(dstAgentId = userAgentId, command = 'authUser')) @TSMethodImpl(UserAgent.authUser) def authUser(self, context, **kw): @inlineCallbacks def implementation(context, userName, userPassword): userSet = yield self.dbStore.find(User, User.name == str(userName)) user = yield userSet.one() self.logger.info('Authorizing user %s', userName) if user is None: self.logger.warning('Error authorizing user: no such user: %s', userName) raise UserAuthError('No such user: %s' % userName) authMethod = self.authServices[user.authService] if authMethod.authentificate(user, userPassword): agentId = context.client.getId() self.agentUsers[agentId] = user.id roles = yield user.roles role = self._setupRoles(context.client, roles) userDescr = TSUserDescriptor() userDescr.name = user.gecosName userDescr.role = role returnValue(userDescr) return implementation(context, **kw) def _setupRoles(self, client, roles): # First pass - identify maximum role maxRole = TSServerClient.AUTH_NONE for role in roles: if role.role == 'admin': maxRole = TSServerClient.AUTH_ADMIN elif role.role == 'operator' and maxRole != TSServerClient.AUTH_ADMIN: maxRole = TSServerClient.AUTH_OPERATOR else: maxRole = TSServerClient.AUTH_USER client.authorize(maxRole) if maxRole != TSServerClient.AUTH_ADMIN: # TODO: For user/operator need to set ACLs pass return maxRole def onDisconnect(self): self.dbStore.close()