def clean_database(self, delete_folders=True): """ Deletes data from all tables """ self.cancel_all_operations() LOGGER.warning("Your Database content will be deleted.") try: session = SessionMaker() for table in reversed(model.Base.metadata.sorted_tables): # We don't delete data from some tables, because those are # imported only during introspection which is done one time if table.name not in self.EXCLUDE_TABLES: try: session.open_session() con = session.connection() LOGGER.debug("Executing Delete From Table " + table.name) con.execute(table.delete()) session.commit() except Exception as e: # We cache exception here, in case some table does not exists and # to allow the others to be deleted LOGGER.warning(e) session.rollback() finally: session.close_session() LOGGER.info("Database was cleanup!") except Exception as excep: LOGGER.warning(excep) raise # Now if the database is clean we can delete also project folders on disk if delete_folders: self.delete_project_folders() dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None))
def clean_database(self, delete_folders=True): """ Deletes data from all tables """ self.cancel_all_operations() LOGGER.warning("Your Database content will be deleted.") try: session = SessionMaker() for table in reversed(model.Base.metadata.sorted_tables): # We don't delete data from some tables, because those are # imported only during introspection which is done one time if table.name not in self.EXCLUDE_TABLES: try: session.open_session() con = session.connection() LOGGER.debug("Executing Delete From Table " + table.name) con.execute(table.delete()) session.commit() except Exception, e: # We cache exception here, in case some table does not exists and # to allow the others to be deleted LOGGER.warning(e) session.rollback() finally: session.close_session()
def get_all_entities(self, entity_type): """ Retrieve all entities of a given type. """ result = [] session = None try: session = SessionMaker() session.open_session() result = session.query(entity_type).all() except Exception, excep: LOGGER.warning(excep)
def count_all_entities(self, entity_type): """ Count all entities of a given type currently stored in DB. """ result = 0 session = None try: session = SessionMaker() session.open_session() result = session.query(entity_type).count() except Exception, excep: LOGGER.warning(excep)
def dec(*args, **kwargs): session_maker = SessionMaker() TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True session_maker.start_transaction() try: try: if hasattr(args[0], 'transactional_setup_method_TVB'): LOGGER.debug(args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional SETUP starting...") args[0].transactional_setup_method_TVB() result = func(*args, **kwargs) finally: if hasattr(args[0], 'transactional_teardown_method_TVB'): LOGGER.debug(args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional TEARDOWN starting...") args[0].transactional_teardown_method_TVB() args[0].delete_project_folders() finally: session_maker.rollback_transaction() session_maker.close_transaction() TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False if callback is not None: callback(*args, **kwargs) return result
def dec(*args, **kwargs): session_maker = SessionMaker() TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True session_maker.start_transaction() try: try: if hasattr(args[0], "setUpTVB"): LOGGER.debug( args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional SETUP starting..." ) args[0].setUpTVB() result = func(*args, **kwargs) finally: if hasattr(args[0], "tearDownTVB"): LOGGER.debug( args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional TEARDOWN starting..." ) args[0].tearDownTVB() args[0].delete_project_folders() finally: session_maker.rollback_transaction() session_maker.close_transaction() TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False if callback is not None: callback(*args, **kwargs) return result
def _evaluate_db_filter(self, filter_chain, expected_number): """ Evaluate filter on DB and assert number of results. """ session = SessionMaker() try: session.open_session() query = session.query(Datatype1) filter_str = filter_chain.get_sql_filter_equivalent("Datatype1") query = query.filter(eval(filter_str)) result = query.all() session.close_session() except Exception, excep: session.close_session() raise excep
def _evaluate_db_filter(self, filter_chain, expected_number): """ Evaluate filter on DB and assert number of results. """ session = SessionMaker() try: session.open_session() query = session.query(Datatype1) filter_str = filter_chain.get_sql_filter_equivalent("Datatype1") query = query.filter(eval(filter_str)) result = query.all() session.close_session() except Exception as excep: session.close_session() raise excep assert expected_number == len(result), "Expected %s DTs after filtering with %s, "\ "but got %s instead." % (expected_number, filter_chain, len(result,))
def _evaluate_db_filter(self, filter_chain, expected_number): """ Evaluate filter on DB and assert number of results. """ session = SessionMaker() try: session.open_session() query = session.query(Datatype1) filter_str = filter_chain.get_sql_filter_equivalent("Datatype1") query = query.filter(eval(filter_str)) result = query.all() session.close_session() except Exception as excep: session.close_session() raise excep self.assertEquals(expected_number, len(result), "Expected %s DTs after filtering with %s, " "but got %s instead." % (expected_number, filter_chain, len(result,)))
def clean_database(self, delete_folders=True): """ Deletes data from all tables """ self.cancel_all_operations() LOGGER.warning("Your Database content will be deleted.") try: session = SessionMaker() for table in reversed(model.Base.metadata.sorted_tables): # We don't delete data from some tables, because those are # imported only during introspection which is done one time if table.name not in self.EXCLUDE_TABLES: try: session.open_session() con = session.connection() LOGGER.debug("Executing Delete From Table " + table.name) con.execute(table.delete()) session.commit() except Exception as e: # We cache exception here, in case some table does not exists and # to allow the others to be deleted LOGGER.warning(e) session.rollback() finally: session.close_session() LOGGER.info("Database was cleanup!") except Exception as excep: LOGGER.warning(excep) raise # Now if the database is clean we can delete also project folders on disk if delete_folders: self.delete_project_folders() dao.store_entity( model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None))
class TestsTransactional(BaseTestCase): """ This class contains tests for the tvb.core.entities.modelmanager module. """ session = SessionMaker() def setup_method(self): """ Set-up the environment for testing; clean the database and save events """ self.clean_database() def teardown_method(self): """ Clean-up after testing; clean the database and restore events """ self.clean_database(True) def test_transaction_happy_flow(self): """ In case no exception is raised the transactional decorator should not influence the data in any way. A successfull commit will be made and the data should be visible in the database. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 n_of_users = 21 self._store_users_happy_flow(n_of_users) final_user_count = dao.get_all_users(is_count=True) error_msg = ( "Transaction should have committed and %s more users should have been available in the database. " "Expected %s but got %s" % (n_of_users, initial_user_count + n_of_users, final_user_count)) assert initial_user_count + n_of_users, final_user_count == error_msg def test_transaction_rollback(self): """ If an unhandled exception is raised by a method marked as transactional, all data should be rolled back properly. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 n_of_users = 6 try: self._store_users_raises_exception(n_of_users) except Exception: pass final_user_count = dao.get_all_users(is_count=True) assert initial_user_count == final_user_count, "Transaction should have rolled back due to exception." \ "Expected %s but got %s" % (initial_user_count, final_user_count) def test_add_entity_forget_commit(self): """ Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 self._dao_add_user_forget_commit() final_user_count = dao.get_all_users(is_count=True) assert initial_user_count + 1 == final_user_count, "Commit should have been done automatically and one " \ "more user expected. Expected %s but got %s" % ( initial_user_count, final_user_count) def test_edit_entity_forget_commit(self): """ Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes. """ stored_user = TestFactory.create_user('username', 'displayname', 'password', 'mail', True, 'role') user_id = stored_user.id self._dao_change_user_forget_commit(user_id, 'new_name') edited_user = dao.get_user_by_id(user_id) assert edited_user.username == 'new_name', \ "User should be edited but it is not. Expected 'new_name' got %s" % edited_user.username def test_delete_entity_forget_commit(self): """ Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 stored_user = TestFactory.create_user('username', 'displayname', 'password', 'mail', True, 'role') user_id = stored_user.id self._dao_delete_user_forget_commit(user_id) final_user_count = dao.get_all_users(is_count=True) assert initial_user_count == final_user_count, \ "Added user should have been deleted even without explicit commit call.." \ "Expected %s but got %s" % (initial_user_count, final_user_count) @pytest.mark.skipif def test_multi_threaded_access(self): """ Test that there is no problem with multiple threads accessing dao. Since cfg.MAX_THREADS_NO is set to 20 we just spawn 4 threads each storing 4 users. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 n_of_threads = 4 n_of_users_per_thread = 4 self._run_transaction_multiple_threads(n_of_threads, n_of_users_per_thread) final_user_count = dao.get_all_users(is_count=True) assert initial_user_count + n_of_threads * n_of_users_per_thread == final_user_count, \ "Each thread should have created %s more users to a total of 16." \ "Expected %s but got %s" % (n_of_threads * n_of_users_per_thread, initial_user_count + n_of_threads * n_of_users_per_thread, final_user_count) def test_multi_threaded_access_overflow_db_connection(self): """ Test that there is no problem with multiple threads accessing dao. Since cfg.MAX_THREADS_NO is set to 20 we just spawn 4 threads each storing 4 users. """ all_users = dao.get_all_users() initial_user_count = len(all_users) if all_users is not None else 0 n_of_threads = 18 n_of_users_per_thread = 6 self._run_transaction_multiple_threads(n_of_threads, n_of_users_per_thread) final_user_count = dao.get_all_users(is_count=True) assert initial_user_count + n_of_threads * n_of_users_per_thread == final_user_count, \ "Each of %s threads should have created %s more users to a total of %s. " \ "Expected %s but got %s" % ( n_of_threads, n_of_users_per_thread, n_of_threads * n_of_users_per_thread, initial_user_count + n_of_threads * n_of_users_per_thread, final_user_count) @transactional_test def test_transaction_nested(self): """ A case of nested transaction. Check that if the ALLOW_NESTED_TRANSACTIONS is set to false, an exception is raised when a nested transaction is attempted. This will be default behaviour in TVB, only overwritten for transactional tests. """ TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False try: with pytest.raises(NestedTransactionUnsupported): self._store_users_nested(4, self._store_users_happy_flow) finally: TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True def _run_transaction_multiple_threads(self, n_of_threads, n_of_users_per_thread): """ Spawn a number of threads each storing a number of users. Wait on them by joining. """ for idx in range(n_of_threads): th = threading.Thread(target=self._store_users_happy_flow, args=(n_of_users_per_thread, ), kwargs={'prefix': str(idx)}) th.start() for t in threading.enumerate(): if t is threading.currentThread(): continue t.join() @add_session def _dao_add_user_forget_commit(self): """ Test use case where you add user but forget to commit. This should be handled automatically. """ self.session.add( model_project.User('username', 'name', 'password', 'mail', True, 'role')) @add_session def _dao_change_user_forget_commit(self, user_id, new_name): """ Test use case where you add user but forget to commit. This should be handled automatically. """ user = self.session.query( model_project.User).filter(model_project.User.id == user_id).one() user.username = new_name @add_session def _dao_delete_user_forget_commit(self, user_id): """ Test use case where you add user but forget to commit. This should be handled automatically. """ user = self.session.query( model_project.User).filter(model_project.User.id == user_id).one() self.session.delete(user) @transactional def _store_users_nested(self, n_users, inner_trans_func): """ This method stores n_users, after which it calls inner_trans_func with n_users as parameter. At the end it raises an exception so transaction will fail. All changes should be reverted regardless if inner_trans_func succeeds or fails. :param n_users: number of users to be stored both by this method and by the passed inner_trans_func :param inner_trans_func: either _store_users_happy_flow or _store_users_raises_exception """ for idx in range(n_users): TestFactory.create_user('test_user_nested' + str(idx), 'test_user_nested' + str(idx), 'pass', '*****@*****.**', True, 'test') inner_trans_func(n_users) raise Exception( "This is just so transactional kicks in and a rollback should be done." ) @transactional def _store_users_happy_flow(self, n_users, prefix=""): """ Store users in happy flow. In this case the transaction should just be commited properly and changes should be visible in database. :param n_users: number of users to be stored by this method """ for idx in range(n_users): TestFactory.create_user(prefix + 'test_user' + str(idx), prefix + 'test_user' + str(idx), 'pass', '*****@*****.**', True, 'test') @transactional def _store_users_raises_exception(self, n_users): """ Store users but at the end raise an exception. In case the exception is not handled up until the transactional decorator, all changes should be rolled back. :param n_users: number of users to be stored by this method """ for idx in range(n_users): TestFactory.create_user('test_user' + str(idx), 'test_user' + str(idx), 'pass', '*****@*****.**', True, 'test') raise Exception( "This is just so transactional kicks in and a rollback should be done." )
""" .. moduleauthor:: Bogdan Neacsa <*****@*****.**> """ import unittest import threading import tvb.config as config from tvb.basic.profile import TvbProfile from tvb.core.entities import model from tvb.core.entities.storage import dao, transactional from tvb.core.entities.storage.session_maker import add_session, SessionMaker from tvb.core.entities.storage.exceptions import NestedTransactionUnsupported from tvb.tests.framework.core.test_factory import TestFactory from tvb.tests.framework.core.base_testcase import BaseTestCase, transactional_test SESSIONMAKER = SessionMaker() class TransactionalTests(BaseTestCase): """ This class contains tests for the tvb.core.entities.modelmanager module. """ session = SessionMaker() def setUp(self): """ Set-up the environment for testing; clean the database and save events """ self.clean_database() self.old_events = config.EVENTS_FOLDER