def clean_database(self, delete_folders=True):
"""
Deletes data from all tables
"""
self.cancel_all_operations()
LOGGER.warning("Your Database content will be deleted.")
try:
session = SessionMaker()
for table in reversed(model.Base.metadata.sorted_tables):
# We don't delete data from some tables, because those are
# imported only during introspection which is done one time
if table.name not in self.EXCLUDE_TABLES:
try:
session.open_session()
con = session.connection()
LOGGER.debug("Executing Delete From Table " + table.name)
con.execute(table.delete())
session.commit()
except Exception as e:
# We cache exception here, in case some table does not exists and
# to allow the others to be deleted
LOGGER.warning(e)
session.rollback()
finally:
session.close_session()
LOGGER.info("Database was cleanup!")
except Exception as excep:
LOGGER.warning(excep)
raise
# Now if the database is clean we can delete also project folders on disk
if delete_folders:
self.delete_project_folders()
dao.store_entity(model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None, None, True, None))
def clean_database(self, delete_folders=True):
"""
Deletes data from all tables
"""
self.cancel_all_operations()
LOGGER.warning("Your Database content will be deleted.")
try:
session = SessionMaker()
for table in reversed(model.Base.metadata.sorted_tables):
# We don't delete data from some tables, because those are
# imported only during introspection which is done one time
if table.name not in self.EXCLUDE_TABLES:
try:
session.open_session()
con = session.connection()
LOGGER.debug("Executing Delete From Table " + table.name)
con.execute(table.delete())
session.commit()
except Exception, e:
# We cache exception here, in case some table does not exists and
# to allow the others to be deleted
LOGGER.warning(e)
session.rollback()
finally:
session.close_session()
def get_all_entities(self, entity_type):
"""
Retrieve all entities of a given type.
"""
result = []
session = None
try:
session = SessionMaker()
session.open_session()
result = session.query(entity_type).all()
except Exception, excep:
LOGGER.warning(excep)
def count_all_entities(self, entity_type):
"""
Count all entities of a given type currently stored in DB.
"""
result = 0
session = None
try:
session = SessionMaker()
session.open_session()
result = session.query(entity_type).count()
except Exception, excep:
LOGGER.warning(excep)
def get_all_entities(self, entity_type):
"""
Retrieve all entities of a given type.
"""
result = []
session = None
try:
session = SessionMaker()
session.open_session()
result = session.query(entity_type).all()
except Exception, excep:
LOGGER.warning(excep)
def count_all_entities(self, entity_type):
"""
Count all entities of a given type currently stored in DB.
"""
result = 0
session = None
try:
session = SessionMaker()
session.open_session()
result = session.query(entity_type).count()
except Exception, excep:
LOGGER.warning(excep)
def dec(*args, **kwargs):
session_maker = SessionMaker()
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True
session_maker.start_transaction()
try:
try:
if hasattr(args[0], 'transactional_setup_method_TVB'):
LOGGER.debug(args[0].__class__.__name__ + "->" +
func.__name__ +
"- Transactional SETUP starting...")
args[0].transactional_setup_method_TVB()
result = func(*args, **kwargs)
finally:
if hasattr(args[0], 'transactional_teardown_method_TVB'):
LOGGER.debug(args[0].__class__.__name__ + "->" +
func.__name__ +
"- Transactional TEARDOWN starting...")
args[0].transactional_teardown_method_TVB()
args[0].delete_project_folders()
finally:
session_maker.rollback_transaction()
session_maker.close_transaction()
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False
if callback is not None:
callback(*args, **kwargs)
return result
def dec(*args, **kwargs):
session_maker = SessionMaker()
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True
session_maker.start_transaction()
try:
try:
if hasattr(args[0], "setUpTVB"):
LOGGER.debug(
args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional SETUP starting..."
)
args[0].setUpTVB()
result = func(*args, **kwargs)
finally:
if hasattr(args[0], "tearDownTVB"):
LOGGER.debug(
args[0].__class__.__name__ + "->" + func.__name__ + "- Transactional TEARDOWN starting..."
)
args[0].tearDownTVB()
args[0].delete_project_folders()
finally:
session_maker.rollback_transaction()
session_maker.close_transaction()
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False
if callback is not None:
callback(*args, **kwargs)
return result
def _evaluate_db_filter(self, filter_chain, expected_number):
"""
Evaluate filter on DB and assert number of results.
"""
session = SessionMaker()
try:
session.open_session()
query = session.query(Datatype1)
filter_str = filter_chain.get_sql_filter_equivalent("Datatype1")
query = query.filter(eval(filter_str))
result = query.all()
session.close_session()
except Exception, excep:
session.close_session()
raise excep
def _evaluate_db_filter(self, filter_chain, expected_number):
"""
Evaluate filter on DB and assert number of results.
"""
session = SessionMaker()
try:
session.open_session()
query = session.query(Datatype1)
filter_str = filter_chain.get_sql_filter_equivalent("Datatype1")
query = query.filter(eval(filter_str))
result = query.all()
session.close_session()
except Exception as excep:
session.close_session()
raise excep
assert expected_number == len(result), "Expected %s DTs after filtering with %s, "\
"but got %s instead." % (expected_number, filter_chain, len(result,))
def _evaluate_db_filter(self, filter_chain, expected_number):
"""
Evaluate filter on DB and assert number of results.
"""
session = SessionMaker()
try:
session.open_session()
query = session.query(Datatype1)
filter_str = filter_chain.get_sql_filter_equivalent("Datatype1")
query = query.filter(eval(filter_str))
result = query.all()
session.close_session()
except Exception, excep:
session.close_session()
raise excep
def _evaluate_db_filter(self, filter_chain, expected_number):
"""
Evaluate filter on DB and assert number of results.
"""
session = SessionMaker()
try:
session.open_session()
query = session.query(Datatype1)
filter_str = filter_chain.get_sql_filter_equivalent("Datatype1")
query = query.filter(eval(filter_str))
result = query.all()
session.close_session()
except Exception as excep:
session.close_session()
raise excep
self.assertEquals(expected_number, len(result), "Expected %s DTs after filtering with %s, "
"but got %s instead." % (expected_number, filter_chain, len(result,)))
def clean_database(self, delete_folders=True):
"""
Deletes data from all tables
"""
self.cancel_all_operations()
LOGGER.warning("Your Database content will be deleted.")
try:
session = SessionMaker()
for table in reversed(model.Base.metadata.sorted_tables):
# We don't delete data from some tables, because those are
# imported only during introspection which is done one time
if table.name not in self.EXCLUDE_TABLES:
try:
session.open_session()
con = session.connection()
LOGGER.debug("Executing Delete From Table " +
table.name)
con.execute(table.delete())
session.commit()
except Exception as e:
# We cache exception here, in case some table does not exists and
# to allow the others to be deleted
LOGGER.warning(e)
session.rollback()
finally:
session.close_session()
LOGGER.info("Database was cleanup!")
except Exception as excep:
LOGGER.warning(excep)
raise
# Now if the database is clean we can delete also project folders on disk
if delete_folders:
self.delete_project_folders()
dao.store_entity(
model.User(TvbProfile.current.web.admin.SYSTEM_USER_NAME, None,
None, True, None))
class TestsTransactional(BaseTestCase):
"""
This class contains tests for the tvb.core.entities.modelmanager module.
"""
session = SessionMaker()
def setup_method(self):
"""
Set-up the environment for testing; clean the database and save events
"""
self.clean_database()
def teardown_method(self):
"""
Clean-up after testing; clean the database and restore events
"""
self.clean_database(True)
def test_transaction_happy_flow(self):
"""
In case no exception is raised the transactional decorator should not influence the data in any way.
A successfull commit will be made and the data should be visible in the database.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
n_of_users = 21
self._store_users_happy_flow(n_of_users)
final_user_count = dao.get_all_users(is_count=True)
error_msg = (
"Transaction should have committed and %s more users should have been available in the database. "
"Expected %s but got %s" %
(n_of_users, initial_user_count + n_of_users, final_user_count))
assert initial_user_count + n_of_users, final_user_count == error_msg
def test_transaction_rollback(self):
"""
If an unhandled exception is raised by a method marked as transactional, all data should be rolled
back properly.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
n_of_users = 6
try:
self._store_users_raises_exception(n_of_users)
except Exception:
pass
final_user_count = dao.get_all_users(is_count=True)
assert initial_user_count == final_user_count, "Transaction should have rolled back due to exception." \
"Expected %s but got %s" % (initial_user_count, final_user_count)
def test_add_entity_forget_commit(self):
"""
Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
self._dao_add_user_forget_commit()
final_user_count = dao.get_all_users(is_count=True)
assert initial_user_count + 1 == final_user_count, "Commit should have been done automatically and one " \
"more user expected. Expected %s but got %s" % (
initial_user_count, final_user_count)
def test_edit_entity_forget_commit(self):
"""
Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes.
"""
stored_user = TestFactory.create_user('username', 'displayname',
'password', 'mail', True, 'role')
user_id = stored_user.id
self._dao_change_user_forget_commit(user_id, 'new_name')
edited_user = dao.get_user_by_id(user_id)
assert edited_user.username == 'new_name', \
"User should be edited but it is not. Expected 'new_name' got %s" % edited_user.username
def test_delete_entity_forget_commit(self):
"""
Commit should be done automatically if you forget for some reason to do so in case of new/update/deletes.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
stored_user = TestFactory.create_user('username', 'displayname',
'password', 'mail', True, 'role')
user_id = stored_user.id
self._dao_delete_user_forget_commit(user_id)
final_user_count = dao.get_all_users(is_count=True)
assert initial_user_count == final_user_count, \
"Added user should have been deleted even without explicit commit call.." \
"Expected %s but got %s" % (initial_user_count, final_user_count)
@pytest.mark.skipif
def test_multi_threaded_access(self):
"""
Test that there is no problem with multiple threads accessing dao. Since cfg.MAX_THREADS_NO is set to 20 we just
spawn 4 threads each storing 4 users.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
n_of_threads = 4
n_of_users_per_thread = 4
self._run_transaction_multiple_threads(n_of_threads,
n_of_users_per_thread)
final_user_count = dao.get_all_users(is_count=True)
assert initial_user_count + n_of_threads * n_of_users_per_thread == final_user_count, \
"Each thread should have created %s more users to a total of 16." \
"Expected %s but got %s" % (n_of_threads * n_of_users_per_thread,
initial_user_count + n_of_threads * n_of_users_per_thread,
final_user_count)
def test_multi_threaded_access_overflow_db_connection(self):
"""
Test that there is no problem with multiple threads accessing dao. Since cfg.MAX_THREADS_NO is set to 20 we just
spawn 4 threads each storing 4 users.
"""
all_users = dao.get_all_users()
initial_user_count = len(all_users) if all_users is not None else 0
n_of_threads = 18
n_of_users_per_thread = 6
self._run_transaction_multiple_threads(n_of_threads,
n_of_users_per_thread)
final_user_count = dao.get_all_users(is_count=True)
assert initial_user_count + n_of_threads * n_of_users_per_thread == final_user_count, \
"Each of %s threads should have created %s more users to a total of %s. " \
"Expected %s but got %s" % (
n_of_threads, n_of_users_per_thread, n_of_threads * n_of_users_per_thread,
initial_user_count + n_of_threads * n_of_users_per_thread, final_user_count)
@transactional_test
def test_transaction_nested(self):
"""
A case of nested transaction. Check that if the ALLOW_NESTED_TRANSACTIONS is set to false,
an exception is raised when a nested transaction is attempted. This will be default behaviour in TVB, only
overwritten for transactional tests.
"""
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = False
try:
with pytest.raises(NestedTransactionUnsupported):
self._store_users_nested(4, self._store_users_happy_flow)
finally:
TvbProfile.current.db.ALLOW_NESTED_TRANSACTIONS = True
def _run_transaction_multiple_threads(self, n_of_threads,
n_of_users_per_thread):
"""
Spawn a number of threads each storing a number of users. Wait on them by joining.
"""
for idx in range(n_of_threads):
th = threading.Thread(target=self._store_users_happy_flow,
args=(n_of_users_per_thread, ),
kwargs={'prefix': str(idx)})
th.start()
for t in threading.enumerate():
if t is threading.currentThread():
continue
t.join()
@add_session
def _dao_add_user_forget_commit(self):
"""
Test use case where you add user but forget to commit. This should be handled automatically.
"""
self.session.add(
model_project.User('username', 'name', 'password', 'mail', True,
'role'))
@add_session
def _dao_change_user_forget_commit(self, user_id, new_name):
"""
Test use case where you add user but forget to commit. This should be handled automatically.
"""
user = self.session.query(
model_project.User).filter(model_project.User.id == user_id).one()
user.username = new_name
@add_session
def _dao_delete_user_forget_commit(self, user_id):
"""
Test use case where you add user but forget to commit. This should be handled automatically.
"""
user = self.session.query(
model_project.User).filter(model_project.User.id == user_id).one()
self.session.delete(user)
@transactional
def _store_users_nested(self, n_users, inner_trans_func):
"""
This method stores n_users, after which it calls inner_trans_func with n_users as parameter.
At the end it raises an exception so transaction will fail.
All changes should be reverted regardless if inner_trans_func succeeds or fails.
:param n_users: number of users to be stored both by this method and by the passed inner_trans_func
:param inner_trans_func: either _store_users_happy_flow or _store_users_raises_exception
"""
for idx in range(n_users):
TestFactory.create_user('test_user_nested' + str(idx),
'test_user_nested' + str(idx), 'pass',
'*****@*****.**', True, 'test')
inner_trans_func(n_users)
raise Exception(
"This is just so transactional kicks in and a rollback should be done."
)
@transactional
def _store_users_happy_flow(self, n_users, prefix=""):
"""
Store users in happy flow. In this case the transaction should just be commited properly and changes
should be visible in database.
:param n_users: number of users to be stored by this method
"""
for idx in range(n_users):
TestFactory.create_user(prefix + 'test_user' + str(idx),
prefix + 'test_user' + str(idx), 'pass',
'*****@*****.**', True, 'test')
@transactional
def _store_users_raises_exception(self, n_users):
"""
Store users but at the end raise an exception. In case the exception is not handled up until the
transactional decorator, all changes should be rolled back.
:param n_users: number of users to be stored by this method
"""
for idx in range(n_users):
TestFactory.create_user('test_user' + str(idx),
'test_user' + str(idx), 'pass',
'*****@*****.**', True, 'test')
raise Exception(
"This is just so transactional kicks in and a rollback should be done."
)
def clean_database(self, delete_folders=True):
"""
Deletes data from all tables
"""
self.cancel_all_operations()
LOGGER.warning("Your Database content will be deleted.")
try:
session = SessionMaker()
for table in reversed(model.Base.metadata.sorted_tables):
# We don't delete data from some tables, because those are
# imported only during introspection which is done one time
if table.name not in self.EXCLUDE_TABLES:
try:
session.open_session()
con = session.connection()
LOGGER.debug("Executing Delete From Table " +
table.name)
con.execute(table.delete())
session.commit()
except Exception, e:
# We cache exception here, in case some table does not exists and
# to allow the others to be deleted
LOGGER.warning(e)
session.rollback()
finally:
session.close_session()
"""
.. moduleauthor:: Bogdan Neacsa <*****@*****.**>
"""
import unittest
import threading
import tvb.config as config
from tvb.basic.profile import TvbProfile
from tvb.core.entities import model
from tvb.core.entities.storage import dao, transactional
from tvb.core.entities.storage.session_maker import add_session, SessionMaker
from tvb.core.entities.storage.exceptions import NestedTransactionUnsupported
from tvb.tests.framework.core.test_factory import TestFactory
from tvb.tests.framework.core.base_testcase import BaseTestCase, transactional_test
SESSIONMAKER = SessionMaker()
class TransactionalTests(BaseTestCase):
"""
This class contains tests for the tvb.core.entities.modelmanager module.
"""
session = SessionMaker()
def setUp(self):
"""
Set-up the environment for testing; clean the database and save events
"""
self.clean_database()
self.old_events = config.EVENTS_FOLDER
