def setUp(self):
TestCase.setUp(self)
TestHelper.setUp(self)
self.threadpool = FakeThreadPool()
self.transaction = self.mocker.mock()
self.transactor = Transactor(self.threadpool, self.transaction)
self.function = self.mocker.mock()
def setUp(self):
TestCase.setUp(self)
TestHelper.setUp(self)
self.threadpool = FakeThreadPool()
self.transaction = self.mocker.mock()
self.transactor = Transactor(self.threadpool, self.transaction)
self.function = self.mocker.mock()
def getService(config, reactor=None, web=True):
if reactor is None:
from twisted.internet import reactor
root = service.MultiService()
sm = ServerManager(reactor, config["servers"])
smTrigId = reactor.addSystemEventTrigger("before", "shutdown",
sm.loseConnections)
tp = reactor.getThreadPool()
root.updater = Updater(Transactor(tp), sm)
updater = UpdaterService(
int(config.get("workers", 10)),
int(config.get("interval", 300)),
)
root.addService(updater)
updater.parent = root
if web:
site = Site(getResource(config.get("rest", {}), root.updater))
reactor.listenTCP(8080, site, interface="127.0.0.1")
def _cleanup(res=None):
sm.loseConnections()
reactor.removeSystemEventTrigger(smTrigId)
return root
def __init__(self):
super(Model, self).__init__()
if not self.database.started:
self.database.start()
self.transactor = Transactor(self.database.pool)
def __init__(self):
super(Model, self).__init__()
self.__mamba_async__ = getattr(self, '__mamba_async__', True)
if not self.database.started:
self.database.start()
self.transactor = Transactor(self.database.pool)
def setUp(self):
self.db = MockStore()
GSM.registerUtility(MockZStorm(self.db))
self.tp = ThreadPool(0, 2)
self.sm = MockServerManager(reactor, SERVERS)
self.updater = Updater(Transactor(self.tp), self.sm)
self.tp.start()
def __init__(self, pool=None, testing=False):
if pool is not None:
self.pool = pool
self.transactor = Transactor(pool)
self.started = False
self.__testing = testing
if not self.zstorm_configured:
provideUtility(global_zstorm, IZStorm)
self._patch_sqlite_uris()
self._set_zstorm_default_uris(getUtility(IZStorm))
SQLite.register()
MySQL.register()
PostgreSQL.register()
def find(klass, *args, **kwargs):
"""Find an object in the underlying database
Some examples:
model.find(Customer.name == u"John")
model.find(name=u"John")
model.find((Customer, City), Customer.city_id == City.id)
.. versionadded:: 0.3.6
"""
obj = klass
if len(args) > 0 and (type(args[0]) == tuple or type(args[0]) == list):
obj = args[0]
return Transactor(klass.database.pool).run(
klass.database.store(klass.mamba_database()).find, obj, *args,
**kwargs)
def __init__(self, pool=None, testing=False):
if pool is not None:
self.pool = pool
self.transactor = Transactor(pool)
self.started = False
self.__testing = testing
if not self.zstorm_configured:
provideUtility(global_zstorm, IZStorm)
zstorm = getUtility(IZStorm)
if self.backend == 'sqlite' and sqlite_version_info >= (3, 6, 19):
uri = '{}?foreign_keys=1'.format(config.Database().uri)
else:
uri = config.Database().uri
zstorm.set_default_uri('mamba', uri)
SQLite.register()
MySQL.register()
PostgreSQL.register()
def all(klass, order_by=None, desc=False, *args, **kwargs):
"""Return back all the rows in the database for this model
:param order_by: order the resultset by the given field/property
:type order_by: model property
:param desc: if True, order the resultset by descending order
:type desc: bool
.. versionadded:: 0.3.6
"""
def inner_transaction():
store = klass.database.store(klass.mamba_database())
data = store.find(klass)
if order_by is not None:
data.order_by(Desc(order_by) if desc else order_by)
return data
return Transactor(klass.database.pool).run(inner_transaction, *args,
**kwargs)
class TransactorTest(TestCase, TestHelper):
def is_supported(self):
return has_transaction and has_zope_component and has_twisted
def setUp(self):
TestCase.setUp(self)
TestHelper.setUp(self)
self.threadpool = FakeThreadPool()
self.transaction = self.mocker.mock()
self.transactor = Transactor(self.threadpool, self.transaction)
self.function = self.mocker.mock()
def test_run(self):
"""
L{Transactor.run} executes a function in a thread, commits
the transaction and returns a deferred firing the function result.
"""
self.mocker.order()
self.expect(self.function(1, arg=2)).result(3)
self.expect(self.transaction.commit())
self.mocker.replay()
deferred = self.transactor.run(self.function, 1, arg=2)
deferred.addCallback(self.assertEqual, 3)
return deferred
def test_run_with_async_false(self):
self.mocker.order()
self.expect(self.function(1, arg=2)).result(3)
self.expect(self.transaction.commit())
self.mocker.replay()
kwargs = {'async': False, 'arg': 2}
result = self.transactor.run(self.function, 1, **kwargs)
self.assertEqual(result, 3)
def test_run_with_function_failure(self):
"""
If the given function raises an error, then L{Transactor.run}
aborts the transaction and re-raises the same error.
"""
self.mocker.order()
self.expect(self.function()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_run_with_disconnection_error(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection.
"""
self.transactor.retries = 0
self.mocker.order()
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter((("store1", store1),
("store2", store2))))
self.expect(store1.execute("SELECT 1"))
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_disconnection_error_in_execute_is_ignored(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection. If another L{DisconnectionError}
happens during C{execute}, then it is ignored.
"""
self.transactor.retries = 0
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter((("store1", store1),
("store2", store2))))
self.expect(store1.execute("SELECT 1")).throw(DisconnectionError())
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_auto_commit_false(self):
"""
If the user pass a key as "auto_commit": Fale in the kwargs for
a mamba @transact decorated method, we don't call transaction.commit
"""
function = self.mocker.mock()
self.mocker.order()
self.expect(function())
self.mocker.replay()
kwargs = {'auto_commit': False}
deferred = self.transactor.run(function, **kwargs)
return deferred
def test_run_with_auto_commit_true(self):
function = self.mocker.mock()
self.mocker.order()
self.expect(function())
self.expect(self.transaction.commit())
self.mocker.replay()
kwargs = {'auto_commit': True}
deferred = self.transactor.run(function, **kwargs)
return deferred
def test_run_with_commit_failure(self):
"""
If the given function succeeds but the transaction fails to commit,
then L{Transactor.run} aborts the transaction and re-raises
the commit exception.
"""
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_wb_default_transaction(self):
"""
By default L{Transact} uses the global transaction manager.
"""
transactor = Transactor(self.threadpool)
self.assertIdentical(transaction, transactor._transaction)
def test_decorate(self):
"""
A L{transact} decorator can be used with methods of an object that
contains a L{Transactor} instance as a C{transactor} instance variable,
ensuring that the decorated function is called via L{Transactor.run}.
"""
self.mocker.order()
self.expect(self.transaction.commit())
self.mocker.replay()
@transact
def function(self):
"""docstring"""
return "result"
# Function metadata is copied to the wrapper.
self.assertEqual("docstring", function.__doc__)
deferred = function(self)
deferred.addCallback(self.assertEqual, "result")
return deferred
def test_run_with_integrity_error_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
def test_run_with_transaction_rollback_error_retries(self):
"""
If the given function raises a L{TransactionRollbackError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
if not has_psycopg:
return
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, TransactionRollbackError)
return deferred
def test_run_with_disconnection_error_retries(self):
"""
If the given function raises a L{DisconnectionError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
zstorm = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_integrity_error_on_commit_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
def test_run_with_on_retry_callback(self):
"""
If a retry callback is passed with the C{on_retry} parameter, then
it's invoked with the number of retries performed so far.
"""
calls = []
def on_retry(context):
calls.append(context)
self.transactor.on_retry = on_retry
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function(1, a=2))
error = IntegrityError()
self.expect(self.transaction.commit()).throw(error)
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function(1, a=2))
self.expect(self.transaction.commit())
self.mocker.replay()
deferred = self.transactor.run(self.function, 1, a=2)
def check(_):
[context] = calls
self.assertEqual(self.function, context.function)
self.assertEqual((1,), context.args)
self.assertEqual({"a": 2}, context.kwargs)
self.assertEqual(1, context.retry)
self.assertEqual(1, context.retry)
self.assertIs(error, context.error)
return deferred.addCallback(check)
def test_run_with_readonly_error(self):
"""
If the given function raises a L{ReadOnlyError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection.
"""
self.transactor.retries = 0
self.mocker.order()
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.expect(self.function()).throw(ReadOnlyError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter((("store1", store1),
("store2", store2))))
self.expect(store1.execute("SELECT 1"))
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ReadOnlyError)
return deferred
def test_run_with_readonly_error_retries(self):
"""
If the given function raises a L{ReadOnlyError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
zstorm = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(ReadOnlyError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(ReadOnlyError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2 ** 2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(ReadOnlyError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ReadOnlyError)
return deferred
class TransactorTest(TestCase, TestHelper):
def is_supported(self):
return has_transaction and has_zope_component and has_twisted
def setUp(self):
TestCase.setUp(self)
TestHelper.setUp(self)
self.threadpool = FakeThreadPool()
self.transaction = self.mocker.mock()
self.transactor = Transactor(self.threadpool, self.transaction)
self.function = self.mocker.mock()
def test_run(self):
"""
L{Transactor.run} executes a function in a thread, commits
the transaction and returns a deferred firing the function result.
"""
self.mocker.order()
self.expect(self.function(1, arg=2)).result(3)
self.expect(self.transaction.commit())
self.mocker.replay()
deferred = self.transactor.run(self.function, 1, arg=2)
deferred.addCallback(self.assertEqual, 3)
return deferred
def test_run_with_function_failure(self):
"""
If the given function raises an error, then L{Transactor.run}
aborts the transaction and re-raises the same error.
"""
self.mocker.order()
self.expect(self.function()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_run_with_disconnection_error(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection.
"""
self.transactor.retries = 0
self.mocker.order()
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(
iter((("store1", store1), ("store2", store2))))
self.expect(store1.execute("SELECT 1"))
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_disconnection_error_in_execute_is_ignored(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection. If another L{DisconnectionError}
happens during C{execute}, then it is ignored.
"""
self.transactor.retries = 0
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(
iter((("store1", store1), ("store2", store2))))
self.expect(store1.execute("SELECT 1")).throw(DisconnectionError())
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_commit_failure(self):
"""
If the given function succeeds but the transaction fails to commit,
then L{Transactor.run} aborts the transaction and re-raises
the commit exception.
"""
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_wb_default_transaction(self):
"""
By default L{Transact} uses the global transaction manager.
"""
transactor = Transactor(self.threadpool)
self.assertIdentical(transaction, transactor._transaction)
def test_decorate(self):
"""
A L{transact} decorator can be used with methods of an object that
contains a L{Transactor} instance as a C{transactor} instance variable,
ensuring that the decorated function is called via L{Transactor.run}.
"""
self.mocker.order()
self.expect(self.transaction.commit())
self.mocker.replay()
@transact
def function(self):
"""docstring"""
return "result"
# Function metadata is copied to the wrapper.
self.assertEqual("docstring", function.__doc__)
deferred = function(self)
deferred.addCallback(self.assertEqual, "result")
return deferred
def test_run_with_integrity_error_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
def test_run_with_transaction_rollback_error_retries(self):
"""
If the given function raises a L{TransactionRollbackError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
if not has_psycopg:
return
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, TransactionRollbackError)
return deferred
def test_run_with_disconnection_error_retries(self):
"""
If the given function raises a L{DisconnectionError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
zstorm = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_integrity_error_on_commit_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**2)).result(2)
self.expect(self.transactor.sleep(2))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
def test_run_with_on_retry_callback(self):
"""
If a retry callback is passed with the C{on_retry} parameter, then
it's invoked with the number of retries performed so far.
"""
calls = []
def on_retry(context):
calls.append(context)
self.transactor.on_retry = on_retry
self.transactor.sleep = self.mocker.mock()
self.transactor.uniform = self.mocker.mock()
self.mocker.order()
self.expect(self.function(1, a=2))
error = IntegrityError()
self.expect(self.transaction.commit()).throw(error)
self.expect(self.transaction.abort())
self.expect(self.transactor.uniform(1, 2**1)).result(1)
self.expect(self.transactor.sleep(1))
self.expect(self.function(1, a=2))
self.expect(self.transaction.commit())
self.mocker.replay()
deferred = self.transactor.run(self.function, 1, a=2)
def check(_):
[context] = calls
self.assertEqual(self.function, context.function)
self.assertEqual((1, ), context.args)
self.assertEqual({"a": 2}, context.kwargs)
self.assertEqual(1, context.retry)
self.assertEqual(1, context.retry)
self.assertIs(error, context.error)
return deferred.addCallback(check)
def test_wb_default_transaction(self):
"""
By default L{Transact} uses the global transaction manager.
"""
transactor = Transactor(self.threadpool)
self.assertIdentical(transaction, transactor._transaction)
class Database(object):
"""
Storm ORM database provider for Mamba.
:param pool: the thrad pool for this database
:type pool: :class:`twisted.python.threadpool.ThreadPool`
"""
monkey_patched = False
pool = ThreadPool(
config.Database().min_threads,
config.Database().max_threads,
'DatabasePool'
)
zstorm_configured = False
transactor = Transactor(pool)
def __init__(self, pool=None, testing=False):
if pool is not None:
self.pool = pool
self.transactor = Transactor(pool)
self.started = False
self.__testing = testing
if not self.zstorm_configured:
provideUtility(global_zstorm, IZStorm)
zstorm = getUtility(IZStorm)
if self.backend == 'sqlite' and sqlite_version_info >= (3, 6, 19):
uri = '{}?foreign_keys=1'.format(config.Database().uri)
else:
uri = config.Database().uri
zstorm.set_default_uri('mamba', uri)
SQLite.register()
MySQL.register()
PostgreSQL.register()
def start(self):
"""Starts the Database (and the threadpool)
"""
if self.started:
return
if self.__testing is True:
self.pool.start()
self.started = True
def stop(self):
"""Stops the Database (and the threadpool)
"""
if not self.started:
return
self.started = False
def adjust_poolsize(self, min_threads=None, max_threads=None):
"""
Adjusts the underlying threadpool size
:param min: minimum number of threads
:type min: int
:param max: maximum number of threads
:type max: int
"""
self.pool.adjustPoolsize(min_threads, max_threads)
def store(self):
"""
Returns a Store per-thread through :class:`storm.zope.zstorm.ZStorm`
"""
if not self.started:
self.start()
zstorm = getUtility(IZStorm)
return zstorm.get('mamba')
def dump(self, model_manager, full=False):
"""
Dumps the full database
:param model_manager: the model manager from mamba application
:type model_manager: :class:`~mamba.application.model.ModelManager`
:param full: should be dumped full?
:type full: bool
"""
references = []
indexes = []
sql = [
'--',
'-- Mamba SQL dump {}'.format(version.short()),
'--',
'-- Database Backend: {}'.format(self.backend),
'-- Host: {}\tDatabase: {}'.format(self.host, self.database)
]
app = config.Application('config/application.json')
try:
sql += [
'-- Application: {}'.format(app.name.decode('utf-8')),
'-- Application Version: {}'.format(app.version),
'-- Application Description: {}'.format(
app.description.encode('utf-8')
)
]
except AttributeError:
pass
sql += [
'-- ---------------------------------------------------------',
'-- Dumped on: {}'.format(datetime.datetime.now().isoformat()),
'--'
]
if self.backend == 'mysql':
sql += [
'-- Disable foreign key checks for table creation',
'--',
'SET FOREIGN_KEY_CHECKS = 0;'
]
if full is False:
sql.append('')
for model in model_manager.get_models().values():
if not model.get('object').on_schema():
continue
if self.backend == 'postgres':
references.append(model.get('object').dump_references())
if self.backend in ('postgres', 'sqlite'):
if model.get('object').dump_indexes():
indexes.append(model.get('object').dump_indexes())
sql += [model.get('object').dump_table() + '\n']
else:
for model in model_manager.get_models().values():
model_object = model.get('object')
if not model_object.on_schema():
continue
sql.append('--')
sql.append('-- Table structure for table {}'.format(
model_object.__storm_table__
))
sql.append('--\n')
sql.append(model_object.dump_table())
sql.append('--')
sql.append('-- Dumping data for table {}'.format(
model_object.__storm_table__
))
sql.append('--\n')
sql.append(model_object.dump_data())
if self.backend == 'postgres':
references.append(model_object.dump_references())
if self.backend in ('postgres', 'sqlite'):
if model.get('object').dump_indexes():
indexes.append(model_object.dump_indexes())
if self.backend == 'mysql':
sql += [
'--',
'-- Enable foreign key checks',
'--',
'SET FOREIGN_KEY_CHECKS = 1;'
]
for reference in references:
sql.append(reference)
for index in indexes:
sql.append(index)
return '\n'.join(sql)
def reset(self, model_manager):
"""
Delete all the data in the database and return it to primitive state
:param model_manager: the model manager from mamba application
:type model_manager: :class:`~mamba.application.model.ModelManager`
"""
cfg = config.Database()
cfg.create_table_behaviours['create_table_if_not_exists'] = False
cfg.create_table_behaviours['drop_table'] = True
sql = [
model.get('object').dump_table()
for model in model_manager.get_models().values()
if model.get('object').on_schema() is True
]
return '\n'.join(sql)
@property
def backend(self):
"""Return the type of backend this databse is using
"""
return URI(config.Database().uri).scheme
@property
def host(self):
"""Return the hostname this database is using
"""
return URI(config.Database().uri).host
@property
def database(self):
"""Return the database name we are using
"""
return URI(config.Database().uri).database
class Database(object):
"""
Storm ORM database provider for Mamba.
:param pool: the thrad pool for this database
:type pool: :class:`twisted.python.threadpool.ThreadPool`
"""
monkey_patched = False
pool = ThreadPool(
config.Database().min_threads,
config.Database().max_threads,
'DatabasePool'
)
zstorm_configured = False
transactor = Transactor(pool)
def __init__(self, pool=None, testing=False):
if pool is not None:
self.pool = pool
self.transactor = Transactor(pool)
self.started = False
self.__testing = testing
if not self.zstorm_configured:
provideUtility(global_zstorm, IZStorm)
self._patch_sqlite_uris()
self._set_zstorm_default_uris(getUtility(IZStorm))
SQLite.register()
MySQL.register()
PostgreSQL.register()
@property
def backend(self):
"""Return the type or types of backends this databse is using
"""
return self._parse_uri('scheme')
@property
def host(self):
"""Return the hostname or hostnames this database is using
"""
return self._parse_uri('host')
@property
def database(self):
"""Return the database name or names we are using
"""
return self._parse_uri('database')
def start(self):
"""Starts the Database (and the threadpool)
"""
if self.started:
return
if self.__testing is True:
self.pool.start()
self.started = True
def stop(self):
"""Stops the Database (and the threadpool)
"""
if not self.started:
return
self.started = False
def adjust_poolsize(self, min_threads=None, max_threads=None):
"""
Adjusts the underlying threadpool size
:param min: minimum number of threads
:type min: int
:param max: maximum number of threads
:type max: int
"""
self.pool.adjustPoolsize(min_threads, max_threads)
def store(self, database='mamba', ensure_connect=False):
"""
Returns a Store per-thread through :class:`storm.zope.zstorm.ZStorm`
"""
if not self.started:
self.start()
if ensure_connect is True:
self._ensure_connect()
zstorm = getUtility(IZStorm)
return zstorm.get(database)
def dump(self, model_manager, scheme=None, full=False):
"""
Dumps the full database
:param model_manager: the model manager from mamba application
:type model_manager: :class:`~mamba.application.model.ModelManager`
:param scheme: dump which scheme? if None just everything
:type scheme: str
:param full: should be dumped full?
:type full: bool
"""
references = []
indexes = []
backend, host, database = self._parse_config_scheme(scheme)
sql = [
'--',
'-- Mamba SQL dump {}'.format(version.short()),
'--',
'-- Database Backend: {}'.format(backend),
'-- Host: {}\tDatabase: {}'.format(host, database)
]
app = config.Application('config/application.json')
try:
sql += [
'-- Application: {}'.format(app.name.decode('utf-8')),
'-- Application Version: {}'.format(app.version),
'-- Application Description: {}'.format(
app.description.encode('utf-8')
)
]
except AttributeError:
pass
sql += [
'-- ---------------------------------------------------------',
'-- Dumped on: {}'.format(datetime.datetime.now().isoformat()),
'--'
]
if self.backend == 'mysql':
sql += [
'-- Disable foreign key checks for table creation',
'--',
'SET FOREIGN_KEY_CHECKS = 0;'
]
if full is False:
self._dump_scheme(sql, references, indexes, model_manager, scheme)
else:
self._dump_data(sql, references, indexes, model_manager, scheme)
if self.backend == 'mysql':
sql += [
'--',
'-- Enable foreign key checks',
'--',
'SET FOREIGN_KEY_CHECKS = 1;'
]
for reference in references:
sql.append(reference)
for index in indexes:
sql.append(index)
return '\n'.join(sql)
def reset(self, model_manager, scheme=None):
"""
Delete all the data in the database and return it to primitive state
:param model_manager: the model manager from mamba application
:type model_manager: :class:`~mamba.application.model.ModelManager`
:param scheme: the specific scheme to reset (if any)
:type scheme: str
"""
sql = []
for model in model_manager.get_models().values():
if model.get('object').on_schema() is not True:
continue
if scheme is not None:
if model.get('object').mamba_database() != scheme:
continue
sql.append(
model.get('object').drop_table(script=True, async=False))
sql.append(model.get('object').dump_table())
del self[key]
except KeyError, k:
raise AttributeError, k
def __repr__(self):
return '<Storage ' + dict.__repr__(self) + '>'
def __getstate__(self):
return dict(self)
def __setstate__(self, value):
for (k, v) in value.items():
self[k] = v
def randomStr(length, num=True):
"""
Returns a random a mixed lowercase, uppercase, alfanumerical (if num True)
string long length
"""
chars = string.ascii_lowercase + string.ascii_uppercase
if num:
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
from oonib import config
database = SQLite(URI(config.main.database_uri))
db_threadpool = ThreadPool(0, config.main.db_threadpool_size)
db_threadpool.start()
transactor = Transactor(db_threadpool)
class TransactorTest(TestCase, TestHelper):
def is_supported(self):
return has_transaction and has_zope_component and has_twisted
def setUp(self):
TestCase.setUp(self)
TestHelper.setUp(self)
self.threadpool = FakeThreadPool()
self.transaction = self.mocker.mock()
self.transactor = Transactor(self.threadpool, self.transaction)
self.function = self.mocker.mock()
def test_run(self):
"""
L{Transactor.run} executes a function in a thread, commits
the transaction and returns a deferred firing the function result.
"""
self.mocker.order()
self.expect(self.function(1, arg=2)).result(3)
self.expect(self.transaction.commit())
self.mocker.replay()
deferred = self.transactor.run(self.function, 1, arg=2)
deferred.addCallback(self.assertEqual, 3)
return deferred
def test_run_with_function_failure(self):
"""
If the given function raises an error, then L{Transactor.run}
aborts the transaction and re-raises the same error.
"""
self.mocker.order()
self.expect(self.function()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_run_with_disconnection_error(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection.
"""
self.transactor.retries = 0
self.mocker.order()
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter((("store1", store1),
("store2", store2))))
self.expect(store1.execute("SELECT 1"))
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_disconnection_error_in_execute_is_ignored(self):
"""
If the given function raises a L{DisconnectionError}, then a C{SELECT
1} will be executed in each registered store such that C{psycopg}
actually detects the disconnection. If another L{DisconnectionError}
happens during C{execute}, then it is ignored.
"""
self.transactor.retries = 0
zstorm = self.mocker.mock()
store1 = self.mocker.mock()
store2 = self.mocker.mock()
gu = self.mocker.replace(getUtility)
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter((("store1", store1),
("store2", store2))))
self.expect(store1.execute("SELECT 1")).throw(DisconnectionError())
self.expect(store2.execute("SELECT 1"))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_commit_failure(self):
"""
If the given function succeeds but the transaction fails to commit,
then L{Transactor.run} aborts the transaction and re-raises
the commit exception.
"""
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(ZeroDivisionError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, ZeroDivisionError)
return deferred
def test_wb_default_transaction(self):
"""
By default L{Transact} uses the global transaction manager.
"""
transactor = Transactor(self.threadpool)
self.assertIdentical(transaction, transactor._transaction)
def test_decorate(self):
"""
A L{transact} decorator can be used with methods of an object that
contains a L{Transactor} instance as a C{transactor} instance variable,
ensuring that the decorated function is called via L{Transactor.run}.
"""
self.mocker.order()
self.expect(self.transaction.commit())
self.mocker.replay()
@transact
def function(self):
"""docstring"""
return "result"
# Function metadata is copied to the wrapper.
self.assertEqual("docstring", function.__doc__)
deferred = function(self)
deferred.addCallback(self.assertEqual, "result")
return deferred
def test_run_with_integrity_error_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
sleep = self.mocker.replace(time.sleep)
uniform = self.mocker.replace(random).uniform
self.mocker.order()
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 1)).result(1)
self.expect(sleep(1))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 2)).result(2)
self.expect(sleep(2))
self.expect(self.function()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
def test_run_with_transaction_rollback_error_retries(self):
"""
If the given function raises a L{TransactionRollbackError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
if not has_psycopg:
return
sleep = self.mocker.replace(time.sleep)
uniform = self.mocker.replace(random).uniform
self.mocker.order()
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 1)).result(1)
self.expect(sleep(1))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 2)).result(2)
self.expect(sleep(2))
self.expect(self.function()).throw(TransactionRollbackError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, TransactionRollbackError)
return deferred
def test_run_with_disconnection_error_retries(self):
"""
If the given function raises a L{DisconnectionError}, then the
function will be retried another two times before letting the exception
bubble up.
"""
zstorm = self.mocker.mock()
gu = self.mocker.replace(getUtility)
sleep = self.mocker.replace(time.sleep)
uniform = self.mocker.replace(random).uniform
self.mocker.order()
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 1)).result(1)
self.expect(sleep(1))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 2)).result(2)
self.expect(sleep(2))
self.expect(self.function()).throw(DisconnectionError())
self.expect(gu(IZStorm)).result(zstorm)
self.expect(zstorm.iterstores()).result(iter(()))
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, DisconnectionError)
return deferred
def test_run_with_integrity_error_on_commit_retries(self):
"""
If the given function raises a L{IntegrityError}, then the function
will be retried another two times before letting the exception bubble
up.
"""
sleep = self.mocker.replace(time.sleep)
uniform = self.mocker.replace(random).uniform
self.mocker.order()
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 1)).result(1)
self.expect(sleep(1))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.expect(uniform(1, 2 ** 2)).result(2)
self.expect(sleep(2))
self.expect(self.function())
self.expect(self.transaction.commit()).throw(IntegrityError())
self.expect(self.transaction.abort())
self.mocker.replay()
deferred = self.transactor.run(self.function)
self.assertFailure(deferred, IntegrityError)
return deferred
