async def test_should_not_resolve_conflict_error_with_resolve(
db, dummy_guillotina):
aps = await get_aps(db, "resolve")
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob1 = create_content()
txn.register(ob1)
await tm.commit(txn=txn)
# 1 started before 2
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1.__uuid__)
ob2 = await txn2.get(ob1.__uuid__)
txn1.register(ob1)
txn2.register(ob2)
# commit 2 before 1
await tm.commit(txn=txn2)
ob1.__serial__ = txn2._tid # prevent tid error since we're testing trns conflict error
with pytest.raises(ConflictError):
await tm.commit(txn=txn1)
await aps.remove()
await cleanup(aps)
async def test_should_raise_conflict_error_when_editing_diff_data_with_resolve_strat(
db, dummy_guillotina):
aps = await get_aps(db, "resolve")
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob = create_content()
ob.title = "foobar"
ob.description = "foobar"
txn.register(ob)
await tm.commit(txn=txn)
# 1 started before 2
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob.__uuid__)
ob2 = await txn2.get(ob.__uuid__)
ob1.title = "foobar1"
ob2.description = "foobar2"
txn1.register(ob1)
txn2.register(ob2)
# commit 2 before 1
await tm.commit(txn=txn2)
with pytest.raises(ConflictError):
await tm.commit(txn=txn1)
await aps.remove()
await cleanup(aps)
async def test_should_resolve_conflict_error(db, dummy_guillotina):
aps = await get_aps(db, "resolve")
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob1 = create_content()
ob2 = create_content()
txn.register(ob1)
txn.register(ob2)
await tm.commit(txn=txn)
# 1 started before 2
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1.__uuid__)
ob2 = await txn2.get(ob2.__uuid__)
txn1.register(ob1)
txn2.register(ob2)
# commit 2 before 1
await tm.commit(txn=txn2)
# should not raise conflict error
await tm.commit(txn=txn1)
await aps.remove()
await cleanup(aps)
async def test_restart_connection_pg(db, dummy_guillotina):
aps = await get_aps(db)
with TransactionManager(aps) as tm:
# Test it works
await tm._storage.get_next_tid(Mock())
# Simulate connection was initialized a long time ago
tm._storage._connection_initialized_on = 0
async def fetchval_conn_closed():
raise asyncpg.exceptions.InterfaceError(
"cannot call PreparedStatement.fetchval(): the underlying connection is closed"
)
# Simulate underlying connection is closed
tm._storage._connection_manager._stmt_next_tid = Mock(
**{"fetchval": fetchval_conn_closed})
with pytest.raises(ConflictError):
await tm._storage.get_next_tid(Mock())
# Test works again
await tm._storage.get_next_tid(Mock())
await aps.remove()
await cleanup(aps)
async def test_delete_resource_deletes_blob(db, dummy_guillotina):
aps = await get_aps(db)
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob = create_content()
txn.register(ob)
await txn.write_blob_chunk("X" * 32, ob.__uuid__, 0, b"foobar")
await tm.commit(txn=txn)
txn = await tm.begin()
ob = await txn.get(ob.__uuid__)
txn.delete(ob)
await tm.commit(txn=txn)
await asyncio.sleep(0.1) # make sure cleanup runs
txn = await tm.begin()
assert await txn.read_blob_chunk("X" * 32, 0) is None
with pytest.raises(KeyError):
await txn.get(ob.__uuid__)
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_handles_asyncpg_trying_txn_with_manual_txn(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
# simulate transaction already started(should not happen)
for conn in tm._storage.pool._queue._queue:
if conn._con is None:
await conn.connect()
await conn._con.execute('BEGIN;')
txn = await tm.begin()
# then, try doing stuff...
ob = create_content()
txn.register(ob)
assert len(txn.modified) == 1
await tm.commit(txn=txn)
txn = await tm.begin()
ob2 = await txn.get(ob._p_oid)
assert ob2._p_oid == ob._p_oid
await tm.commit(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_restart_connection(db, dummy_request):
"""Low level test checks that root is not there"""
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
txn = await tm.begin()
ob = create_content()
txn.register(ob)
assert len(txn.modified) == 1
await tm.commit(txn=txn)
with pytest.raises(ConflictError):
await aps.restart_connection()
txn = await tm.begin()
ob2 = await txn.get(ob._p_oid)
assert ob2._p_oid == ob._p_oid
await tm.commit(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_iterate_keys(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
# base aps uses 1 connection from the pool for starting transactions
aps = await get_aps(db)
tm = TransactionManager(aps)
txn = await tm.begin()
parent = create_content()
txn.register(parent)
original_keys = []
for _ in range(50):
item = create_content()
original_keys.append(item.id)
item.__parent__ = parent
txn.register(item)
await tm.commit(txn=txn)
txn = await tm.begin()
keys = []
async for key in txn.iterate_keys(parent._p_oid, 2):
keys.append(key)
assert len(keys) == 50
assert len(set(keys) - set(original_keys)) == 0
await tm.abort(txn=txn)
def get_transaction_manager(self):
"""
New transaction manager for every request
"""
if self._tm is None:
self._tm = TransactionManager(self._storage)
return self._tm
async def test_handles_asyncpg_trying_savepoints(postgres, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(postgres)
tm = TransactionManager(aps)
# simulate transaction already started(should not happen)
for conn in tm._storage._pool._queue._queue:
if conn._con is None:
await conn.connect()
conn._con._top_xact = asyncpg.transaction.Transaction(conn, 'read_committed', False, False)
txn = await tm.begin()
# then, try doing stuff...
ob = create_content()
txn.register(ob)
assert len(txn.modified) == 1
await tm.commit(txn=txn)
txn = await tm.begin()
ob2 = await txn.get(ob._p_oid)
assert ob2._p_oid == ob._p_oid
await tm.commit(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_none_strat_allows_trans_commits(db, dummy_guillotina):
aps = await get_aps(db, "none")
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob1 = create_content()
txn.register(ob1)
await tm.commit(txn=txn)
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1.__uuid__)
ob2 = await txn2.get(ob1.__uuid__)
ob1.title = "foobar1"
ob2.title = "foobar2"
txn1.register(ob1)
txn2.register(ob2)
await tm.commit(txn=txn2)
await tm.commit(txn=txn1)
txn = await tm.begin()
ob1 = await txn.get(ob1.__uuid__)
assert ob1.title == "foobar1"
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_iterate_keys(db, dummy_guillotina):
# base aps uses 1 connection from the pool for starting transactions
aps = await get_aps(db)
with TransactionManager(aps) as tm, await tm.begin() as txn:
parent = create_content()
txn.register(parent)
original_keys = []
for _ in range(50):
item = create_content()
original_keys.append(item.id)
item.__parent__ = parent
txn.register(item)
await tm.commit(txn=txn)
txn = await tm.begin()
keys = []
async for key in txn.iterate_keys(parent.__uuid__, 2):
keys.append(key)
assert len(keys) == 50
assert len(set(keys) - set(original_keys)) == 0
await tm.abort(txn=txn)
async def test_restart_connection_pg(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
# Test it works
await tm._storage.get_next_tid(Mock())
# Simulate connection was initialized a long time ago
tm._storage._connection_initialized_on = 0
async def fetchval_conn_closed():
raise asyncpg.exceptions.InterfaceError(
'cannot call PreparedStatement.fetchval(): the underlying connection is closed'
)
# Simulate underlying connection is closed
tm._storage._stmt_next_tid = Mock(**{'fetchval': fetchval_conn_closed})
with pytest.raises(ConflictError):
await tm._storage.get_next_tid(Mock())
# Test works again
await tm._storage.get_next_tid(Mock())
await aps.remove()
await cleanup(aps)
async def test_get_resources_of_type(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
# create object first, commit it...
txn = await tm.begin()
ob = create_content()
txn.register(ob)
await tm.commit(txn=txn)
txn = await tm.begin()
count = 0
async for item in txn._get_resources_of_type('Item'):
assert item['type'] == 'Item'
count += 1
assert count == 1
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_using_gather_with_queries_after_prepare(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
# create object first, commit it...
txn = await tm.begin()
ob1 = create_content()
txn.register(ob1)
await tm.commit(txn=txn)
txn = await tm.begin()
async def get_ob():
await txn.get(ob1._p_oid)
# one initial call should load prepared statement
await txn.get(ob1._p_oid)
# before we introduction locking on the connection, this would error
await asyncio.gather(get_ob(), get_ob(), get_ob(), get_ob(), get_ob())
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_should_not_resolve_conflict_error_with_simple_strat(
db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db, 'simple')
tm = TransactionManager(aps)
# create object first, commit it...
txn = await tm.begin()
ob1 = create_content()
ob2 = create_content()
txn.register(ob1)
txn.register(ob2)
await tm.commit(txn=txn)
# 1 started before 2
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1._p_oid)
ob2 = await txn2.get(ob2._p_oid)
txn1.register(ob1)
txn2.register(ob2)
# commit 2 before 1
await tm.commit(txn=txn2)
with pytest.raises(ConflictError):
await tm.commit(txn=txn1)
await aps.remove()
await cleanup(aps)
async def test_none_strat_allows_trans_commits(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db, 'none')
tm = TransactionManager(aps)
# create object first, commit it...
txn = await tm.begin()
ob1 = create_content()
txn.register(ob1)
await tm.commit(txn=txn)
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1._p_oid)
ob2 = await txn2.get(ob1._p_oid)
ob1.title = 'foobar1'
ob2.title = 'foobar2'
txn1.register(ob1)
txn2.register(ob2)
await tm.commit(txn=txn2)
await tm.commit(txn=txn1)
txn = await tm.begin()
ob1 = await txn.get(ob1._p_oid)
assert ob1.title == 'foobar1'
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_delete_resource_deletes_blob(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
txn = await tm.begin()
ob = create_content()
txn.register(ob)
await txn.write_blob_chunk('X' * 32, ob._p_oid, 0, b'foobar')
await tm.commit(txn=txn)
txn = await tm.begin()
ob = await txn.get(ob._p_oid)
txn.delete(ob)
await tm.commit(txn=txn)
await asyncio.sleep(0.1) # make sure cleanup runs
txn = await tm.begin()
assert await txn.read_blob_chunk('X' * 32, 0) is None
with pytest.raises(KeyError):
await txn.get(ob._p_oid)
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_create_blob(db, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(db)
tm = TransactionManager(aps)
txn = await tm.begin()
ob = create_content()
txn.register(ob)
await txn.write_blob_chunk('X' * 32, ob._p_oid, 0, b'foobar')
await tm.commit(txn=txn)
txn = await tm.begin()
blob_record = await txn.read_blob_chunk('X' * 32, 0)
assert blob_record['data'] == b'foobar'
# also get data from ob that started as a stub...
ob2 = await txn.get(ob._p_oid)
assert ob2.type_name == 'Item'
assert 'foobar' in ob2.id
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_should_resolve_conflict_error(postgres, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
aps = await get_aps(postgres, 'resolve')
tm = TransactionManager(aps)
# create object first, commit it...
txn = await tm.begin()
ob1 = create_content()
ob2 = create_content()
txn.register(ob1)
txn.register(ob2)
await tm.commit(txn=txn)
# 1 started before 2
txn1 = await tm.begin()
txn2 = await tm.begin()
ob1 = await txn1.get(ob1._p_oid)
ob2 = await txn2.get(ob2._p_oid)
txn1.register(ob1)
txn2.register(ob2)
# commit 2 before 1
await tm.commit(txn=txn2)
# should not raise conflict error
await tm.commit(txn=txn1)
await aps.remove()
await cleanup(aps)
async def test_handles_asyncpg_trying_txn_with_manual_txn(
db, dummy_guillotina):
aps = await get_aps(db)
tm = TransactionManager(aps)
# simulate transaction already started(should not happen)
for conn in tm._storage.pool._queue._queue:
if conn._con is None:
await conn.connect()
await conn._con.execute("BEGIN;")
with await tm.begin() as txn, tm:
# then, try doing stuff...
ob = create_content()
txn.register(ob)
assert len(txn.modified) == 1
await tm.commit(txn=txn)
txn = await tm.begin()
ob2 = await txn.get(ob.__uuid__)
assert ob2.__uuid__ == ob.__uuid__
await tm.commit(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_exhausting_pool_size(db, dummy_guillotina):
# base aps uses 1 connection from the pool for starting transactions
aps = await get_aps(db, pool_size=2)
with TransactionManager(aps) as tm, await tm.begin() as txn:
await txn.get_connection()
with pytest.raises(concurrent.futures._base.TimeoutError):
# should throw an error because we've run out of connections in pool
txn2 = await tm.begin()
await asyncio.wait_for(txn2.get_connection(), 0.5)
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_get_total_resources_of_type(db, dummy_guillotina):
aps = await get_aps(db)
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob = create_content()
txn.register(ob)
await tm.commit(txn=txn)
txn = await tm.begin()
assert 1 == await txn.get_total_resources_of_type('Item')
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_exhausting_pool_size(postgres, dummy_request):
request = dummy_request # noqa so magically get_current_request can find
# base aps uses 1 connection from the pool for starting transactions
aps = await get_aps(postgres, pool_size=2)
tm = TransactionManager(aps)
txn = await tm.begin()
with pytest.raises(concurrent.futures._base.TimeoutError):
# should throw an error because we've run out of connections in pool
await tm.begin()
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def initialize(self):
# Make sure we have a root:
request = make_mocked_request('POST', '/')
request._db_write_enabled = True
request._tm = TransactionManager(self.storage)
t = await request._tm.begin(request=request)
self.request = request
try:
assert request._tm.get() == t
await t.get(0)
except KeyError:
root = Root()
t.register(root, new_oid=ROOT_ID)
await request._tm.commit()
async def test_count_total_objects(db, dummy_guillotina):
aps = await get_aps(db)
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob = create_content()
txn.register(ob)
await tm.commit(txn=txn)
txn = await tm.begin()
assert await txn.get_total_number_of_objects() == 2
assert await txn.get_total_number_of_resources() == 1
await tm.abort(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_handle_serialization_error(cockroach_storage):
async with cockroach_storage as storage:
tm = TransactionManager(storage)
txn = await tm.begin()
folder1 = create_content()
txn.register(folder1)
await tm.commit(txn=txn)
txn = await tm.begin()
with mock.patch('asyncpg.prepared_stmt.PreparedStatement._PreparedStatement__bind_execute') as exe_mock: # noqa
exe_mock.side_effect = asyncpg.exceptions.SerializationError(
'restart transaction: HandledRetryableTxnError: '
'ReadWithinUncertaintyIntervalError: read at time '
'1511374585.730535846,0 encountered')
with pytest.raises(ConflictError):
await txn.get(folder1._p_oid)
await tm.abort(txn=txn)
async def _test_read_something(postgres, guillotina_main):
"""Low level test checks that root is there"""
dsn = "postgres://postgres:@localhost:5432/guillotina"
partition_object = "guillotina.db.interfaces.IPartition"
aps = APgStorage(dsn=dsn, partition=partition_object, name='db')
await aps.initialize()
conn = await aps.open()
tm = TransactionManager(aps)
txn = Transaction(tm)
await txn.tpc_begin(conn)
lasttid = await aps.last_transaction(txn)
await aps.load(txn, ROOT_ID)
await aps.abort(txn)
await aps.remove()
await aps.close(txn._db_conn)
await cleanup(aps)
assert lasttid == 1
async def test_mismatched_tid_causes_conflict_error(db, dummy_guillotina):
# base aps uses 1 connection from the pool for starting transactions
aps = await get_aps(db)
with TransactionManager(aps) as tm, await tm.begin() as txn:
ob1 = create_content()
txn.register(ob1)
await tm.commit(txn=txn)
txn = await tm.begin()
ob1 = await txn.get(ob1.__uuid__)
# modify p_serial, try committing, should raise conflict error
ob1.__serial__ = 3242432
txn.register(ob1)
with pytest.raises(ConflictError):
await tm.commit(txn=txn)
await aps.remove()
await cleanup(aps)
async def test_record_transaction_cache_empty(self, dummy_guillotina,
metrics_registry):
storage = AsyncMock()
mng = TransactionManager(storage)
cache = AsyncMock()
cache.get.return_value = transaction._EMPTY
strategy = AsyncMock()
txn = Transaction(mng, cache=cache, strategy=strategy)
ob = create_content(Container)
with pytest.raises(KeyError):
await txn.get_annotation(ob, "foobar")
assert (metrics_registry.get_sample_value(
"guillotina_cache_ops_total", {
"type": "_get_annotation",
"result": "hit_empty"
}) == 1.0)
