async def test_should_not_resolve_conflict_error_with_resolve( db, dummy_guillotina): aps = await get_aps(db, "resolve") with TransactionManager(aps) as tm, await tm.begin() as txn: ob1 = create_content() txn.register(ob1) await tm.commit(txn=txn) # 1 started before 2 txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1.__uuid__) ob2 = await txn2.get(ob1.__uuid__) txn1.register(ob1) txn2.register(ob2) # commit 2 before 1 await tm.commit(txn=txn2) ob1.__serial__ = txn2._tid # prevent tid error since we're testing trns conflict error with pytest.raises(ConflictError): await tm.commit(txn=txn1) await aps.remove() await cleanup(aps)
async def test_should_raise_conflict_error_when_editing_diff_data_with_resolve_strat( db, dummy_guillotina): aps = await get_aps(db, "resolve") with TransactionManager(aps) as tm, await tm.begin() as txn: ob = create_content() ob.title = "foobar" ob.description = "foobar" txn.register(ob) await tm.commit(txn=txn) # 1 started before 2 txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob.__uuid__) ob2 = await txn2.get(ob.__uuid__) ob1.title = "foobar1" ob2.description = "foobar2" txn1.register(ob1) txn2.register(ob2) # commit 2 before 1 await tm.commit(txn=txn2) with pytest.raises(ConflictError): await tm.commit(txn=txn1) await aps.remove() await cleanup(aps)
async def test_should_resolve_conflict_error(db, dummy_guillotina): aps = await get_aps(db, "resolve") with TransactionManager(aps) as tm, await tm.begin() as txn: ob1 = create_content() ob2 = create_content() txn.register(ob1) txn.register(ob2) await tm.commit(txn=txn) # 1 started before 2 txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1.__uuid__) ob2 = await txn2.get(ob2.__uuid__) txn1.register(ob1) txn2.register(ob2) # commit 2 before 1 await tm.commit(txn=txn2) # should not raise conflict error await tm.commit(txn=txn1) await aps.remove() await cleanup(aps)
async def test_restart_connection_pg(db, dummy_guillotina): aps = await get_aps(db) with TransactionManager(aps) as tm: # Test it works await tm._storage.get_next_tid(Mock()) # Simulate connection was initialized a long time ago tm._storage._connection_initialized_on = 0 async def fetchval_conn_closed(): raise asyncpg.exceptions.InterfaceError( "cannot call PreparedStatement.fetchval(): the underlying connection is closed" ) # Simulate underlying connection is closed tm._storage._connection_manager._stmt_next_tid = Mock( **{"fetchval": fetchval_conn_closed}) with pytest.raises(ConflictError): await tm._storage.get_next_tid(Mock()) # Test works again await tm._storage.get_next_tid(Mock()) await aps.remove() await cleanup(aps)
async def test_delete_resource_deletes_blob(db, dummy_guillotina): aps = await get_aps(db) with TransactionManager(aps) as tm, await tm.begin() as txn: ob = create_content() txn.register(ob) await txn.write_blob_chunk("X" * 32, ob.__uuid__, 0, b"foobar") await tm.commit(txn=txn) txn = await tm.begin() ob = await txn.get(ob.__uuid__) txn.delete(ob) await tm.commit(txn=txn) await asyncio.sleep(0.1) # make sure cleanup runs txn = await tm.begin() assert await txn.read_blob_chunk("X" * 32, 0) is None with pytest.raises(KeyError): await txn.get(ob.__uuid__) await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_handles_asyncpg_trying_txn_with_manual_txn(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) # simulate transaction already started(should not happen) for conn in tm._storage.pool._queue._queue: if conn._con is None: await conn.connect() await conn._con.execute('BEGIN;') txn = await tm.begin() # then, try doing stuff... ob = create_content() txn.register(ob) assert len(txn.modified) == 1 await tm.commit(txn=txn) txn = await tm.begin() ob2 = await txn.get(ob._p_oid) assert ob2._p_oid == ob._p_oid await tm.commit(txn=txn) await aps.remove() await cleanup(aps)
async def test_restart_connection(db, dummy_request): """Low level test checks that root is not there""" request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) txn = await tm.begin() ob = create_content() txn.register(ob) assert len(txn.modified) == 1 await tm.commit(txn=txn) with pytest.raises(ConflictError): await aps.restart_connection() txn = await tm.begin() ob2 = await txn.get(ob._p_oid) assert ob2._p_oid == ob._p_oid await tm.commit(txn=txn) await aps.remove() await cleanup(aps)
async def test_iterate_keys(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find # base aps uses 1 connection from the pool for starting transactions aps = await get_aps(db) tm = TransactionManager(aps) txn = await tm.begin() parent = create_content() txn.register(parent) original_keys = [] for _ in range(50): item = create_content() original_keys.append(item.id) item.__parent__ = parent txn.register(item) await tm.commit(txn=txn) txn = await tm.begin() keys = [] async for key in txn.iterate_keys(parent._p_oid, 2): keys.append(key) assert len(keys) == 50 assert len(set(keys) - set(original_keys)) == 0 await tm.abort(txn=txn)
def get_transaction_manager(self): """ New transaction manager for every request """ if self._tm is None: self._tm = TransactionManager(self._storage) return self._tm
async def test_handles_asyncpg_trying_savepoints(postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(postgres) tm = TransactionManager(aps) # simulate transaction already started(should not happen) for conn in tm._storage._pool._queue._queue: if conn._con is None: await conn.connect() conn._con._top_xact = asyncpg.transaction.Transaction(conn, 'read_committed', False, False) txn = await tm.begin() # then, try doing stuff... ob = create_content() txn.register(ob) assert len(txn.modified) == 1 await tm.commit(txn=txn) txn = await tm.begin() ob2 = await txn.get(ob._p_oid) assert ob2._p_oid == ob._p_oid await tm.commit(txn=txn) await aps.remove() await cleanup(aps)
async def test_none_strat_allows_trans_commits(db, dummy_guillotina): aps = await get_aps(db, "none") with TransactionManager(aps) as tm, await tm.begin() as txn: ob1 = create_content() txn.register(ob1) await tm.commit(txn=txn) txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1.__uuid__) ob2 = await txn2.get(ob1.__uuid__) ob1.title = "foobar1" ob2.title = "foobar2" txn1.register(ob1) txn2.register(ob2) await tm.commit(txn=txn2) await tm.commit(txn=txn1) txn = await tm.begin() ob1 = await txn.get(ob1.__uuid__) assert ob1.title == "foobar1" await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_iterate_keys(db, dummy_guillotina): # base aps uses 1 connection from the pool for starting transactions aps = await get_aps(db) with TransactionManager(aps) as tm, await tm.begin() as txn: parent = create_content() txn.register(parent) original_keys = [] for _ in range(50): item = create_content() original_keys.append(item.id) item.__parent__ = parent txn.register(item) await tm.commit(txn=txn) txn = await tm.begin() keys = [] async for key in txn.iterate_keys(parent.__uuid__, 2): keys.append(key) assert len(keys) == 50 assert len(set(keys) - set(original_keys)) == 0 await tm.abort(txn=txn)
async def test_restart_connection_pg(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) # Test it works await tm._storage.get_next_tid(Mock()) # Simulate connection was initialized a long time ago tm._storage._connection_initialized_on = 0 async def fetchval_conn_closed(): raise asyncpg.exceptions.InterfaceError( 'cannot call PreparedStatement.fetchval(): the underlying connection is closed' ) # Simulate underlying connection is closed tm._storage._stmt_next_tid = Mock(**{'fetchval': fetchval_conn_closed}) with pytest.raises(ConflictError): await tm._storage.get_next_tid(Mock()) # Test works again await tm._storage.get_next_tid(Mock()) await aps.remove() await cleanup(aps)
async def test_get_resources_of_type(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) # create object first, commit it... txn = await tm.begin() ob = create_content() txn.register(ob) await tm.commit(txn=txn) txn = await tm.begin() count = 0 async for item in txn._get_resources_of_type('Item'): assert item['type'] == 'Item' count += 1 assert count == 1 await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_using_gather_with_queries_after_prepare(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) # create object first, commit it... txn = await tm.begin() ob1 = create_content() txn.register(ob1) await tm.commit(txn=txn) txn = await tm.begin() async def get_ob(): await txn.get(ob1._p_oid) # one initial call should load prepared statement await txn.get(ob1._p_oid) # before we introduction locking on the connection, this would error await asyncio.gather(get_ob(), get_ob(), get_ob(), get_ob(), get_ob()) await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_should_not_resolve_conflict_error_with_simple_strat( db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db, 'simple') tm = TransactionManager(aps) # create object first, commit it... txn = await tm.begin() ob1 = create_content() ob2 = create_content() txn.register(ob1) txn.register(ob2) await tm.commit(txn=txn) # 1 started before 2 txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1._p_oid) ob2 = await txn2.get(ob2._p_oid) txn1.register(ob1) txn2.register(ob2) # commit 2 before 1 await tm.commit(txn=txn2) with pytest.raises(ConflictError): await tm.commit(txn=txn1) await aps.remove() await cleanup(aps)
async def test_none_strat_allows_trans_commits(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db, 'none') tm = TransactionManager(aps) # create object first, commit it... txn = await tm.begin() ob1 = create_content() txn.register(ob1) await tm.commit(txn=txn) txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1._p_oid) ob2 = await txn2.get(ob1._p_oid) ob1.title = 'foobar1' ob2.title = 'foobar2' txn1.register(ob1) txn2.register(ob2) await tm.commit(txn=txn2) await tm.commit(txn=txn1) txn = await tm.begin() ob1 = await txn.get(ob1._p_oid) assert ob1.title == 'foobar1' await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_delete_resource_deletes_blob(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) txn = await tm.begin() ob = create_content() txn.register(ob) await txn.write_blob_chunk('X' * 32, ob._p_oid, 0, b'foobar') await tm.commit(txn=txn) txn = await tm.begin() ob = await txn.get(ob._p_oid) txn.delete(ob) await tm.commit(txn=txn) await asyncio.sleep(0.1) # make sure cleanup runs txn = await tm.begin() assert await txn.read_blob_chunk('X' * 32, 0) is None with pytest.raises(KeyError): await txn.get(ob._p_oid) await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_create_blob(db, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(db) tm = TransactionManager(aps) txn = await tm.begin() ob = create_content() txn.register(ob) await txn.write_blob_chunk('X' * 32, ob._p_oid, 0, b'foobar') await tm.commit(txn=txn) txn = await tm.begin() blob_record = await txn.read_blob_chunk('X' * 32, 0) assert blob_record['data'] == b'foobar' # also get data from ob that started as a stub... ob2 = await txn.get(ob._p_oid) assert ob2.type_name == 'Item' assert 'foobar' in ob2.id await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_should_resolve_conflict_error(postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find aps = await get_aps(postgres, 'resolve') tm = TransactionManager(aps) # create object first, commit it... txn = await tm.begin() ob1 = create_content() ob2 = create_content() txn.register(ob1) txn.register(ob2) await tm.commit(txn=txn) # 1 started before 2 txn1 = await tm.begin() txn2 = await tm.begin() ob1 = await txn1.get(ob1._p_oid) ob2 = await txn2.get(ob2._p_oid) txn1.register(ob1) txn2.register(ob2) # commit 2 before 1 await tm.commit(txn=txn2) # should not raise conflict error await tm.commit(txn=txn1) await aps.remove() await cleanup(aps)
async def test_handles_asyncpg_trying_txn_with_manual_txn( db, dummy_guillotina): aps = await get_aps(db) tm = TransactionManager(aps) # simulate transaction already started(should not happen) for conn in tm._storage.pool._queue._queue: if conn._con is None: await conn.connect() await conn._con.execute("BEGIN;") with await tm.begin() as txn, tm: # then, try doing stuff... ob = create_content() txn.register(ob) assert len(txn.modified) == 1 await tm.commit(txn=txn) txn = await tm.begin() ob2 = await txn.get(ob.__uuid__) assert ob2.__uuid__ == ob.__uuid__ await tm.commit(txn=txn) await aps.remove() await cleanup(aps)
async def test_exhausting_pool_size(db, dummy_guillotina): # base aps uses 1 connection from the pool for starting transactions aps = await get_aps(db, pool_size=2) with TransactionManager(aps) as tm, await tm.begin() as txn: await txn.get_connection() with pytest.raises(concurrent.futures._base.TimeoutError): # should throw an error because we've run out of connections in pool txn2 = await tm.begin() await asyncio.wait_for(txn2.get_connection(), 0.5) await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_get_total_resources_of_type(db, dummy_guillotina): aps = await get_aps(db) with TransactionManager(aps) as tm, await tm.begin() as txn: ob = create_content() txn.register(ob) await tm.commit(txn=txn) txn = await tm.begin() assert 1 == await txn.get_total_resources_of_type('Item') await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_exhausting_pool_size(postgres, dummy_request): request = dummy_request # noqa so magically get_current_request can find # base aps uses 1 connection from the pool for starting transactions aps = await get_aps(postgres, pool_size=2) tm = TransactionManager(aps) txn = await tm.begin() with pytest.raises(concurrent.futures._base.TimeoutError): # should throw an error because we've run out of connections in pool await tm.begin() await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def initialize(self): # Make sure we have a root: request = make_mocked_request('POST', '/') request._db_write_enabled = True request._tm = TransactionManager(self.storage) t = await request._tm.begin(request=request) self.request = request try: assert request._tm.get() == t await t.get(0) except KeyError: root = Root() t.register(root, new_oid=ROOT_ID) await request._tm.commit()
async def test_count_total_objects(db, dummy_guillotina): aps = await get_aps(db) with TransactionManager(aps) as tm, await tm.begin() as txn: ob = create_content() txn.register(ob) await tm.commit(txn=txn) txn = await tm.begin() assert await txn.get_total_number_of_objects() == 2 assert await txn.get_total_number_of_resources() == 1 await tm.abort(txn=txn) await aps.remove() await cleanup(aps)
async def test_handle_serialization_error(cockroach_storage): async with cockroach_storage as storage: tm = TransactionManager(storage) txn = await tm.begin() folder1 = create_content() txn.register(folder1) await tm.commit(txn=txn) txn = await tm.begin() with mock.patch('asyncpg.prepared_stmt.PreparedStatement._PreparedStatement__bind_execute') as exe_mock: # noqa exe_mock.side_effect = asyncpg.exceptions.SerializationError( 'restart transaction: HandledRetryableTxnError: ' 'ReadWithinUncertaintyIntervalError: read at time ' '1511374585.730535846,0 encountered') with pytest.raises(ConflictError): await txn.get(folder1._p_oid) await tm.abort(txn=txn)
async def _test_read_something(postgres, guillotina_main): """Low level test checks that root is there""" dsn = "postgres://postgres:@localhost:5432/guillotina" partition_object = "guillotina.db.interfaces.IPartition" aps = APgStorage(dsn=dsn, partition=partition_object, name='db') await aps.initialize() conn = await aps.open() tm = TransactionManager(aps) txn = Transaction(tm) await txn.tpc_begin(conn) lasttid = await aps.last_transaction(txn) await aps.load(txn, ROOT_ID) await aps.abort(txn) await aps.remove() await aps.close(txn._db_conn) await cleanup(aps) assert lasttid == 1
async def test_mismatched_tid_causes_conflict_error(db, dummy_guillotina): # base aps uses 1 connection from the pool for starting transactions aps = await get_aps(db) with TransactionManager(aps) as tm, await tm.begin() as txn: ob1 = create_content() txn.register(ob1) await tm.commit(txn=txn) txn = await tm.begin() ob1 = await txn.get(ob1.__uuid__) # modify p_serial, try committing, should raise conflict error ob1.__serial__ = 3242432 txn.register(ob1) with pytest.raises(ConflictError): await tm.commit(txn=txn) await aps.remove() await cleanup(aps)
async def test_record_transaction_cache_empty(self, dummy_guillotina, metrics_registry): storage = AsyncMock() mng = TransactionManager(storage) cache = AsyncMock() cache.get.return_value = transaction._EMPTY strategy = AsyncMock() txn = Transaction(mng, cache=cache, strategy=strategy) ob = create_content(Container) with pytest.raises(KeyError): await txn.get_annotation(ob, "foobar") assert (metrics_registry.get_sample_value( "guillotina_cache_ops_total", { "type": "_get_annotation", "result": "hit_empty" }) == 1.0)