async def test_skeleton_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_1000):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_1000.db)
peer_pair = ETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer, server_peer):
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer],
event_bus=event_bus)
client = FastChainSyncer(LatestTestChain(chaindb_fresh.db),
chaindb_fresh, client_peer_pool)
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus, AsyncChainDB(chaindb_1000.db)):
client_peer.logger.info("%s is serving 1000 blocks", client_peer)
server_peer.logger.info("%s is syncing up 1000 blocks",
server_peer)
await asyncio.wait_for(client.run(), timeout=20)
head = chaindb_fresh.get_canonical_head()
assert head == chaindb_1000.get_canonical_head()
async def test_light_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_20):
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_peer_class=LESPeer,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
client = LightChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh,
MockPeerPoolWithConnectedPeers([client_peer]))
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=LESPeerPoolEventServer), run_request_server(
event_bus,
FakeAsyncChainDB(chaindb_20.db),
server_type=LightRequestServer):
server_peer.logger.info("%s is serving 20 blocks", server_peer)
client_peer.logger.info("%s is syncing up 20", client_peer)
def finalizer():
event_loop.run_until_complete(client.cancel())
# Yield control so that client/server.run() returns, otherwise asyncio will complain.
event_loop.run_until_complete(asyncio.sleep(0.1))
request.addfinalizer(finalizer)
asyncio.ensure_future(client.run())
await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
async def test_skeleton_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_1000):
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_1000.db))
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
client = FastChainSyncer(LatestTestChain(chaindb_fresh.db), chaindb_fresh,
client_peer_pool)
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus, FakeAsyncChainDB(chaindb_1000.db)):
client_peer.logger.info("%s is serving 1000 blocks", client_peer)
server_peer.logger.info("%s is syncing up 1000 blocks", server_peer)
await asyncio.wait_for(client.run(), timeout=20)
head = chaindb_fresh.get_canonical_head()
assert head == chaindb_1000.get_canonical_head()
# Now download the state for the chain's head.
state_downloader = StateDownloader(chaindb_fresh, chaindb_fresh.db,
head.state_root, client_peer_pool)
await asyncio.wait_for(state_downloader.run(), timeout=20)
assert head.state_root in chaindb_fresh.db
async def test_regular_syncer_fallback(request, event_loop, event_bus, chaindb_fresh, chaindb_20):
"""
Test the scenario where a header comes in that's not in memory (but is in the DB)
"""
client_peer, server_peer = await get_directly_linked_peers(
request, event_loop,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
client = FallbackTesting_RegularChainSyncer(
ByzantiumTestChain(chaindb_fresh.db),
chaindb_fresh,
MockPeerPoolWithConnectedPeers([client_peer]))
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool, handler_type=ETHPeerPoolEventServer
), run_request_server(
event_bus, FakeAsyncChainDB(chaindb_20.db)
):
server_peer.logger.info("%s is serving 20 blocks", server_peer)
client_peer.logger.info("%s is syncing up 20", client_peer)
def finalizer():
event_loop.run_until_complete(client.cancel())
# Yield control so that client/server.run() returns, otherwise asyncio will complain.
event_loop.run_until_complete(asyncio.sleep(0.1))
request.addfinalizer(finalizer)
asyncio.ensure_future(client.run())
await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
head = chaindb_fresh.get_canonical_head()
assert head.state_root in chaindb_fresh.db
async def test_proxy_peer_requests(request, event_bus, other_event_bus,
event_loop, chaindb_fresh, chaindb_20):
server_event_bus = event_bus
client_event_bus = other_event_bus
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncChainDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncChainDB(chaindb_20.db),
)
client_peer_pool = MockPeerPoolWithConnectedPeers(
[client_peer], event_bus=client_event_bus)
server_peer_pool = MockPeerPoolWithConnectedPeers(
[server_peer], event_bus=server_event_bus)
async with run_peer_pool_event_server(
client_event_bus,
client_peer_pool,
handler_type=ETHPeerPoolEventServer), run_peer_pool_event_server(
server_event_bus,
server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
server_event_bus,
FakeAsyncChainDB(chaindb_20.db)), run_proxy_peer_pool(
client_event_bus
) as client_proxy_peer_pool, run_proxy_peer_pool(
server_event_bus):
proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer(
client_peer.remote)
headers = await proxy_peer.requests.get_block_headers(0, 1, 0, False)
assert len(headers) == 1
block_header = headers[0]
assert block_header.block_number == 0
receipts = await proxy_peer.requests.get_receipts(headers)
assert len(receipts) == 1
receipt = receipts[0]
assert receipt[1][0] == block_header.receipt_root
block_bundles = await proxy_peer.requests.get_block_bodies(headers)
assert len(block_bundles) == 1
first_bundle = block_bundles[0]
assert first_bundle[1][0] == block_header.transaction_root
node_data = await proxy_peer.requests.get_node_data(
(block_header.state_root, ))
assert node_data[0][0] == block_header.state_root
async def get_peer_and_receive_server(
request, event_loop, event_bus
) -> Tuple[BCCPeer, BCCRequestServer, BCCReceiveServer, asyncio.Queue]:
alice_chain = await get_fake_chain()
bob_chain = await get_fake_chain()
(alice, alice_peer_pool, bob, bob_peer_pool
) = await bcc_helpers.get_directly_linked_peers_in_peer_pools(
request,
event_loop,
alice_chain_db=alice_chain.chaindb,
bob_chain_db=bob_chain.chaindb,
)
msg_queue = asyncio.Queue()
orig_handle_msg = BCCReceiveServer._handle_msg
# Inject a queue to each `BCCReceiveServer`, which puts the message passed to `_handle_msg` to
# the queue, right after every `_handle_msg` finishes.
# This is crucial to make the test be able to wait until `_handle_msg` finishes.
async def _handle_msg(self, base_peer, cmd, msg):
task = asyncio.ensure_future(orig_handle_msg(self, base_peer, cmd,
msg))
def enqueue_msg(future, msg):
msg_queue.put_nowait(msg)
task.add_done_callback(functools.partial(enqueue_msg, msg=msg))
await task
BCCReceiveServer._handle_msg = _handle_msg
async with run_peer_pool_event_server(
event_bus, alice_peer_pool,
BCCPeerPoolEventServer), run_request_server(
event_bus, alice_chain.chaindb,
server_type=BCCRequestServer) as alice_req_server:
bob_recv_server = BCCReceiveServer(chain=bob_chain,
peer_pool=bob_peer_pool)
asyncio.ensure_future(bob_recv_server.run())
await bob_recv_server.events.started.wait()
def finalizer():
event_loop.run_until_complete(bob_recv_server.cancel())
request.addfinalizer(finalizer)
yield alice, alice_req_server, bob_recv_server, msg_queue
async def test_requests_when_peer_in_client_vanishs(request, event_bus,
other_event_bus,
event_loop, chaindb_fresh,
chaindb_20):
server_event_bus = event_bus
client_event_bus = other_event_bus
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncChainDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncChainDB(chaindb_20.db),
)
client_peer_pool = MockPeerPoolWithConnectedPeers(
[client_peer], event_bus=client_event_bus)
server_peer_pool = MockPeerPoolWithConnectedPeers(
[server_peer], event_bus=server_event_bus)
async with run_peer_pool_event_server(
client_event_bus,
client_peer_pool,
handler_type=ETHPeerPoolEventServer), run_peer_pool_event_server(
server_event_bus,
server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
server_event_bus,
FakeAsyncChainDB(chaindb_20.db)), run_proxy_peer_pool(
client_event_bus
) as client_proxy_peer_pool, run_proxy_peer_pool(
server_event_bus):
proxy_peer = await client_proxy_peer_pool.ensure_proxy_peer(
client_peer.remote)
# We remove the peer from the client and assume to see PeerConnectionLost exceptions raised
client_peer_pool.connected_nodes.pop(client_peer.remote)
with pytest.raises(PeerConnectionLost):
await proxy_peer.requests.get_block_headers(0, 1, 0, False)
with pytest.raises(PeerConnectionLost):
await proxy_peer.requests.get_receipts(())
with pytest.raises(PeerConnectionLost):
await proxy_peer.requests.get_block_bodies(())
with pytest.raises(PeerConnectionLost):
await proxy_peer.requests.get_node_data(())
async def test_fast_syncer(request, event_bus, event_loop, chaindb_fresh,
chaindb_20):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_20.db)
peer_pair = ETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer, server_peer):
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
client = FastChainSyncer(LatestTestChain(chaindb_fresh.db),
chaindb_fresh, client_peer_pool)
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus,
server_peer_pool,
handler_type=ETHPeerPoolEventServer,
), run_request_server(
event_bus,
AsyncChainDB(chaindb_20.db),
):
server_peer.logger.info("%s is serving 20 blocks", server_peer)
client_peer.logger.info("%s is syncing up 20", client_peer)
# FastChainSyncer.run() will return as soon as it's caught up with the peer.
await asyncio.wait_for(client.run(), timeout=5)
head = chaindb_fresh.get_canonical_head()
assert head == chaindb_20.get_canonical_head()
# Now download the state for the chain's head.
state_downloader = StateDownloader(chaindb_fresh, chaindb_fresh.db,
head.state_root,
client_peer_pool)
await asyncio.wait_for(state_downloader.run(), timeout=5)
assert head.state_root in chaindb_fresh.db
async def test_regular_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_20):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_20.db)
peer_pair = ETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer, server_peer):
client = RegularChainSyncer(
ByzantiumTestChain(chaindb_fresh.db), chaindb_fresh,
MockPeerPoolWithConnectedPeers([client_peer]))
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus, AsyncChainDB(chaindb_20.db)):
server_peer.logger.info("%s is serving 20 blocks", server_peer)
client_peer.logger.info("%s is syncing up 20", client_peer)
def finalizer():
event_loop.run_until_complete(client.cancel())
# Yield control so that client/server.run() returns, otherwise
# asyncio will complain.
event_loop.run_until_complete(asyncio.sleep(0.1))
request.addfinalizer(finalizer)
asyncio.ensure_future(client.run())
await wait_for_head(chaindb_fresh, chaindb_20.get_canonical_head())
head = chaindb_fresh.get_canonical_head()
assert head.state_root in chaindb_fresh.db
async def test_fast_syncer(request, event_bus, event_loop, chaindb_fresh,
chaindb_20):
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_20.db))
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
client = FastChainSyncer(ByzantiumTestChain(chaindb_fresh.db),
chaindb_fresh, client_peer_pool)
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus,
server_peer_pool,
handler_type=ETHPeerPoolEventServer,
), run_request_server(
event_bus,
FakeAsyncChainDB(chaindb_20.db),
):
server_peer.logger.info("%s is serving 20 blocks", server_peer)
client_peer.logger.info("%s is syncing up 20", client_peer)
# FastChainSyncer.run() will return as soon as it's caught up with the peer.
await asyncio.wait_for(client.run(), timeout=2)
head = chaindb_fresh.get_canonical_head()
assert head == chaindb_20.get_canonical_head()
# Now download the state for the chain's head.
state_downloader = StateDownloader(chaindb_fresh, chaindb_fresh.db,
head.state_root, client_peer_pool)
await asyncio.wait_for(state_downloader.run(), timeout=2)
assert head.state_root in chaindb_fresh.db
async def test_beam_syncer(request,
event_loop,
event_bus,
chaindb_fresh,
chaindb_churner,
beam_to_block,
checkpoint=None):
client_context = ChainContextFactory(headerdb__db=chaindb_fresh.db)
server_context = ChainContextFactory(headerdb__db=chaindb_churner.db)
peer_pair = ETHPeerPairFactory(
alice_peer_context=client_context,
bob_peer_context=server_context,
event_bus=event_bus,
)
async with peer_pair as (client_peer, server_peer):
# Need a name that will be unique per xdist-process, otherwise
# lahja IPC endpoints in each process will clobber each other
unique_process_name = uuid.uuid4()
# manually add endpoint for beam vm to make requests
pausing_config = ConnectionConfig.from_name(
f"PausingEndpoint-{unique_process_name}")
# manually add endpoint for trie data gatherer to serve requests
gatherer_config = ConnectionConfig.from_name(
f"GathererEndpoint-{unique_process_name}")
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus, AsyncChainDB(chaindb_churner.db)
), AsyncioEndpoint.serve(
pausing_config) as pausing_endpoint, AsyncioEndpoint.serve(
gatherer_config) as gatherer_endpoint:
client_chain = make_pausing_beam_chain(
((0, PetersburgVM), ),
chain_id=999,
db=chaindb_fresh.db,
event_bus=pausing_endpoint,
loop=event_loop,
)
client = BeamSyncer(
client_chain,
chaindb_fresh.db,
AsyncChainDB(chaindb_fresh.db),
client_peer_pool,
gatherer_endpoint,
force_beam_block_number=beam_to_block,
checkpoint=checkpoint,
)
client_peer.logger.info("%s is serving churner blocks",
client_peer)
server_peer.logger.info("%s is syncing up churner blocks",
server_peer)
import_server = BlockImportServer(
pausing_endpoint,
client_chain,
token=client.cancel_token,
)
asyncio.ensure_future(import_server.run())
await pausing_endpoint.connect_to_endpoints(gatherer_config)
asyncio.ensure_future(client.run())
# We can sync at least 10 blocks in 1s at current speeds, (or reach the current one)
# Trying to keep the tests short-ish. A fuller test could always set the target header
# to the chaindb_churner canonical head, and increase the timeout significantly
target_block_number = min(beam_to_block + 10, 129)
target_head = chaindb_churner.get_canonical_block_header_by_number(
target_block_number)
await wait_for_head(chaindb_fresh, target_head, sync_timeout=10)
assert target_head.state_root in chaindb_fresh.db
# first stop the import server, so it doesn't hang waiting for state data
await import_server.cancel()
await client.cancel()
async def test_beam_syncer(request, event_loop, event_bus, chaindb_fresh,
chaindb_churner, beam_to_block):
client_peer, server_peer = await get_directly_linked_peers(
request,
event_loop,
alice_headerdb=FakeAsyncHeaderDB(chaindb_fresh.db),
bob_headerdb=FakeAsyncHeaderDB(chaindb_churner.db))
# manually add endpoint for beam vm to make requests
pausing_config = ConnectionConfig.from_name("PausingEndpoint")
# manually add endpoint for trie data gatherer to serve requests
gatherer_config = ConnectionConfig.from_name("GathererEndpoint")
client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer])
server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer],
event_bus=event_bus)
async with run_peer_pool_event_server(
event_bus, server_peer_pool,
handler_type=ETHPeerPoolEventServer), run_request_server(
event_bus,
FakeAsyncChainDB(chaindb_churner.db)), AsyncioEndpoint.serve(
pausing_config) as pausing_endpoint, AsyncioEndpoint.serve(
gatherer_config) as gatherer_endpoint:
BeamPetersburgVM = pausing_vm_decorator(PetersburgVM, pausing_endpoint)
class BeamPetersburgTestChain(FakeAsyncChain):
vm_configuration = ((0, BeamPetersburgVM), )
network_id = 999
client_chain = BeamPetersburgTestChain(chaindb_fresh.db)
client = BeamSyncer(
client_chain,
chaindb_fresh.db,
client_chain.chaindb,
client_peer_pool,
gatherer_endpoint,
beam_to_block,
)
client_peer.logger.info("%s is serving churner blocks", client_peer)
server_peer.logger.info("%s is syncing up churner blocks", server_peer)
import_server = BlockImportServer(pausing_endpoint,
client_chain,
token=client.cancel_token)
asyncio.ensure_future(import_server.run())
await pausing_endpoint.connect_to_endpoints(gatherer_config)
asyncio.ensure_future(client.run())
# We can sync at least 10 blocks in 1s at current speeds, (or reach the current one)
# Trying to keep the tests short-ish. A fuller test could always set the target header
# to the chaindb_churner canonical head, and increase the timeout significantly
target_block_number = min(beam_to_block + 10, 129)
target_head = chaindb_churner.get_canonical_block_header_by_number(
target_block_number)
await wait_for_head(chaindb_fresh, target_head, sync_timeout=4)
assert target_head.state_root in chaindb_fresh.db
# first stop the import server, so it doesn't hang waiting for state data
await import_server.cancel()
await client.cancel()
async def get_peer_and_receive_server(
request, event_loop, event_bus
) -> Tuple[BCCPeer, BCCRequestServer, BCCReceiveServer, asyncio.Queue]:
alice_chain = await get_fake_chain()
bob_chain = await get_fake_chain()
alice_private_key = PrivateKeyFactory()
bob_private_key = PrivateKeyFactory()
alice_context = BeaconContextFactory(chain_db=alice_chain.chaindb)
bob_context = BeaconContextFactory(chain_db=bob_chain.chaindb)
peer_pair = BCCPeerPairFactory(
alice_peer_context=alice_context,
alice_private_key=alice_private_key,
bob_peer_context=bob_context,
bob_private_key=bob_private_key,
event_bus=event_bus,
)
async with peer_pair as (alice, bob):
alice_pool_ctx = BCCPeerPoolFactory.run_for_peer(
alice,
privkey=alice_private_key,
context=alice_context,
event_bus=event_bus,
)
bob_pool_ctx = BCCPeerPoolFactory.run_for_peer(
bob,
privkey=bob_private_key,
context=bob_context,
event_bus=event_bus,
)
async with alice_pool_ctx as alice_peer_pool, bob_pool_ctx as bob_peer_pool:
msg_queue = asyncio.Queue()
orig_handle_msg = BCCReceiveServer._handle_msg
# Inject a queue to each `BCCReceiveServer`, which puts the message
# passed to `_handle_msg` to the queue, right after every `_handle_msg`
# finishes. This is crucial to make the test be able to wait until
# `_handle_msg` finishes.
async def _handle_msg(self, base_peer, cmd, msg):
task = asyncio.ensure_future(
orig_handle_msg(self, base_peer, cmd, msg))
def enqueue_msg(future, msg):
msg_queue.put_nowait(msg)
task.add_done_callback(functools.partial(enqueue_msg, msg=msg))
await task
BCCReceiveServer._handle_msg = _handle_msg
async with run_peer_pool_event_server(
event_bus, alice_peer_pool,
BCCPeerPoolEventServer), run_request_server(
event_bus,
alice_chain.chaindb,
server_type=BCCRequestServer) as alice_req_server:
bob_recv_server = BCCReceiveServer(chain=bob_chain,
peer_pool=bob_peer_pool)
asyncio.ensure_future(bob_recv_server.run())
await bob_recv_server.events.started.wait()
def finalizer():
event_loop.run_until_complete(bob_recv_server.cancel())
request.addfinalizer(finalizer)
yield alice, alice_req_server, bob_recv_server, msg_queue