async def test_syncer_proposing(request, event_loop):
# setup a-b topology
peer_a_b, peer_b_a = await get_directly_linked_sharding_peers(request, event_loop)
peer_a_b_subscriber = MockPeerPoolSubscriber()
peer_a_b.add_subscriber(peer_a_b_subscriber)
peer_pool_b = MockPeerPoolWithConnectedPeers([peer_b_a])
# setup shard dbs at b
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
# start shard syncer
syncer = ShardSyncer(shard, peer_pool_b)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(syncer.cancel())
request.addfinalizer(finalizer)
# propose at b and check that it announces its proposal
syncer.propose()
peer, cmd, msg = await asyncio.wait_for(
peer_a_b_subscriber.msg_queue.get(),
timeout=1,
)
assert peer == peer_a_b
assert isinstance(cmd, NewCollationHashes)
assert len(msg["collation_hashes_and_periods"]) == 1
proposed_hash = msg["collation_hashes_and_periods"][0][0]
# test that the collation has been added to the shard
shard.get_collation_by_hash(proposed_hash)
async def test_shard_syncer(connections, request, event_loop):
peers_by_server = {}
for server_id1, server_id2 in connections:
peer1, peer2 = await get_directly_linked_sharding_peers(
request, event_loop)
peers_by_server.setdefault(server_id1, []).append(peer1)
peers_by_server.setdefault(server_id2, []).append(peer2)
syncers = []
for _, peers in sorted(peers_by_server.items()):
peer_pool = MockPeerPoolWithConnectedPeers(peers)
shard_db = ShardDB(MemoryDB())
syncer = ShardSyncer(Shard(shard_db, 0), peer_pool)
syncers.append(syncer)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(
asyncio.gather(*[syncer.cancel() for syncer in syncers]))
request.addfinalizer(finalizer)
# let each node propose and check that collation appears at all other nodes
for proposer in syncers:
collation = proposer.propose()
await asyncio.wait_for(asyncio.gather(*[
syncer.collations_received_event.wait() for syncer in syncers
if syncer != proposer
]),
timeout=2)
for syncer in syncers:
assert syncer.shard.get_collation_by_hash(
collation.hash) == collation
async def test_syncer_requests_new_collations(request, event_loop):
# setup a-b topology
peer_a_b, peer_b_a = await get_directly_linked_sharding_peers(request, event_loop)
peer_a_b_subscriber = MockPeerPoolSubscriber()
peer_a_b.add_subscriber(peer_a_b_subscriber)
peer_pool_b = MockPeerPoolWithConnectedPeers([peer_b_a])
# setup shard dbs at b
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
# start shard syncer
syncer = ShardSyncer(shard, peer_pool_b)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(syncer.cancel())
request.addfinalizer(finalizer)
# notify b about new hashes at a and check that it requests them
hashes_and_periods = ((b"\xaa" * 32, 0),)
peer_a_b.sub_proto.send_new_collation_hashes(hashes_and_periods)
peer, cmd, msg = await asyncio.wait_for(
peer_a_b_subscriber.msg_queue.get(),
timeout=1,
)
assert peer == peer_a_b
assert isinstance(cmd, GetCollations)
assert msg["collation_hashes"] == (hashes_and_periods[0][0],)
def __init__(
self,
privkey: datatypes.PrivateKey,
address: Address,
network_id: int,
min_peers: int = 0,
peer_class: Type[BasePeer] = ShardingPeer,
peer_pool_class: Type[PeerPool] = PeerPool,
bootstrap_nodes: List[str] = [],
) -> None:
BaseService.__init__(self, CancelToken('ShardingServer'))
self.privkey = privkey
self.address = address
self.network_id = network_id
self.peer_class = peer_class
self.discovery = DiscoveryProtocol(self.privkey,
self.address,
bootstrap_nodes=bootstrap_nodes)
# XXX: This is not supposed to work and causes both the PeerPool and Server to crash, but
# the tests in test_sharding.py don't seem to care
self.headerdb = None
self.peer_pool = peer_pool_class(
peer_class,
self.headerdb,
self.network_id,
self.privkey,
self.discovery,
min_peers=min_peers,
)
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
self.syncer = ShardSyncer(shard, self.peer_pool,
self.cancel_token) # type: ignore
async def test_new_collations_notification(request, event_loop):
# setup a-b-c topology
peer_a_b, peer_b_a = await get_directly_linked_sharding_peers(
request, event_loop)
peer_b_c, peer_c_b = await get_directly_linked_sharding_peers(
request, event_loop)
peer_c_b_subscriber = asyncio.Queue()
peer_c_b.add_subscriber(peer_c_b_subscriber)
peer_pool_b = MockPeerPoolWithConnectedPeers([peer_b_a, peer_b_c])
# setup shard dbs at b
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
# start shard syncer
syncer = ShardSyncer(shard, peer_pool_b)
asyncio.ensure_future(syncer.run())
def finalizer():
event_loop.run_until_complete(syncer.cancel())
request.addfinalizer(finalizer)
# send collation from a to b and check that c gets notified
c1 = next(collations)
peer_a_b.sub_proto.send_collations(0, [c1])
peer, cmd, msg = await asyncio.wait_for(
peer_c_b_subscriber.get(),
timeout=1,
)
assert peer == peer_c_b
assert isinstance(cmd, NewCollationHashes)
assert msg["collation_hashes_and_periods"] == ((c1.hash, c1.period), )
# check that c won't be notified about c1 again
c2 = next(collations)
peer_a_b.sub_proto.send_collations(0, [c1, c2])
peer, cmd, msg = await asyncio.wait_for(
peer_c_b_subscriber.get(),
timeout=1,
)
assert peer == peer_c_b
assert isinstance(cmd, NewCollationHashes)
assert msg["collation_hashes_and_periods"] == ((c2.hash, c2.period), )
async def test_collation_requests(request, event_loop):
# setup two peers
sender, receiver = await get_directly_linked_sharding_peers(request, event_loop)
receiver_peer_pool = MockPeerPoolWithConnectedPeers([receiver])
# setup shard db for request receiving node
receiver_db = ShardDB(MemoryDB())
receiver_shard = Shard(receiver_db, 0)
# create three collations and add two to the shard of the receiver
# body is shared to avoid unnecessary chunk root calculation
body = zpad_right(b"body", COLLATION_SIZE)
chunk_root = calc_chunk_root(body)
c1 = Collation(CollationHeader(0, chunk_root, 0, zpad_right(b"proposer1", 20)), body)
c2 = Collation(CollationHeader(0, chunk_root, 1, zpad_right(b"proposer2", 20)), body)
c3 = Collation(CollationHeader(0, chunk_root, 2, zpad_right(b"proposer3", 20)), body)
for collation in [c1, c2]:
receiver_shard.add_collation(collation)
# start shard syncer
receiver_syncer = ShardSyncer(receiver_shard, receiver_peer_pool)
asyncio.ensure_future(receiver_syncer.run())
def finalizer():
event_loop.run_until_complete(receiver_syncer.cancel())
request.addfinalizer(finalizer)
cancel_token = CancelToken("test")
# request single collation
received_collations = await asyncio.wait_for(
sender.get_collations([c1.hash], cancel_token),
timeout=1,
)
assert received_collations == set([c1])
# request multiple collations
received_collations = await asyncio.wait_for(
sender.get_collations([c1.hash, c2.hash], cancel_token),
timeout=1,
)
assert received_collations == set([c1, c2])
# request no collations
received_collations = await asyncio.wait_for(
sender.get_collations([], cancel_token),
timeout=1,
)
assert received_collations == set()
# request unknown collation
received_collations = await asyncio.wait_for(
sender.get_collations([c3.hash], cancel_token),
timeout=1,
)
assert received_collations == set()
# request multiple collations, including unknown one
received_collations = await asyncio.wait_for(
sender.get_collations([c1.hash, c2.hash, c3.hash], cancel_token),
timeout=1,
)
assert received_collations == set([c1, c2])
def shard_db():
return ShardDB(get_db_backend())
def _make_syncer(self, peer_pool: PeerPool) -> BaseService:
shard_db = ShardDB(MemoryDB())
shard = Shard(shard_db, 0)
return ShardSyncer(shard, peer_pool, self.cancel_token)