def test_scaled_batches_are_complete(scaled_workers, source):
batches = get_scaled_batches(scaled_workers, source)
# every resulting worker must be in the input workers set
assert all(worker in scaled_workers for worker in batches.keys())
# no batches may be empty
assert all(len(batch) > 0 for batch in batches.values())
# the total number of elements returned must be equal to the length of the source
assert sum(map(len, batches.values())) == len(source)
# all the elements must be found in the batches
batched_set = set(el for batch in batches.values() for el in batch)
assert batched_set == set(source)
# there must be no duplicates
batched_tuple = tuple(el for batch in batches.values() for el in batch)
assert len(batched_tuple) == len(batched_set)
def _request_block_parts(
self,
target_td: int,
headers: List[BlockHeader],
request_func: Callable[[ETHPeer, List[BlockHeader]], None],
block_part: BlockPartEnum,
) -> int:
"""
:return: how many requests were made (one request per peer)
"""
peers = self.peer_pool.get_peers(target_td)
if not peers:
raise NoEligiblePeers()
# request headers proportionally, so faster peers are asked for more parts than slow peers
speeds = {peer: self._get_peer_stats(peer, block_part).get_throughput() for peer in peers}
peer_batches = get_scaled_batches(speeds, headers)
for peer, batch in peer_batches.items():
self._get_peer_stats(peer, block_part).begin_work()
request_func(cast(ETHPeer, peer), batch)
return len(peer_batches)
def test_scaled_batches_nan():
with pytest.raises(ValidationError):
get_scaled_batches({
b'a': 1.2,
b'b': float('nan')
}, ['has source data'])
def test_scaled_batches_empty():
with pytest.raises(ValidationError):
get_scaled_batches({}, ['has source data'])
def test_scaled_batches(scaled_workers, source, expected_batch_sizes):
batches = get_scaled_batches(scaled_workers, source)
batch_sizes = {worker: len(batch) for worker, batch in batches.items()}
assert batch_sizes == expected_batch_sizes