def test_partial_proof_serialization_and_deserialization(data):
content = data.draw(st.binary(min_size=1, max_size=10240))
slice_start = data.draw(st.integers(min_value=0, max_value=len(content) - 1))
slice_stop = data.draw(st.integers(min_value=slice_start, max_value=len(content)))
data_slice = slice(slice_start, slice_stop)
full_proof = compute_proof(content, sedes=content_sedes)
slice_length = max(0, data_slice.stop - data_slice.start)
partial_proof = full_proof.to_partial(
start_at=data_slice.start, partial_data_length=slice_length,
)
assert partial_proof.get_hash_tree_root() == full_proof.get_hash_tree_root()
serialized = partial_proof.serialize()
result = Proof.deserialize(serialized)
validate_proof(result)
assert result == partial_proof
partial = result.get_proven_data()
data_from_partial = partial[data_slice]
assert data_from_partial == content[data_slice]
def test_ssz_partial_proof_fuzzy(data):
content = data.draw(st.binary(min_size=1, max_size=10240))
content_length = len(content)
slice_start = data.draw(
st.integers(min_value=0, max_value=max(0, content_length - 1)))
slice_stop = data.draw(
st.integers(min_value=slice_start, max_value=content_length))
data_slice = slice(slice_start, slice_stop)
full_proof = compute_proof(content, sedes=content_sedes)
slice_length = max(0, data_slice.stop - data_slice.start - 1)
partial_proof = full_proof.to_partial(
start_at=data_slice.start,
partial_data_length=slice_length,
)
assert partial_proof.get_hash_tree_root() == full_proof.get_hash_tree_root(
)
validate_proof(partial_proof)
assert is_proof_valid(partial_proof)
partial = partial_proof.get_proven_data()
data_from_partial = partial[data_slice]
assert data_from_partial == content[data_slice]
async def test_content_provider_restricts_max_chunks(
alice,
bob,
alice_alexandria_network,
bob_alexandria_client,
):
content = ContentFactory(length=1024 * 10)
content_key = b"test-content-key"
content_storage = MemoryContentStorage({content_key: content})
proof = compute_proof(content, sedes=content_sedes)
content_provider = ContentProvider(bob_alexandria_client,
(content_storage, ),
max_chunks_per_request=16)
async with background_trio_service(content_provider):
# this ensures that the subscription is in place.
await content_provider.ready()
with trio.fail_after(2):
proof = await alice_alexandria_network.get_content_proof(
bob.node_id,
hash_tree_root=proof.get_hash_tree_root(),
content_key=content_key,
start_chunk_index=0,
max_chunks=100,
endpoint=bob.endpoint,
)
validate_proof(proof)
num_leaf_elements = len(
tuple(element for element in proof.elements
if len(element.path) == proof.path_bit_length))
assert num_leaf_elements == 16
async def test_content_provider_serves_large_content(
alice,
bob,
alice_alexandria_network,
bob_alexandria_client,
):
content = ContentFactory(length=1024 * 10)
content_key = b"test-content-key"
content_storage = MemoryContentStorage({content_key: content})
proof = compute_proof(content, sedes=content_sedes)
content_provider = ContentProvider(bob_alexandria_client,
(content_storage, ))
async with background_trio_service(content_provider):
async with alice_alexandria_network.client.subscribe(
ContentMessage) as subscription:
# this ensures that the subscription is in place.
await content_provider.ready()
with trio.fail_after(2):
content_retrieval_ctx = alice_alexandria_network.retrieve_content(
content_key,
proof.get_hash_tree_root(),
)
async with content_retrieval_ctx as content_retrieval:
await content_retrieval.node_queue.add(bob.node_id)
result = await content_retrieval.wait_content_proof()
validate_proof(result)
result_data = result.get_proven_data()
assert result_data[0:len(content)] == content
response = await subscription.receive()
assert response.message.payload.is_proof is True
def test_full_proof_serialization_and_deserialization(content):
proof = compute_proof(content, sedes=content_sedes)
serialized = proof.serialize()
result = Proof.deserialize(serialized)
assert result.get_hash_tree_root() == proof.get_hash_tree_root()
validate_proof(result)
assert result == proof
async def test_content_retrieval_with_griefing_peer_sending_tiny_chunks(
alice,
bob,
alice_alexandria_network,
bob_alexandria_client,
):
content = ContentFactory(length=1024 * 10)
proof = compute_proof(content, sedes=content_sedes)
content_retrieval = ContentRetrieval(
alice_alexandria_network,
content_key=b"test-key",
hash_tree_root=proof.get_hash_tree_root(),
)
async with bob_alexandria_client.subscribe(
GetContentMessage) as subscription:
async with trio.open_nursery() as nursery:
async def _serve():
async for request in subscription:
start_at = request.message.payload.start_chunk_index * 32
# We only every return a proof for 1 chunk of data
end_at = min(len(content), start_at + 32)
partial = proof.to_partial(start_at, end_at - start_at)
payload = partial.serialize()
await bob_alexandria_client.send_content(
request.sender_node_id,
request.sender_endpoint,
is_proof=True,
payload=payload,
request_id=request.request_id,
)
nursery.start_soon(_serve)
await content_retrieval.node_queue.add(bob.node_id)
with trio.fail_after(10):
async with background_trio_service(content_retrieval):
with trio.fail_after(5):
result = await content_retrieval.wait_content_proof()
validate_proof(result)
result_data = result.get_proven_data()
assert result_data[0:len(content)] == content
nursery.cancel_scope.cancel()
async def test_alexandria_network_get_content_proof_api(
alice,
bob,
alice_alexandria_network,
bob_alexandria_client,
content_size,
):
content = ContentFactory(length=content_size)
proof = compute_proof(content, sedes=content_sedes)
async with bob_alexandria_client.subscribe(
GetContentMessage) as subscription:
async with trio.open_nursery() as nursery:
async def _serve():
request = await subscription.receive()
if content_size > 1024:
partial = proof.to_partial(
request.message.payload.start_chunk_index * 32,
request.message.payload.max_chunks * 32,
)
payload = partial.serialize()
is_proof = True
else:
payload = content
is_proof = False
await bob_alexandria_client.send_content(
request.sender_node_id,
request.sender_endpoint,
is_proof=is_proof,
payload=payload,
request_id=request.request_id,
)
nursery.start_soon(_serve)
with trio.fail_after(2):
partial = await alice_alexandria_network.get_content_proof(
bob.node_id,
hash_tree_root=proof.get_hash_tree_root(),
content_key=b"test-content-key",
start_chunk_index=0,
max_chunks=16,
)
validate_proof(partial)
partial_data = partial.get_proven_data()
assert partial_data[0:16 * 32] == content[0:16 * 32]
def test_ssz_partial_proof_merge_fuzzy(data):
content = data.draw(st.binary(min_size=1, max_size=10240))
full_proof = compute_proof(content, sedes=content_sedes)
slice_a_start = data.draw(
st.integers(min_value=0, max_value=max(0,
len(content) - 1)))
slice_a_stop = data.draw(
st.integers(min_value=slice_a_start, max_value=len(content)))
data_slice_a = slice(slice_a_start, slice_a_stop)
slice_a_length = max(0, data_slice_a.stop - data_slice_a.start - 1)
slice_b_start = data.draw(
st.integers(min_value=0, max_value=max(0,
len(content) - 1)))
slice_b_stop = data.draw(
st.integers(min_value=slice_b_start, max_value=len(content)))
data_slice_b = slice(slice_b_start, slice_b_stop)
slice_b_length = max(0, data_slice_b.stop - data_slice_b.start - 1)
partial_a = full_proof.to_partial(
start_at=data_slice_a.start,
partial_data_length=slice_a_length,
)
partial_a_data = partial_a.get_proven_data()
partial_b = full_proof.to_partial(
start_at=data_slice_b.start,
partial_data_length=slice_b_length,
)
partial_b_data = partial_b.get_proven_data()
combined_proof = partial_a.merge(partial_b)
assert combined_proof.get_hash_tree_root(
) == full_proof.get_hash_tree_root()
validate_proof(combined_proof)
combined_data = combined_proof.get_proven_data()
assert combined_data[data_slice_a] == partial_a_data[data_slice_a]
assert combined_data[data_slice_a] == content[data_slice_a]
assert combined_data[data_slice_b] == partial_b_data[data_slice_b]
assert combined_data[data_slice_b] == content[data_slice_b]
def test_ssz_full_proofs(content):
expected_hash_tree_root = get_hash_tree_root(content, sedes=content_sedes)
proof = compute_proof(content, sedes=content_sedes)
validate_proof(proof)
assert is_proof_valid(proof)
assert proof.get_hash_tree_root() == expected_hash_tree_root
proven_data_segments = proof.get_proven_data_segments()
assert len(proven_data_segments) == 1
start_index, proven_data = proven_data_segments[0]
assert start_index == 0
assert proven_data == content
proven_data = proof.get_proven_data()
assert proven_data[0:len(content)] == content
def test_ssz_partial_proof_construction(content, data_slice):
full_proof = compute_proof(content, sedes=short_content_sedes)
slice_length = data_slice.stop - data_slice.start
partial_proof = full_proof.to_partial(
start_at=data_slice.start,
partial_data_length=slice_length,
)
assert partial_proof.get_hash_tree_root() == full_proof.get_hash_tree_root(
)
validate_proof(partial_proof)
assert is_proof_valid(partial_proof)
partial = partial_proof.get_proven_data()
data_from_partial = partial[data_slice]
assert data_from_partial == content[data_slice]
def test_ssz_partial_proof_merge():
full_proof = compute_proof(CONTENT_12345, sedes=short_content_sedes)
proof_a = full_proof.to_partial(0, 64)
proof_b = full_proof.to_partial(64, 64)
proof_a_data = proof_a.get_proven_data()
proof_b_data = proof_b.get_proven_data()
with pytest.raises(IndexError):
proof_a_data[64:128]
with pytest.raises(IndexError):
proof_b_data[0:64]
with pytest.raises(IndexError):
proof_a_data[0:128]
with pytest.raises(IndexError):
proof_b_data[0:128]
combined_proof = proof_a.merge(proof_b)
validate_proof(combined_proof)
combined_data = combined_proof.get_proven_data()
assert combined_data[0:128] == CONTENT_12345[0:128]
async def _worker(
self, worker_id: int, send_channel: trio.abc.SendChannel[Proof]
) -> None:
worker_name = f"Worker[{self.content_key.hex()}:{worker_id}]"
while True:
async with self.node_queue.reserve() as node_id:
self.logger.debug("%s: reserved node: %s", worker_name, node_id.hex())
async with self._segment_queue.reserve() as segment:
start_data_index, data_length = segment
start_chunk_index = start_data_index // CHUNK_SIZE
max_chunks = get_chunk_count_for_data_length(data_length)
self.logger.debug(
"%s: reserved chunk: start_index=%d max_chunks=%d",
worker_name,
start_chunk_index,
max_chunks,
)
try:
proof = await self._network.get_content_proof(
node_id,
hash_tree_root=self.hash_tree_root,
content_key=self.content_key,
start_chunk_index=start_chunk_index,
max_chunks=max_chunks,
)
except trio.TooSlowError:
self.logger.debug(
"%s: timeout: node=%s", worker_name, node_id.hex(),
)
continue
try:
validate_proof(proof)
except ValidationError:
self.logger.debug(
"%s: removing node for sending invalid proof: node=%s",
worker_name,
node_id.hex(),
)
# If a peer gives us an invalid proof, remove them
# from rotation.
await self.node_queue.remove(node_id)
continue
# check that the proof contains at minimum the first chunk we requested.
if not proof.has_chunk(start_chunk_index):
self.logger.debug(
"%s: removing node for not returning requested chunk: node=%s",
worker_name,
node_id.hex(),
)
# If the peer didn't include the start chunk,
# remove them from rotation.
await self.node_queue.remove(node_id)
continue
self.logger.debug(
"%s: sending partial proof: node=%s",
worker_name,
node_id.hex(),
)
await send_channel.send(proof)
# Determine if there are any subsections to this
# segment that are still missing.
remaining_segments = segment.intersection(
tuple(proof.get_missing_segments())
)
# This *timeout* ensures that the workers will not deadlock on
# a full `segment_queue`. In the case where we hit this
# timeout we may end up re-requesting a proof that we already
# have but that is *ok* and doesn't cause anything to break.
with trio.move_on_after(2):
# Remove the segment and push any still missing
# sub-segments onto the queue
for sub_segment in remaining_segments:
await self._segment_queue.add(sub_segment)
# It is important that the removal happen *after*
# we push the sub-segments on, otherwise, if we
# timeout after the main segment has been removed
# but before the sub-segments have been added,
# we'll lose track of the still missing
# sub-segments.
await self._segment_queue.remove(segment)
def test_proof_get_minimal_proof_elements_mixed_depths():
r"""
0: X
/ \
/ \
1: 0 1
/ \ (length)
/ \
/ \
/ \
/ \
/ \
/ \
2: 0 1
/ \ / \
/ \ / \
/ \ / \
3: 0 1 0 1
/ \ / \ / \ / \
/ \ / \ / \ / \
4: 0 1 0 1 X X 0 1
/ \ / \ / \ / \ / \ / \ / \ / \
5: 0 1 X X 0 1 0 1 X X X X 0 1 X X
Nodes marked with `X` have already been collapsed
Tests:
A: |<--------->|
B: |<----------------------->|
"""
full_proof = compute_proof(CONTENT_512, sedes=short_content_sedes)
section_a = full_proof.get_elements_under(p(0, 0, 0, 0))
section_b = full_proof.get_minimal_proof_elements(2, 2)
section_c = full_proof.get_elements_under(p(0, 0, 1))
section_d = full_proof.get_minimal_proof_elements(8, 4)
section_e = full_proof.get_elements_under(p(0, 1, 1, 0))
section_f = full_proof.get_minimal_proof_elements(14, 2)
section_g = (full_proof.get_element((True, )), )
sparse_elements = sum(
(
section_a,
section_b,
section_c,
section_d,
section_e,
section_f,
section_g,
),
(),
)
sparse_proof = Proof(
elements=sparse_elements,
sedes=short_content_sedes,
)
validate_proof(sparse_proof)
# spans nodes at different levels
elements_a = sparse_proof.get_minimal_proof_elements(0, 4)
assert len(elements_a) == 1
assert elements_a[0].path == p(0, 0, 0)
assert elements_a[0].value == hash_eth2(
hash_eth2(CONTENT_512[0:64]) + hash_eth2(CONTENT_512[64:128]))
elements_b = sparse_proof.get_minimal_proof_elements(8, 8)
assert len(elements_b) == 1
assert elements_b[0].path == p(0, 1)
hash_010 = hash_eth2(
hash_eth2(CONTENT_512[256:320]) + hash_eth2(CONTENT_512[320:384]))
hash_011 = hash_eth2(
hash_eth2(CONTENT_512[384:448]) + hash_eth2(CONTENT_512[448:512]))
assert elements_b[0].value == hash_eth2(hash_010 + hash_011)
