def test_unsign_integers_more_than_32_bytes(data, num_bits):
uint_n = UInt(num_bits)
value = data.draw(st.integers(
min_value=0,
max_value=2**num_bits - 1,
))
expected = hash_eth2(value.to_bytes(num_bits // 8, "little"))
assert hash_tree_root(value, uint_n) == expected
def hash_layer(child_layer: Sequence[bytes]) -> Tuple[Hash32, ...]:
if len(child_layer) % 2 != 0:
raise ValueError("Layer must have an even number of elements")
child_pairs = partition(2, child_layer)
parent_layer = tuple(
hash_eth2(left_child + right_child)
for left_child, right_child in child_pairs)
return parent_layer
def merge(leaf: bytes, leaf_index: int) -> None:
node = leaf
layer = 0
while True:
if leaf_index & (1 << layer) == 0:
if leaf_index == chunk_len and layer < chunk_depth:
# Keep going if we are complementing the void to the next power of 2
key = node + ZERO_HASHES[layer]
if key not in cache:
cache[key] = hash_eth2(key)
node = cache[key]
else:
break
else:
key = merkleized_result_per_layers[layer] + node
if key not in cache:
cache[key] = hash_eth2(key)
node = cache[key]
layer += 1
merkleized_result_per_layers[layer] = node
def hash_layer(child_layer: RawHashTreeLayer,
layer_index: int) -> RawHashTreeLayer:
if len(child_layer) % 2 == 0:
padded_child_layer = child_layer
else:
padded_child_layer = child_layer.append(ZERO_HASHES[layer_index])
child_pairs = partition(2, padded_child_layer)
parent_layer = pvector(
hash_eth2(left_child + right_child)
for left_child, right_child in child_pairs)
return parent_layer
def _get_merkleized_result(
chunks: Sequence[Hash32],
chunk_len: int,
chunk_depth: int,
max_depth: int,
cache: CacheObj,
) -> Tuple[Hash32, CacheObj]:
merkleized_result_per_layers = [None for _ in range(max_depth + 1)]
def merge(leaf: bytes, leaf_index: int) -> None:
node = leaf
layer = 0
while True:
if leaf_index & (1 << layer) == 0:
if leaf_index == chunk_len and layer < chunk_depth:
# Keep going if we are complementing the void to the next power of 2
key = node + ZERO_HASHES[layer]
if key not in cache:
cache[key] = hash_eth2(key)
node = cache[key]
else:
break
else:
key = merkleized_result_per_layers[layer] + node
if key not in cache:
cache[key] = hash_eth2(key)
node = cache[key]
layer += 1
merkleized_result_per_layers[layer] = node
# Merge in leaf by leaf.
for leaf_index in range(chunk_len):
merge(chunks[leaf_index], leaf_index)
# Complement with 0 if empty, or if not the right power of 2
if 1 << chunk_depth != chunk_len:
merge(ZERO_HASHES[0], chunk_len)
# The next power of two may be smaller than the ultimate virtual size,
# complement with zero-hashes at each depth.
for layer in range(chunk_depth, max_depth):
key = merkleized_result_per_layers[layer] + ZERO_HASHES[layer]
if key not in cache:
cache[key] = hash_eth2(
merkleized_result_per_layers[layer] + ZERO_HASHES[layer]
)
merkleized_result_per_layers[layer + 1] = cache[key]
root = merkleized_result_per_layers[max_depth]
return root, cache
def merklize_elements(elements: Sequence[ProofElement]) -> Hash32:
"""
Given a set of `ProofElement` compute the `hash_tree_root`.
This also verifies that the proof is both "well-formed" and "minimal".
"""
elements_by_depth = groupby(operator.attrgetter("depth"), elements)
max_depth = max(elements_by_depth.keys())
for depth in range(max_depth, 0, -1):
try:
elements_at_depth = sorted(elements_by_depth.pop(depth))
except KeyError:
continue
# Verify that all of the paths at this level are unique
paths = set(el.path for el in elements_at_depth)
if len(paths) != len(elements_at_depth):
raise BrokenTree(
f"Duplicate paths detected: depth={depth} elements={elements_at_depth}"
)
sibling_pairs = tuple(
(left, right)
for left, right in sliding_window(2, elements_at_depth)
if left.path[:-1] == right.path[:-1])
# Check to see if any of the elements didn't have a sibling which
# indicates either a missing sibling, or a duplicate node.
orphans = set(elements_at_depth).difference(
itertools.chain(*sibling_pairs))
if orphans:
raise BrokenTree(
f"Orphaned tree elements: dept={depth} orphans={orphans}")
parents = tuple(
ProofElement(path=left.path[:-1],
value=hash_eth2(left.value + right.value))
for left, right in sibling_pairs)
if not elements_by_depth and len(parents) == 1:
return parents[0].value
else:
elements_by_depth.setdefault(depth - 1, [])
elements_by_depth[depth - 1].extend(parents)
else:
raise BrokenTree("Unable to fully collapse tree within 32 rounds")
def generate_chunk_tree_padding(
unpadded_chunk_tree: PVector[Hash32],
chunk_count: Optional[int]) -> Generator[Hash32, None, None]:
if chunk_count is None:
return
num_chunks = len(unpadded_chunk_tree[0])
if num_chunks > chunk_count:
raise ValueError(
f"Number of chunks in tree ({num_chunks}) exceeds chunk count {chunk_count}"
)
num_existing_layers = len(unpadded_chunk_tree)
num_target_layers = get_next_power_of_two(chunk_count).bit_length()
previous_root = unpadded_chunk_tree[-1][0]
for previous_layer_index in range(num_existing_layers - 1,
num_target_layers - 1):
next_root = hash_eth2(previous_root +
ZERO_HASHES[previous_layer_index])
yield pvector([next_root])
previous_root = next_root
def recompute_hash_in_tree(hash_tree: RawHashTree, layer_index: int,
hash_index: int) -> RawHashTree:
if layer_index == 0:
raise ValueError(
"Cannot recompute hash in leaf layer as it consists of chunks not hashes"
)
child_layer_index = layer_index - 1
left_child_hash_index = hash_index * 2
right_child_hash_index = left_child_hash_index + 1
child_layer = hash_tree[child_layer_index]
left_child_hash = child_layer[left_child_hash_index]
try:
right_child_hash = child_layer[right_child_hash_index]
except IndexError:
right_child_hash = ZERO_HASHES[child_layer_index]
# create the layer if it doesn't exist yet (otherwise, pyrsistent would create a PMap)
if layer_index == len(hash_tree):
hash_tree = hash_tree.append(pvector())
parent_hash = hash_eth2(left_child_hash + right_child_hash)
return hash_tree.transform((layer_index, hash_index), parent_hash)
def test_merkle_hash(value, expected):
assert merkle_hash(value) == hash_eth2(expected)
_type_2 = SSZType2(_type_1_a.copy(), [_type_1_a.copy(), _type_1_b.copy()])
_type_3 = SSZType3(1, 2, 3)
_type_undeclared_fields = SSZUndeclaredFieldsType()
@pytest.mark.parametrize(
'value,sedes',
(
(_type_1_a, SSZType1),
(_type_1_b, SSZType1),
(_type_2, SSZType2),
),
)
def test_serializables(value, sedes):
assert len(hash_tree_root(value, sedes)) == 32
# Also make sure `infer_sedes` works
assert len(hash_tree_root(value)) == 32
@pytest.mark.parametrize(
'value,sedes,expected',
(
(_type_3, SSZType3, hash_tree_root(_type_3, SSZType4)),
(_type_undeclared_fields, SSZUndeclaredFieldsType, hash_eth2(b'')),
),
)
def test_special_serializables(value, sedes, expected):
assert hash_tree_root(value, sedes) == expected
assert hash_tree_root(value) == expected
def intermediate_tree_hash(self, value: TSerializable) -> bytes:
serialized = self.serialize(value)
if self.length <= 32:
return serialized
else:
return hash_eth2(serialized)
from eth_typing import Hash32
from eth_utils.toolz import iterate, take
from ssz.hash import hash_eth2
CHUNK_SIZE = 32 # named BYTES_PER_CHUNK in the spec
EMPTY_CHUNK = Hash32(b"\x00" * CHUNK_SIZE)
SIGNATURE_FIELD_NAME = "signature"
# number of bytes for a serialized offset
OFFSET_SIZE = 4
FIELDS_META_ATTR = "fields"
ZERO_BYTES32 = Hash32(b"\x00" * 32)
MAX_ZERO_HASHES_LAYER = 100
ZERO_HASHES = tuple(
take(
MAX_ZERO_HASHES_LAYER,
iterate(lambda child: hash_eth2(child + child), ZERO_BYTES32),
))
BASE_TYPES = (int, bytes, bool)
def intermediate_tree_hash(self, value: BytesOrByteArray) -> bytes:
return hash_eth2(self.serialize(value))
@given(st.binary())
def test_pack_bytes(data):
byte_list = tuple(bytes([byte]) for byte in data)
assert pack_bytes(data) == pack(byte_list)
@pytest.mark.parametrize(("chunks", "root"), (
(
(A_CHUNK,),
A_CHUNK,
),
(
(A_CHUNK, B_CHUNK),
hash_eth2(A_CHUNK + B_CHUNK),
),
(
(A_CHUNK, B_CHUNK, C_CHUNK),
hash_eth2(hash_eth2(A_CHUNK + B_CHUNK) + hash_eth2(C_CHUNK + EMPTY_CHUNK)),
),
(
(A_CHUNK, B_CHUNK, C_CHUNK, D_CHUNK),
hash_eth2(hash_eth2(A_CHUNK + B_CHUNK) + hash_eth2(C_CHUNK + D_CHUNK)),
),
))
def test_merkleize(chunks, root):
assert merkleize(chunks) == root
@pytest.mark.parametrize(("root", "length", "result"), (
(
(
b'\x01',
b'\x01',
b'\x01',
),
b'\x01\x01\x01' + b'\x00' * 125 + int(3).to_bytes(32, "little"),
),
# two items in one chunk
(
(
b'\x55' * 64,
b'\x66' * 64,
b'\x77' * 64,
),
(hash_eth2(b'\x55' * 64 + b'\x66' * 64 + b'\x77' * 64 +
b'\x00' * 64) + int(3).to_bytes(32, "little")),
),
(
(b'\x55' * 96, b'\x66' * 96, b'\x77' * 96, b'\x88' * 96),
(hash_eth2(
(hash_eth2(b'\x55' * 96 + b'\x00' * 32 + b'\x66' * 96 +
b'\x00' * 32) +
hash_eth2(b'\x77' * 96 + b'\x00' * 32 + b'\x88' * 96 +
b'\x00' * 32))) + int(4).to_bytes(32, "little")),
),
),
)
def test_merkle_hash(value, expected):
assert merkle_hash(value) == hash_eth2(expected)
def intermediate_tree_hash(self, value: TAnyTypedDict) -> bytes:
field_hashes = [
field_sedes.intermediate_tree_hash(value[field_name])
for field_name, field_sedes in self.fields
]
return hash_eth2(b"".join(field_hashes))
def test_proof_get_minimal_proof_elements_mixed_depths():
r"""
0: X
/ \
/ \
1: 0 1
/ \ (length)
/ \
/ \
/ \
/ \
/ \
/ \
2: 0 1
/ \ / \
/ \ / \
/ \ / \
3: 0 1 0 1
/ \ / \ / \ / \
/ \ / \ / \ / \
4: 0 1 0 1 X X 0 1
/ \ / \ / \ / \ / \ / \ / \ / \
5: 0 1 X X 0 1 0 1 X X X X 0 1 X X
Nodes marked with `X` have already been collapsed
Tests:
A: |<--------->|
B: |<----------------------->|
"""
full_proof = compute_proof(CONTENT_512, sedes=short_content_sedes)
section_a = full_proof.get_elements_under(p(0, 0, 0, 0))
section_b = full_proof.get_minimal_proof_elements(2, 2)
section_c = full_proof.get_elements_under(p(0, 0, 1))
section_d = full_proof.get_minimal_proof_elements(8, 4)
section_e = full_proof.get_elements_under(p(0, 1, 1, 0))
section_f = full_proof.get_minimal_proof_elements(14, 2)
section_g = (full_proof.get_element((True, )), )
sparse_elements = sum(
(
section_a,
section_b,
section_c,
section_d,
section_e,
section_f,
section_g,
),
(),
)
sparse_proof = Proof(
elements=sparse_elements,
sedes=short_content_sedes,
)
validate_proof(sparse_proof)
# spans nodes at different levels
elements_a = sparse_proof.get_minimal_proof_elements(0, 4)
assert len(elements_a) == 1
assert elements_a[0].path == p(0, 0, 0)
assert elements_a[0].value == hash_eth2(
hash_eth2(CONTENT_512[0:64]) + hash_eth2(CONTENT_512[64:128]))
elements_b = sparse_proof.get_minimal_proof_elements(8, 8)
assert len(elements_b) == 1
assert elements_b[0].path == p(0, 1)
hash_010 = hash_eth2(
hash_eth2(CONTENT_512[256:320]) + hash_eth2(CONTENT_512[320:384]))
hash_011 = hash_eth2(
hash_eth2(CONTENT_512[384:448]) + hash_eth2(CONTENT_512[448:512]))
assert elements_b[0].value == hash_eth2(hash_010 + hash_011)
def test_proof_get_minimal_proof_elements_all_leaves():
r"""
0: X
/ \
/ \
1: 0 1
/ \ (length)
/ \
/ \
/ \
/ \
/ \
/ \
2: 0 1
/ \ / \
/ \ / \
/ \ / \
3: 0 1 0 1
/ \ / \ / \ / \
/ \ / \ / \ / \
4: 0 1 0 1 0 1 0 1
/ \ / \ / \ / \ / \ / \ / \ / \
5: 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1
Tests:
A: |-|
B: |-|
C: |-|
D: |<--->|
E: |<--->|
F: |<->|
G: |<----->|
H: |<----->|
I: |<----------------------->|
"""
proof = compute_proof(CONTENT_512, sedes=short_content_sedes)
# Just a single node
elements_a = proof.get_minimal_proof_elements(0, 1)
assert len(elements_a) == 1
assert elements_a[0].path == p(0, 0, 0, 0, 0)
assert elements_a[0].value == CONTENT_512[:32]
elements_b = proof.get_minimal_proof_elements(1, 1)
assert len(elements_b) == 1
assert elements_b[0].path == p(0, 0, 0, 0, 1)
assert elements_b[0].value == CONTENT_512[32:64]
elements_c = proof.get_minimal_proof_elements(7, 1)
assert len(elements_c) == 1
assert elements_c[0].path == p(0, 0, 1, 1, 1)
assert elements_c[0].value == CONTENT_512[224:256]
# pair of sibling nodes
elements_d = proof.get_minimal_proof_elements(0, 2)
assert len(elements_d) == 1
assert elements_d[0].path == p(0, 0, 0, 0)
assert elements_d[0].value == hash_eth2(CONTENT_512[0:64])
elements_e = proof.get_minimal_proof_elements(4, 2)
assert len(elements_e) == 1
assert elements_e[0].path == p(0, 0, 1, 0)
assert elements_e[0].value == hash_eth2(CONTENT_512[128:192])
# pair of non-sibling nodes
elements_f = proof.get_minimal_proof_elements(1, 2)
assert len(elements_f) == 2
assert elements_f[0].path == p(0, 0, 0, 0, 1)
assert elements_f[0].value == CONTENT_512[32:64]
assert elements_f[1].path == p(0, 0, 0, 1, 0)
assert elements_f[1].value == CONTENT_512[64:96]
# disjoint sets of three
elements_g = proof.get_minimal_proof_elements(0, 3)
assert len(elements_g) == 2
assert elements_g[0].path == p(0, 0, 0, 0)
assert elements_g[0].value == hash_eth2(CONTENT_512[0:64])
assert elements_g[1].path == p(0, 0, 0, 1, 0)
assert elements_g[1].value == CONTENT_512[64:96]
elements_h = proof.get_minimal_proof_elements(1, 3)
assert len(elements_h) == 2
assert elements_h[0].path == p(0, 0, 0, 0, 1)
assert elements_h[0].value == CONTENT_512[32:64]
assert elements_h[1].path == p(0, 0, 0, 1)
assert elements_h[1].value == hash_eth2(CONTENT_512[64:128])
# span of 8
elements_i = proof.get_minimal_proof_elements(1, 8)
assert len(elements_i) == 4
assert elements_i[0].path == p(0, 0, 0, 0, 1)
assert elements_i[0].value == CONTENT_512[32:64]
assert elements_i[1].path == p(0, 0, 0, 1)
assert elements_i[1].value == hash_eth2(CONTENT_512[64:128])
assert elements_i[2].path == p(0, 0, 1)
assert elements_i[2].value == hash_eth2(
hash_eth2(CONTENT_512[128:192]) + hash_eth2(CONTENT_512[192:256]))
assert elements_i[3].path == p(0, 1, 0, 0, 0)
assert elements_i[3].value == CONTENT_512[256:288]
def mix_in_length(root: Hash32, length: int) -> Hash32:
return hash_eth2(root + length.to_bytes(CHUNK_SIZE, "little"))
bytes16 = ByteVector(16)
EMPTY_BYTES = b"\x00" * 16
A_BYTES = b"\xaa" * 16
B_BYTES = b"\xbb" * 16
C_BYTES = b"\xcc" * 16
D_BYTES = b"\xdd" * 16
E_BYTES = b"\xee" * 16
@pytest.mark.parametrize(("serialized_uints128", "result"), (
((), EMPTY_CHUNK),
((A_BYTES, ), A_BYTES + EMPTY_BYTES),
((A_BYTES, B_BYTES), A_BYTES + B_BYTES),
(
(A_BYTES, B_BYTES, C_BYTES),
hash_eth2(A_BYTES + B_BYTES + C_BYTES + EMPTY_BYTES),
),
(
(A_BYTES, B_BYTES, C_BYTES, D_BYTES, E_BYTES),
hash_eth2(
hash_eth2(A_BYTES + B_BYTES + C_BYTES + D_BYTES) +
hash_eth2(E_BYTES + 3 * EMPTY_BYTES)),
),
))
def test_vector_of_basics(serialized_uints128, result):
sedes = Vector(uint128, len(serialized_uints128))
int_values = tuple(
ssz.decode(value, uint128) for value in serialized_uints128)
assert ssz.hash_tree_root(int_values, sedes) == result