def _get_partition_indices(encoded_sizes: Sequence[int],
max_payload_size: int) -> Iterable[int]:
cumulative_sizes = accumulate(operator.add, encoded_sizes)
offset = 0
yield 0
for idx, (last_size,
size) in enumerate(sliding_window(2, cons(0, cumulative_sizes))):
if size - offset > max_payload_size:
offset = last_size
yield idx
yield len(encoded_sizes)
def get_cmd_offsets(protocol_types: Sequence[Type[ProtocolAPI]]) -> Tuple[int, ...]:
"""
Computes the `command_id_offsets` for each protocol. The first offset is
always P2P_PROTOCOL_COMMAND_LENGTH since the first protocol always begins
after the base `p2p` protocol. Each subsequent protocol is the accumulated
sum of all of the protocol offsets that came before it.
"""
return tuple(accumulate(
lambda prev_offset, protocol_class: prev_offset + protocol_class.cmd_length,
protocol_types,
P2P_PROTOCOL_COMMAND_LENGTH,
))[:-1] # the `[:-1]` is to discard the last accumulated offset which is not needed
def serialize(self, value: TSerializable) -> bytes:
self._validate_serializable(value)
if not len(value):
return b""
pairs = self._get_item_sedes_pairs(value) # slow
element_sedes = tuple(sedes for element, sedes in pairs)
has_fixed_size_section_length_cache = hasattr(
value, "_fixed_size_section_length_cache")
if has_fixed_size_section_length_cache:
if value._fixed_size_section_length_cache is None:
fixed_size_section_length = _compute_fixed_size_section_length(
element_sedes)
value._fixed_size_section_length_cache = fixed_size_section_length
else:
fixed_size_section_length = value._fixed_size_section_length_cache
else:
fixed_size_section_length = _compute_fixed_size_section_length(
element_sedes)
variable_size_section_parts = tuple(
sedes.serialize(item) # slow
for item, sedes in pairs if not sedes.is_fixed_sized)
if variable_size_section_parts:
offsets = tuple(
accumulate(
operator.add,
map(len, variable_size_section_parts[:-1]),
fixed_size_section_length,
))
else:
offsets = ()
offsets_iter = iter(offsets)
fixed_size_section_parts = tuple(
sedes.serialize(item) # slow
if sedes.is_fixed_sized else encode_offset(next(offsets_iter))
for item, sedes in pairs)
try:
next(offsets_iter)
except StopIteration:
pass
else:
raise DeserializationError(
"Did not consume all offsets while decoding value")
return b"".join(
concatv(fixed_size_section_parts, variable_size_section_parts))
def test_headerdb_get_score_for_non_genesis_headers(headerdb, genesis_header):
headerdb.persist_header(genesis_header)
headers = mk_header_chain(genesis_header, length=10)
difficulties = tuple(h.difficulty for h in headers)
scores = tuple(accumulate(operator.add, difficulties, genesis_header.difficulty))
headerdb.persist_header_chain(headers)
for header, expected_score in zip(headers, scores[1:]):
actual_score = headerdb.get_score(header.hash)
assert actual_score == expected_score
def test_ephemeral_db_fuzz(capacity):
center_id = 0
keys = tuple(b'key/%d' % idx for idx in range(50))
values = tuple('0' * idx for idx in range(1, 51))
items = list(zip(keys, values))
lookup = dict(items)
sorted_items = list(
sorted(
items,
key=lambda item: compute_distance(center_id,
content_key_to_node_id(item[0])),
))
cumulative_sizes = tuple(
accumulate(operator.add, (len(item[1]) for item in sorted_items)))
cutoff_index = bisect.bisect_left(cumulative_sizes, capacity)
remaining_capacity = capacity - cumulative_sizes[cutoff_index - 1]
sorted_keys = tuple(key for key, value in sorted_items)
expected_keys = sorted_keys[:cutoff_index]
expected_evicted_keys = sorted_keys[cutoff_index:]
random.shuffle(items)
db = EphemeralDB(
capacity=capacity,
distance_fn=lambda key: compute_distance(center_id,
content_key_to_node_id(key)),
)
for key, value in items:
db.set(key, value)
for key in expected_keys:
assert db.has(key)
for key in expected_evicted_keys:
# we can end up with extra keys if the database has extra capacity that
# don't end up getting evicted.
if len(lookup[key]) <= remaining_capacity:
continue
assert not db.has(key)
def assert_is_canonical_chain(headerdb, headers):
if not headers:
return
# verify that the HEAD is correctly set.
head = headerdb.get_canonical_head()
assert_headers_eq(head, headers[-1])
# verify that each header is set as the canonical block.
for header in headers:
canonical = headerdb.get_canonical_block_header_by_number(header.block_number)
assert canonical == header
# verify difficulties are correctly set.
base_header = headerdb.get_block_header_by_hash(headers[0].parent_hash)
difficulties = tuple(h.difficulty for h in headers)
scores = tuple(accumulate(operator.add, difficulties, base_header.difficulty))
for header, expected_score in zip(headers, scores[1:]):
actual_score = headerdb.get_score(header.hash)
assert actual_score == expected_score
def deserialize_paths(data: bytes) -> Tuple[TreePath, ...]:
# Parse the remaining data ase the encoded paths. The `[1:]` at the
# end here is just an implementation detail for how `accumulate` works,
# resulting in an extra element that isn't needed.
return tuple(accumulate(deserialize_path, partition_leb128(data), ()))[1:]
