def handle_ATOMIC_BATCH(self, sock: BufferedSocket) -> None: kv_pair_and_delete_count_data = sock.read_exactly(DOUBLE_LEN_BYTES) kv_pair_count, delete_count = struct.unpack( '<II', kv_pair_and_delete_count_data) total_kv_count = 2 * kv_pair_count if kv_pair_count or delete_count: kv_and_delete_sizes_data = sock.read_exactly( DOUBLE_LEN_BYTES * kv_pair_count + LEN_BYTES * delete_count) fmt_str = '<' + 'I' * (total_kv_count + delete_count) kv_and_delete_sizes = struct.unpack(fmt_str, kv_and_delete_sizes_data) kv_sizes = kv_and_delete_sizes[:total_kv_count] delete_sizes = kv_and_delete_sizes[total_kv_count:total_kv_count + delete_count] with self.db.atomic_batch() as batch: for key_size, value_size in partition(2, kv_sizes): combined_size = key_size + value_size key_and_value_data = sock.read_exactly(combined_size) key = key_and_value_data[:key_size] value = key_and_value_data[key_size:] batch[key] = value for key_size in delete_sizes: key = sock.read_exactly(key_size) del batch[key] sock.sendall(SUCCESS_BYTE)
def hash_layer(child_layer: Sequence[bytes]) -> Tuple[Hash32, ...]: if len(child_layer) % 2 != 0: raise ValueError("Layer must have an even number of elements") child_pairs = partition(2, child_layer) parent_layer = tuple( hash_eth2(left_child + right_child) for left_child, right_child in child_pairs) return parent_layer
def mset(self, *args: Union[int, Hash32]) -> "HashTree": if len(args) % 2 != 0: raise TypeError( f"mset must be called with an even number of arguments, got {len(args)}" ) evolver = self.evolver() for index, value in partition(2, args): evolver[index] = value return evolver.persistent()
def nibbles_to_bytes(nibbles): if any(nibble not in VALID_NIBBLES for nibble in nibbles): raise InvalidNibbles( "Nibbles contained invalid value. Must be constrained between [0, 15]" ) if len(nibbles) % 2: raise InvalidNibbles("Nibbles must be even in length") value = bytes(REVERSE_NIBBLES_LOOKUP[pair] for pair in partition(2, nibbles)) return value
def hash_layer(child_layer: RawHashTreeLayer, layer_index: int) -> RawHashTreeLayer: if len(child_layer) % 2 == 0: padded_child_layer = child_layer else: padded_child_layer = child_layer.append(ZERO_HASHES[layer_index]) child_pairs = partition(2, padded_child_layer) parent_layer = pvector( hash_eth2(left_child + right_child) for left_child, right_child in child_pairs) return parent_layer
def _attach_committees_to_block_tree(state, block_tree, committees_by_slot, config, forking_asymmetry): for level, committees in zip(_iter_block_tree_by_slot(block_tree), committees_by_slot): block_count = len(level) partitions = partition(block_count, committees) for block, committee in zip(_iter_block_level_by_block(level), partitions): if forking_asymmetry: if random.choice([True, False]): # random drop out continue _attach_committee_to_block(block, first(committee))
def get_appended_chunks( *, appended_elements: Sequence[bytes], element_size: int, num_padding_elements: int) -> Generator[Hash32, None, None]: """Get the sequence of appended chunks.""" if len(appended_elements) <= num_padding_elements: return elements_per_chunk = CHUNK_SIZE // element_size chunk_partitioned_elements = partition( elements_per_chunk, appended_elements[num_padding_elements:], pad=b"\x00" * element_size, ) for elements_in_chunk in chunk_partitioned_elements: yield Hash32(b"".join(elements_in_chunk))
def _deserialize_stream(self, stream: IO[bytes]) -> Iterable[TDeserialized]: if self.element_sedes.is_fixed_sized: element_size = self.element_sedes.get_fixed_size() data = stream.read() if len(data) % element_size != 0: raise DeserializationError( f"Invalid length. List is comprised of a fixed size sedes " f"but total serialized data is not an even multiple of the " f"element size. data length: {len(data)} element size: " f"{element_size}" ) for segment in partition(element_size, data): yield self.element_sedes.deserialize(segment) else: stream_zero_loc = stream.tell() try: first_offset = s_decode_offset(stream) except DeserializationError: if stream.tell() == stream_zero_loc: # Empty list return else: raise num_remaining_offset_bytes = first_offset - stream.tell() if num_remaining_offset_bytes % OFFSET_SIZE != 0: raise DeserializationError( f"Offset bytes was not a multiple of {OFFSET_SIZE}. Got " f"{num_remaining_offset_bytes}" ) num_remaining_offsets = num_remaining_offset_bytes // OFFSET_SIZE tail_offsets = tuple(s_decode_offset(stream) for _ in range(num_remaining_offsets)) offsets = tuple(cons(first_offset, tail_offsets)) for left_offset, right_offset in sliding_window(2, offsets): element_length = right_offset - left_offset element_data = read_exact(element_length, stream) yield self.element_sedes.deserialize(element_data) # simply reading to the end of the current stream gives us all of the final element data final_element_data = stream.read() yield self.element_sedes.deserialize(final_element_data)
def pack(serialized_values: Sequence[bytes]) -> Tuple[Hash32, ...]: if len(serialized_values) == 0: return (EMPTY_CHUNK, ) item_size = len(serialized_values[0]) items_per_chunk = get_items_per_chunk(item_size) number_of_items = len(serialized_values) number_of_chunks = (number_of_items + (items_per_chunk - 1)) // items_per_chunk chunk_partitions = partition(items_per_chunk, serialized_values, pad=b"") chunks_unpadded = (b"".join(chunk_partition) for chunk_partition in chunk_partitions) full_chunks = tuple( Hash32(chunk) for chunk in take(number_of_chunks - 1, chunks_unpadded)) last_chunk = first(chunks_unpadded) if len(tuple(chunks_unpadded)) > 0: raise Exception("Invariant: all chunks have been taken") return full_chunks + (Hash32(last_chunk.ljust(CHUNK_SIZE, b"\x00")), )