def deserialize(self, data: bytes) -> Tuple[bool, ...]: # Length of data should be larger than 1 if len(data) < 1: raise DeserializationError( f"Cannot deserialize empty bytes string as Bitlist[{self.max_bit_count}] " ) # Last byte in bytes should be >= 1, if not data is not a serialised bitlist. # len(data) >= 1 and data has a last element that can be compared to 0x00 if data[-1] == 0: raise DeserializationError( f"Cannot deserialize bytes data as Bitlist[{self.max_bit_count}] " f"as last byte is null [{data[len(data) - 1]}]") as_integer = int.from_bytes(data, "little") len_value = get_bitlist_len(as_integer) if len_value > self.max_bit_count: raise DeserializationError( f"Cannot deserialize length {len_value} bytes data as Bitlist[{self.max_bit_count}]" ) for bit_index in range(len_value): yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def deserialize_segment(self, data, start_index): """ Deserialize the data from the given start_index """ # Make sure we have sufficient data for inferring length of list if len(data) < start_index + self.LENGTH_BYTES: raise DeserializationError( 'Insufficient data: Cannot retrieve the length of list', data) # Number of bytes of only the list data, excluding the prepended list length list_length = int.from_bytes( data[start_index:start_index + self.LENGTH_BYTES], 'big') list_end_index = start_index + self.LENGTH_BYTES + list_length # Make sure we have sufficent data for inferring the whole list if len(data) < list_end_index: raise DeserializationError( 'Insufficient data: Cannot retrieve the whole list data', data) deserialized_list = [] # element_start_index is the start index of an element in the serialized bytes string element_start_index = start_index + self.LENGTH_BYTES while element_start_index < list_end_index: element, element_start_index = self.element_sedes.deserialize_segment( data, element_start_index) deserialized_list.append(element) return tuple(deserialized_list), list_end_index
def deserialize(self, data: bytes) -> Optional[TDeserialized]: try: signal_byte = data[0] except IndexError: raise DeserializationError("Cannot deserialize empty byte string") if signal_byte == 0: return None elif signal_byte == 1: return self.element_sedes.deserialize(data[1:]) else: raise DeserializationError( f"Invalid signal byte: {hex(signal_byte)}")
def read_exact(num_bytes: int, stream: IO[bytes]) -> bytes: data = stream.read(num_bytes) if len(data) != num_bytes: raise DeserializationError( f"Tried to read {num_bytes}. Only got {len(data)} bytes" ) return data
def deserialize(self, data): deserialized_data, end_index = self.deserialize_segment(data, 0) if end_index != len(data): raise DeserializationError('Data to be deserialized is too long', data) return deserialized_data
def deserialize(self, data: bytes) -> TDeserialized: stream = io.BytesIO(data) value = self._deserialize_stream(stream) extra_data = stream.read() if extra_data: raise DeserializationError( f"Got {len(extra_data)} superfluous bytes") return value
def deserialize(self, data: bytes) -> bytes: if len(data) > (self.bit_count + 7) // 8: raise DeserializationError( f"Cannot deserialize length {len(data)} bytes data as Bitvector[{self.bit_count}]" ) for bit_index in range(self.bit_count): yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def _deserialize_stream_to_tuple( self, stream: IO[bytes]) -> Iterable[TDeserialized]: if self.element_sedes.is_fixed_sized: element_size = self.element_sedes.get_fixed_size() data = stream.read() if len(data) % element_size != 0: raise DeserializationError( f"Invalid max_length. List is comprised of a fixed size sedes " f"but total serialized data is not an even multiple of the " f"element size. data max_length: {len(data)} element size: " f"{element_size}") for start_idx in range(0, len(data), element_size): segment = data[start_idx:start_idx + element_size] yield self.element_sedes.deserialize(segment) else: stream_zero_loc = stream.tell() try: first_offset = s_decode_offset(stream) except DeserializationError: if stream.tell() == stream_zero_loc: # Empty list return else: raise num_remaining_offset_bytes = first_offset - stream.tell() if num_remaining_offset_bytes % OFFSET_SIZE != 0: raise DeserializationError( f"Offset bytes was not a multiple of {OFFSET_SIZE}. Got " f"{num_remaining_offset_bytes}") num_remaining_offsets = num_remaining_offset_bytes // OFFSET_SIZE tail_offsets = tuple( s_decode_offset(stream) for _ in range(num_remaining_offsets)) offsets = tuple(cons(first_offset, tail_offsets)) for left_offset, right_offset in sliding_window(2, offsets): element_length = right_offset - left_offset element_data = read_exact(element_length, stream) yield self.element_sedes.deserialize(element_data) # simply reading to the end of the current stream gives us all of the final element data final_element_data = stream.read() yield self.element_sedes.deserialize(final_element_data)
def deserialize_content(self, content: bytes) -> bool: if content == b"\x00": return False elif content == b"\x01": return True else: raise DeserializationError( f"Invalid serialized boolean (must be either 0x01 or 0x00, got " f"{encode_hex(content)})", )
def deserialize(self, data: bytes) -> bool: if data == b"\x00": return False elif data == b"\x01": return True else: raise DeserializationError( f"Invalid serialized boolean (must be either 0x01 or 0x00, got " f"{encode_hex(data)})", )
def deserialize_segment(self, data, start_index): """ Deserialize the data from the given start_index """ # Make sure we have sufficient data for deserializing if len(data) + start_index < self.num_bytes: raise DeserializationError('Insufficient data for deserializing', data) end_index = start_index + self.num_bytes return int.from_bytes(data[start_index:end_index], 'big'), end_index
def deserialize_segment(cls, data, start_index): """ Deserialize the data from the given start_index """ # Make sure we have sufficient data for inferring length of container if len(data) < start_index + CONTAINER_PREFIX_LENGTH: raise DeserializationError( 'Insufficient data: Cannot retrieve the length of container', data ) # container_len contains the length of the original container container_len = int.from_bytes( data[start_index:start_index + CONTAINER_PREFIX_LENGTH], 'big' ) container_end_index = start_index + CONTAINER_PREFIX_LENGTH + container_len # Make sure we have sufficent data for inferring the whole container if len(data) < container_end_index: raise DeserializationError( 'Insufficient data: Cannot retrieve the whole container', data ) deserialized_field_values = [] field_start_index = start_index + CONTAINER_PREFIX_LENGTH for _, field_sedes in cls._meta.fields: field_value, next_field_start_index = field_sedes.deserialize_segment( data, field_start_index, ) deserialized_field_values.append(field_value) field_start_index = next_field_start_index if field_start_index != container_end_index: raise DeserializationError( 'Data to be deserialized is too long', data ) return tuple(deserialized_field_values), container_end_index
def deserialize(self, data: bytes) -> Tuple[bool, ...]: as_integer = int.from_bytes(data, "little") len_value = get_bitlist_len(as_integer) if len_value > self.max_bit_count: raise DeserializationError( f"Cannot deserialize length {len_value} bytes data as Bitlist[{self.max_bit_count}]" ) for bit_index in range(len_value): yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def deserialize(cls, data, **extra_kwargs): # deserialized_field_values stores all the field values in tuple format deserialized_field_values, end_index = cls.deserialize_segment(data, 0) if end_index != len(data): raise DeserializationError( 'Data to be deserialized is too long', data ) args_as_kwargs = merge_args_to_kwargs(deserialized_field_values, {}, cls._meta.field_names) return cls(**args_as_kwargs, **extra_kwargs)
def serialize(self, value: TSerializable) -> bytes: self._validate_serializable(value) if not len(value): return b"" pairs = self._get_item_sedes_pairs(value) # slow element_sedes = tuple(sedes for element, sedes in pairs) has_fixed_size_section_length_cache = hasattr( value, "_fixed_size_section_length_cache") if has_fixed_size_section_length_cache: if value._fixed_size_section_length_cache is None: fixed_size_section_length = _compute_fixed_size_section_length( element_sedes) value._fixed_size_section_length_cache = fixed_size_section_length else: fixed_size_section_length = value._fixed_size_section_length_cache else: fixed_size_section_length = _compute_fixed_size_section_length( element_sedes) variable_size_section_parts = tuple( sedes.serialize(item) # slow for item, sedes in pairs if not sedes.is_fixed_sized) if variable_size_section_parts: offsets = tuple( accumulate( operator.add, map(len, variable_size_section_parts[:-1]), fixed_size_section_length, )) else: offsets = () offsets_iter = iter(offsets) fixed_size_section_parts = tuple( sedes.serialize(item) # slow if sedes.is_fixed_sized else encode_offset(next(offsets_iter)) for item, sedes in pairs) try: next(offsets_iter) except StopIteration: pass else: raise DeserializationError( "Did not consume all offsets while decoding value") return b"".join( concatv(fixed_size_section_parts, variable_size_section_parts))
def _deserialize_stream(self, stream: IO[bytes]) -> Tuple[Any, ...]: if not self.field_sedes: # TODO: likely remove once # https://github.com/ethereum/eth2.0-specs/issues/854 is resolved return tuple() fixed_size_values, offset_pairs = self.deserialize_fixed_size_parts( stream) if not offset_pairs: return fixed_size_values variable_size_values = self.deserialize_variable_size_parts( offset_pairs, stream) fixed_size_parts_iter = iter(fixed_size_values) variable_size_parts_iter = iter(variable_size_values) value = tuple( next(fixed_size_parts_iter) if sedes. is_fixed_sized else next(variable_size_parts_iter) for sedes in self.field_sedes) # Verify that both iterables have been fully consumed. try: next(fixed_size_parts_iter) except StopIteration: pass else: raise DeserializationError("Did not consume all fixed size values") try: next(variable_size_parts_iter) except StopIteration: pass else: raise DeserializationError( "Did not consume all variable size values") return value
def deserialize(self, data: bytes) -> TDeserialized: value, end_index = self.deserialize_segment(data, 0) num_leftover_bytes = len(data) - end_index if num_leftover_bytes > 0: raise DeserializationError( f"The given string ends with {num_leftover_bytes} superfluous bytes", data, ) elif num_leftover_bytes < 0: raise Exception("Invariant: End index cannot exceed size of data") return value
def consume_bytes(data: bytes, start_index: int, num_bytes: int) -> Tuple[bytes, int]: if start_index < 0: raise ValueError("Start index must not be negative") elif num_bytes < 0: raise ValueError("Number of bytes to read must not be negative") elif start_index + num_bytes > len(data): raise DeserializationError( f"Tried to read {num_bytes} bytes starting at index {start_index} but the string " f"is only {len(data)} bytes long") else: continuation_index = start_index + num_bytes return data[start_index:start_index + num_bytes], continuation_index
def deserialize_segment(self, serialized_obj, start_index): """ Deserialize the data from the given start_index """ # Make sure we have sufficient data for inferring length of list if len(serialized_obj) < start_index + 1: raise DeserializationError( 'Insufficient data for deserializing', serialized_obj ) # Deal with only the first byte of the whole serialized data boolean_serialized_obj = serialized_obj[start_index: start_index + 1] if boolean_serialized_obj == b'\x00': deserialized_obj = False elif boolean_serialized_obj == b'\x01': deserialized_obj = True else: raise DeserializationError( 'Invalid serialized boolean. Must be either 0x01 or 0x00', serialized_obj ) return deserialized_obj, start_index + 1
def deserialize_segment(self, data, start_index): """ Deserialize the data from the given start_index """ # Make sure we have sufficient data for inferring length of bytes object if len(data) < start_index + BYTES_PREFIX_LENGTH: raise DeserializationError( 'Insufficient data: Cannot retrieve the length of bytes object', data) # object_len contains the length of the original bytes object object_len = int.from_bytes( data[start_index:start_index + BYTES_PREFIX_LENGTH], 'big') # object_start_index is the start index of bytes object in the serialized bytes string object_start_index = start_index + BYTES_PREFIX_LENGTH object_end_index = object_start_index + object_len # Make sure we have sufficent data for inferring the whole bytes object if len(data) < object_end_index: raise DeserializationError( 'Insufficient data: Cannot retrieve the whole list bytes object', data) return data[object_start_index:object_end_index], object_end_index
def deserialize_content(self, content: bytes) -> Generator[S, None, None]: if self.empty and len(content) > 0: raise DeserializationError(f"Serialized list is not empty") element_start_index = 0 while element_start_index < len(content): element, next_element_start_index = self.element_sedes.deserialize_segment( content, element_start_index, ) if next_element_start_index <= element_start_index: raise Exception("Invariant: must always make progress") element_start_index = next_element_start_index yield element if element_start_index > len(content): raise Exception( "Invariant: must not consume more data than available")
def deserialize_content( self, content: bytes) -> Generator[Tuple[str, Any], None, None]: field_start_index = 0 for field_name, field_sedes in self.fields: field_value, next_field_start_index = field_sedes.deserialize_segment( content, field_start_index, ) yield field_name, field_value if next_field_start_index <= field_start_index: raise Exception("Invariant: must always make progress") field_start_index = next_field_start_index if field_start_index < len(content): extra_bytes = len(content) - field_start_index raise DeserializationError( f"Serialized container ends with {extra_bytes} extra bytes") if field_start_index > len(content): raise Exception( "Invariant: must not consume more data than available")
def deserialize_content( self, content: bytes) -> Generator[TDeserializedElement, None, None]: element_start_index = 0 for element_index in range(self.length): element, next_element_start_index = self.element_sedes.deserialize_segment( content, element_start_index, ) if next_element_start_index <= element_start_index: raise Exception("Invariant: must always make progress") element_start_index = next_element_start_index yield element if element_start_index > len(content): raise Exception( "Invariant: must not consume more data than available") if element_start_index < len(content): raise DeserializationError( f"Serialized tuple ends with {len(content) - element_start_index} extra bytes" )
def deserialize(self, data: bytes) -> Tuple[TDeserialized, ...]: if data: raise DeserializationError("Cannot deserialize non-empty bytes using `EmptyList` sedes") return tuple()
def deserialize(self, data: bytes) -> bytes: if len(data) != self.length: raise DeserializationError( f"Cannot deserialize length {len(data)} data as bytes{self.length}" ) return data
def deserialize(self, data: bytes) -> bytes: if len(data) != 1: raise DeserializationError( f"The `Byte` sedes can only deserialize single bytes. Got: {data!r}" ) return data
def deserialize(self, data: bytes) -> int: if len(data) != self.size: raise DeserializationError( f"Cannot deserialize length {len(data)} byte-string as uint{self.size*8}" ) return int.from_bytes(data, "little")