def deserialize(self, data: bytes) -> Tuple[bool, ...]:
# Length of data should be larger than 1
if len(data) < 1:
raise DeserializationError(
f"Cannot deserialize empty bytes string as Bitlist[{self.max_bit_count}] "
)
# Last byte in bytes should be >= 1, if not data is not a serialised bitlist.
# len(data) >= 1 and data has a last element that can be compared to 0x00
if data[-1] == 0:
raise DeserializationError(
f"Cannot deserialize bytes data as Bitlist[{self.max_bit_count}] "
f"as last byte is null [{data[len(data) - 1]}]")
as_integer = int.from_bytes(data, "little")
len_value = get_bitlist_len(as_integer)
if len_value > self.max_bit_count:
raise DeserializationError(
f"Cannot deserialize length {len_value} bytes data as Bitlist[{self.max_bit_count}]"
)
for bit_index in range(len_value):
yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def deserialize_segment(self, data, start_index):
"""
Deserialize the data from the given start_index
"""
# Make sure we have sufficient data for inferring length of list
if len(data) < start_index + self.LENGTH_BYTES:
raise DeserializationError(
'Insufficient data: Cannot retrieve the length of list', data)
# Number of bytes of only the list data, excluding the prepended list length
list_length = int.from_bytes(
data[start_index:start_index + self.LENGTH_BYTES], 'big')
list_end_index = start_index + self.LENGTH_BYTES + list_length
# Make sure we have sufficent data for inferring the whole list
if len(data) < list_end_index:
raise DeserializationError(
'Insufficient data: Cannot retrieve the whole list data', data)
deserialized_list = []
# element_start_index is the start index of an element in the serialized bytes string
element_start_index = start_index + self.LENGTH_BYTES
while element_start_index < list_end_index:
element, element_start_index = self.element_sedes.deserialize_segment(
data, element_start_index)
deserialized_list.append(element)
return tuple(deserialized_list), list_end_index
def deserialize(self, data: bytes) -> Optional[TDeserialized]:
try:
signal_byte = data[0]
except IndexError:
raise DeserializationError("Cannot deserialize empty byte string")
if signal_byte == 0:
return None
elif signal_byte == 1:
return self.element_sedes.deserialize(data[1:])
else:
raise DeserializationError(
f"Invalid signal byte: {hex(signal_byte)}")
def read_exact(num_bytes: int, stream: IO[bytes]) -> bytes:
data = stream.read(num_bytes)
if len(data) != num_bytes:
raise DeserializationError(
f"Tried to read {num_bytes}. Only got {len(data)} bytes"
)
return data
def deserialize(self, data):
deserialized_data, end_index = self.deserialize_segment(data, 0)
if end_index != len(data):
raise DeserializationError('Data to be deserialized is too long',
data)
return deserialized_data
def deserialize(self, data: bytes) -> TDeserialized:
stream = io.BytesIO(data)
value = self._deserialize_stream(stream)
extra_data = stream.read()
if extra_data:
raise DeserializationError(
f"Got {len(extra_data)} superfluous bytes")
return value
def deserialize(self, data: bytes) -> bytes:
if len(data) > (self.bit_count + 7) // 8:
raise DeserializationError(
f"Cannot deserialize length {len(data)} bytes data as Bitvector[{self.bit_count}]"
)
for bit_index in range(self.bit_count):
yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def _deserialize_stream_to_tuple(
self, stream: IO[bytes]) -> Iterable[TDeserialized]:
if self.element_sedes.is_fixed_sized:
element_size = self.element_sedes.get_fixed_size()
data = stream.read()
if len(data) % element_size != 0:
raise DeserializationError(
f"Invalid max_length. List is comprised of a fixed size sedes "
f"but total serialized data is not an even multiple of the "
f"element size. data max_length: {len(data)} element size: "
f"{element_size}")
for start_idx in range(0, len(data), element_size):
segment = data[start_idx:start_idx + element_size]
yield self.element_sedes.deserialize(segment)
else:
stream_zero_loc = stream.tell()
try:
first_offset = s_decode_offset(stream)
except DeserializationError:
if stream.tell() == stream_zero_loc:
# Empty list
return
else:
raise
num_remaining_offset_bytes = first_offset - stream.tell()
if num_remaining_offset_bytes % OFFSET_SIZE != 0:
raise DeserializationError(
f"Offset bytes was not a multiple of {OFFSET_SIZE}. Got "
f"{num_remaining_offset_bytes}")
num_remaining_offsets = num_remaining_offset_bytes // OFFSET_SIZE
tail_offsets = tuple(
s_decode_offset(stream) for _ in range(num_remaining_offsets))
offsets = tuple(cons(first_offset, tail_offsets))
for left_offset, right_offset in sliding_window(2, offsets):
element_length = right_offset - left_offset
element_data = read_exact(element_length, stream)
yield self.element_sedes.deserialize(element_data)
# simply reading to the end of the current stream gives us all of the final element data
final_element_data = stream.read()
yield self.element_sedes.deserialize(final_element_data)
def deserialize_content(self, content: bytes) -> bool:
if content == b"\x00":
return False
elif content == b"\x01":
return True
else:
raise DeserializationError(
f"Invalid serialized boolean (must be either 0x01 or 0x00, got "
f"{encode_hex(content)})", )
def deserialize(self, data: bytes) -> bool:
if data == b"\x00":
return False
elif data == b"\x01":
return True
else:
raise DeserializationError(
f"Invalid serialized boolean (must be either 0x01 or 0x00, got "
f"{encode_hex(data)})", )
def deserialize_segment(self, data, start_index):
"""
Deserialize the data from the given start_index
"""
# Make sure we have sufficient data for deserializing
if len(data) + start_index < self.num_bytes:
raise DeserializationError('Insufficient data for deserializing',
data)
end_index = start_index + self.num_bytes
return int.from_bytes(data[start_index:end_index], 'big'), end_index
def deserialize_segment(cls, data, start_index):
"""
Deserialize the data from the given start_index
"""
# Make sure we have sufficient data for inferring length of container
if len(data) < start_index + CONTAINER_PREFIX_LENGTH:
raise DeserializationError(
'Insufficient data: Cannot retrieve the length of container',
data
)
# container_len contains the length of the original container
container_len = int.from_bytes(
data[start_index:start_index + CONTAINER_PREFIX_LENGTH],
'big'
)
container_end_index = start_index + CONTAINER_PREFIX_LENGTH + container_len
# Make sure we have sufficent data for inferring the whole container
if len(data) < container_end_index:
raise DeserializationError(
'Insufficient data: Cannot retrieve the whole container',
data
)
deserialized_field_values = []
field_start_index = start_index + CONTAINER_PREFIX_LENGTH
for _, field_sedes in cls._meta.fields:
field_value, next_field_start_index = field_sedes.deserialize_segment(
data,
field_start_index,
)
deserialized_field_values.append(field_value)
field_start_index = next_field_start_index
if field_start_index != container_end_index:
raise DeserializationError(
'Data to be deserialized is too long',
data
)
return tuple(deserialized_field_values), container_end_index
def deserialize(self, data: bytes) -> Tuple[bool, ...]:
as_integer = int.from_bytes(data, "little")
len_value = get_bitlist_len(as_integer)
if len_value > self.max_bit_count:
raise DeserializationError(
f"Cannot deserialize length {len_value} bytes data as Bitlist[{self.max_bit_count}]"
)
for bit_index in range(len_value):
yield bool((data[bit_index // 8] >> bit_index % 8) % 2)
def deserialize(cls, data, **extra_kwargs):
# deserialized_field_values stores all the field values in tuple format
deserialized_field_values, end_index = cls.deserialize_segment(data, 0)
if end_index != len(data):
raise DeserializationError(
'Data to be deserialized is too long',
data
)
args_as_kwargs = merge_args_to_kwargs(deserialized_field_values, {}, cls._meta.field_names)
return cls(**args_as_kwargs, **extra_kwargs)
def serialize(self, value: TSerializable) -> bytes:
self._validate_serializable(value)
if not len(value):
return b""
pairs = self._get_item_sedes_pairs(value) # slow
element_sedes = tuple(sedes for element, sedes in pairs)
has_fixed_size_section_length_cache = hasattr(
value, "_fixed_size_section_length_cache")
if has_fixed_size_section_length_cache:
if value._fixed_size_section_length_cache is None:
fixed_size_section_length = _compute_fixed_size_section_length(
element_sedes)
value._fixed_size_section_length_cache = fixed_size_section_length
else:
fixed_size_section_length = value._fixed_size_section_length_cache
else:
fixed_size_section_length = _compute_fixed_size_section_length(
element_sedes)
variable_size_section_parts = tuple(
sedes.serialize(item) # slow
for item, sedes in pairs if not sedes.is_fixed_sized)
if variable_size_section_parts:
offsets = tuple(
accumulate(
operator.add,
map(len, variable_size_section_parts[:-1]),
fixed_size_section_length,
))
else:
offsets = ()
offsets_iter = iter(offsets)
fixed_size_section_parts = tuple(
sedes.serialize(item) # slow
if sedes.is_fixed_sized else encode_offset(next(offsets_iter))
for item, sedes in pairs)
try:
next(offsets_iter)
except StopIteration:
pass
else:
raise DeserializationError(
"Did not consume all offsets while decoding value")
return b"".join(
concatv(fixed_size_section_parts, variable_size_section_parts))
def _deserialize_stream(self, stream: IO[bytes]) -> Tuple[Any, ...]:
if not self.field_sedes:
# TODO: likely remove once
# https://github.com/ethereum/eth2.0-specs/issues/854 is resolved
return tuple()
fixed_size_values, offset_pairs = self.deserialize_fixed_size_parts(
stream)
if not offset_pairs:
return fixed_size_values
variable_size_values = self.deserialize_variable_size_parts(
offset_pairs, stream)
fixed_size_parts_iter = iter(fixed_size_values)
variable_size_parts_iter = iter(variable_size_values)
value = tuple(
next(fixed_size_parts_iter) if sedes.
is_fixed_sized else next(variable_size_parts_iter)
for sedes in self.field_sedes)
# Verify that both iterables have been fully consumed.
try:
next(fixed_size_parts_iter)
except StopIteration:
pass
else:
raise DeserializationError("Did not consume all fixed size values")
try:
next(variable_size_parts_iter)
except StopIteration:
pass
else:
raise DeserializationError(
"Did not consume all variable size values")
return value
def deserialize(self, data: bytes) -> TDeserialized:
value, end_index = self.deserialize_segment(data, 0)
num_leftover_bytes = len(data) - end_index
if num_leftover_bytes > 0:
raise DeserializationError(
f"The given string ends with {num_leftover_bytes} superfluous bytes",
data,
)
elif num_leftover_bytes < 0:
raise Exception("Invariant: End index cannot exceed size of data")
return value
def consume_bytes(data: bytes, start_index: int,
num_bytes: int) -> Tuple[bytes, int]:
if start_index < 0:
raise ValueError("Start index must not be negative")
elif num_bytes < 0:
raise ValueError("Number of bytes to read must not be negative")
elif start_index + num_bytes > len(data):
raise DeserializationError(
f"Tried to read {num_bytes} bytes starting at index {start_index} but the string "
f"is only {len(data)} bytes long")
else:
continuation_index = start_index + num_bytes
return data[start_index:start_index +
num_bytes], continuation_index
def deserialize_segment(self, serialized_obj, start_index):
"""
Deserialize the data from the given start_index
"""
# Make sure we have sufficient data for inferring length of list
if len(serialized_obj) < start_index + 1:
raise DeserializationError(
'Insufficient data for deserializing',
serialized_obj
)
# Deal with only the first byte of the whole serialized data
boolean_serialized_obj = serialized_obj[start_index: start_index + 1]
if boolean_serialized_obj == b'\x00':
deserialized_obj = False
elif boolean_serialized_obj == b'\x01':
deserialized_obj = True
else:
raise DeserializationError(
'Invalid serialized boolean. Must be either 0x01 or 0x00',
serialized_obj
)
return deserialized_obj, start_index + 1
def deserialize_segment(self, data, start_index):
"""
Deserialize the data from the given start_index
"""
# Make sure we have sufficient data for inferring length of bytes object
if len(data) < start_index + BYTES_PREFIX_LENGTH:
raise DeserializationError(
'Insufficient data: Cannot retrieve the length of bytes object',
data)
# object_len contains the length of the original bytes object
object_len = int.from_bytes(
data[start_index:start_index + BYTES_PREFIX_LENGTH], 'big')
# object_start_index is the start index of bytes object in the serialized bytes string
object_start_index = start_index + BYTES_PREFIX_LENGTH
object_end_index = object_start_index + object_len
# Make sure we have sufficent data for inferring the whole bytes object
if len(data) < object_end_index:
raise DeserializationError(
'Insufficient data: Cannot retrieve the whole list bytes object',
data)
return data[object_start_index:object_end_index], object_end_index
def deserialize_content(self, content: bytes) -> Generator[S, None, None]:
if self.empty and len(content) > 0:
raise DeserializationError(f"Serialized list is not empty")
element_start_index = 0
while element_start_index < len(content):
element, next_element_start_index = self.element_sedes.deserialize_segment(
content,
element_start_index,
)
if next_element_start_index <= element_start_index:
raise Exception("Invariant: must always make progress")
element_start_index = next_element_start_index
yield element
if element_start_index > len(content):
raise Exception(
"Invariant: must not consume more data than available")
def deserialize_content(
self, content: bytes) -> Generator[Tuple[str, Any], None, None]:
field_start_index = 0
for field_name, field_sedes in self.fields:
field_value, next_field_start_index = field_sedes.deserialize_segment(
content,
field_start_index,
)
yield field_name, field_value
if next_field_start_index <= field_start_index:
raise Exception("Invariant: must always make progress")
field_start_index = next_field_start_index
if field_start_index < len(content):
extra_bytes = len(content) - field_start_index
raise DeserializationError(
f"Serialized container ends with {extra_bytes} extra bytes")
if field_start_index > len(content):
raise Exception(
"Invariant: must not consume more data than available")
def deserialize_content(
self,
content: bytes) -> Generator[TDeserializedElement, None, None]:
element_start_index = 0
for element_index in range(self.length):
element, next_element_start_index = self.element_sedes.deserialize_segment(
content,
element_start_index,
)
if next_element_start_index <= element_start_index:
raise Exception("Invariant: must always make progress")
element_start_index = next_element_start_index
yield element
if element_start_index > len(content):
raise Exception(
"Invariant: must not consume more data than available")
if element_start_index < len(content):
raise DeserializationError(
f"Serialized tuple ends with {len(content) - element_start_index} extra bytes"
)
def deserialize(self, data: bytes) -> Tuple[TDeserialized, ...]:
if data:
raise DeserializationError("Cannot deserialize non-empty bytes using `EmptyList` sedes")
return tuple()
def deserialize(self, data: bytes) -> bytes:
if len(data) != self.length:
raise DeserializationError(
f"Cannot deserialize length {len(data)} data as bytes{self.length}"
)
return data
def deserialize(self, data: bytes) -> bytes:
if len(data) != 1:
raise DeserializationError(
f"The `Byte` sedes can only deserialize single bytes. Got: {data!r}"
)
return data
def deserialize(self, data: bytes) -> int:
if len(data) != self.size:
raise DeserializationError(
f"Cannot deserialize length {len(data)} byte-string as uint{self.size*8}"
)
return int.from_bytes(data, "little")