def _detail_dictionary(worker: AbstractWorker, my_dict: Tuple) -> Dict:
"""
This function is designed to operate in the opposite direction of
_simplify_dictionary. It takes a dictionary of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
Args:
worker: the worker doing the deserialization
my_dict (Tuple): a simplified dictionary of simple python objects (including binary).
Returns:
Dict: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
pieces = {}
# for dictionaries we want to detail both the key and the value
for key, value in my_dict:
detailed_key = serde._detail(worker, key)
if isinstance(detailed_key, bytes):
detailed_key = detailed_key.decode("utf-8")
detailed_value = serde._detail(worker, value)
if isinstance(detailed_value, bytes):
detailed_value = detailed_value.decode("utf-8")
pieces[detailed_key] = detailed_value
return pieces
def _detail_collection_set(worker: AbstractWorker,
my_collection: Tuple) -> Collection:
"""
This function is designed to operate in the opposite direction of
_simplify_collection. It takes a tuple of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
Args:
worker: the worker doing the deserialization
my_collection (Tuple): a tuple of simple python objects (including binary).
Returns:
Collection: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
pieces = list()
# Step 1: deserialize each part of the collection
for part in my_collection:
try:
pieces.append(serde._detail(
worker,
part).decode("utf-8")) # transform bytes back to string
except AttributeError:
pieces.append(serde._detail(worker, part))
return set(pieces)
def _detail_ndarray(
worker: AbstractWorker,
arr_representation: Tuple[bin, Tuple, str]) -> numpy.ndarray:
"""
This function reconstruct a numpy array from it's byte data, the shape and the dtype
by first loading the byte data with the appropiate dtype and then reshaping it into the
original shape
Args:
worker: the worker doing the deserialization
arr_representation (tuple): a tuple holding the byte representation, shape
and dtype of the array
Returns:
numpy.ndarray: a numpy array
Examples:
arr = _detail_ndarray(arr_representation)
"""
arr_shape = serde._detail(worker, arr_representation[1])
arr_dtype = serde._detail(worker, arr_representation[2])
res = numpy.frombuffer(arr_representation[0],
dtype=arr_dtype).reshape(arr_shape)
assert type(res) == numpy.ndarray
return res
def _detail_collection_tuple(worker: AbstractWorker, my_tuple: Tuple) -> Tuple:
"""
This function is designed to operate in the opposite direction of
_simplify_collection. It takes a tuple of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
This is only applicable to tuples. They need special handling because
`msgpack` is encoding a tuple as a list.
Args:
worker: the worker doing the deserialization
my_tuple (Tuple): a collection of simple python objects (including binary).
Returns:
tuple: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
pieces = list()
# Step 1: deserialize each part of the collection
for part in my_tuple:
pieces.append(serde._detail(worker, part))
return tuple(pieces)
def _detail_collection_set(worker: AbstractWorker,
my_collection: Tuple,
shallow: bool = False) -> Collection:
"""
This function is designed to operate in the opposite direction of
_simplify_collection. It takes a tuple of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
Args:
worker: the worker doing the deserialization
my_collection (Tuple): a tuple of simple python objects (including binary).
Returns:
Collection: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
# Don't detail contents
if shallow:
return set(my_collection)
pieces = list()
# Step 1: deserialize each part of the collection
for part in my_collection:
detailed = serde._detail(worker, part)
pieces.append(detailed)
return set(pieces)
def _detail_numpy_number(
worker: AbstractWorker, nb_representation: Tuple[bin, Tuple, str]
) -> Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]:
"""
This function reconstruct a numpy number from it's byte data, dtype
by first loading the byte data with the appropiate dtype
Args:
worker: the worker doing the deserialization
np_representation (tuple): a tuple holding the byte representation
and dtype of the numpy number
Returns:
numpy.float or numpy.int: a numpy number
Examples:
nb = _detail_numpy_number(nb_representation)
"""
nb_dtype = serde._detail(worker, nb_representation[1])
nb = numpy.frombuffer(nb_representation[0], dtype=nb_dtype)[0]
assert type(nb) in [numpy.float32, numpy.float64, numpy.int32, numpy.int64]
return nb
