def _detail_dictionary(worker: AbstractWorker, my_dict: Tuple) -> Dict: """ This function is designed to operate in the opposite direction of _simplify_dictionary. It takes a dictionary of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. Args: worker: the worker doing the deserialization my_dict (Tuple): a simplified dictionary of simple python objects (including binary). Returns: Dict: a collection of the same type as the input where the objects in the collection have been detailed. """ pieces = {} # for dictionaries we want to detail both the key and the value for key, value in my_dict: detailed_key = serde._detail(worker, key) if isinstance(detailed_key, bytes): detailed_key = detailed_key.decode("utf-8") detailed_value = serde._detail(worker, value) if isinstance(detailed_value, bytes): detailed_value = detailed_value.decode("utf-8") pieces[detailed_key] = detailed_value return pieces
def _detail_collection_set(worker: AbstractWorker, my_collection: Tuple) -> Collection: """ This function is designed to operate in the opposite direction of _simplify_collection. It takes a tuple of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. Args: worker: the worker doing the deserialization my_collection (Tuple): a tuple of simple python objects (including binary). Returns: Collection: a collection of the same type as the input where the objects in the collection have been detailed. """ pieces = list() # Step 1: deserialize each part of the collection for part in my_collection: try: pieces.append(serde._detail( worker, part).decode("utf-8")) # transform bytes back to string except AttributeError: pieces.append(serde._detail(worker, part)) return set(pieces)
def _detail_ndarray( worker: AbstractWorker, arr_representation: Tuple[bin, Tuple, str]) -> numpy.ndarray: """ This function reconstruct a numpy array from it's byte data, the shape and the dtype by first loading the byte data with the appropiate dtype and then reshaping it into the original shape Args: worker: the worker doing the deserialization arr_representation (tuple): a tuple holding the byte representation, shape and dtype of the array Returns: numpy.ndarray: a numpy array Examples: arr = _detail_ndarray(arr_representation) """ arr_shape = serde._detail(worker, arr_representation[1]) arr_dtype = serde._detail(worker, arr_representation[2]) res = numpy.frombuffer(arr_representation[0], dtype=arr_dtype).reshape(arr_shape) assert type(res) == numpy.ndarray return res
def _detail_collection_tuple(worker: AbstractWorker, my_tuple: Tuple) -> Tuple: """ This function is designed to operate in the opposite direction of _simplify_collection. It takes a tuple of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. This is only applicable to tuples. They need special handling because `msgpack` is encoding a tuple as a list. Args: worker: the worker doing the deserialization my_tuple (Tuple): a collection of simple python objects (including binary). Returns: tuple: a collection of the same type as the input where the objects in the collection have been detailed. """ pieces = list() # Step 1: deserialize each part of the collection for part in my_tuple: pieces.append(serde._detail(worker, part)) return tuple(pieces)
def _detail_collection_set(worker: AbstractWorker, my_collection: Tuple, shallow: bool = False) -> Collection: """ This function is designed to operate in the opposite direction of _simplify_collection. It takes a tuple of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. Args: worker: the worker doing the deserialization my_collection (Tuple): a tuple of simple python objects (including binary). Returns: Collection: a collection of the same type as the input where the objects in the collection have been detailed. """ # Don't detail contents if shallow: return set(my_collection) pieces = list() # Step 1: deserialize each part of the collection for part in my_collection: detailed = serde._detail(worker, part) pieces.append(detailed) return set(pieces)
def _detail_numpy_number( worker: AbstractWorker, nb_representation: Tuple[bin, Tuple, str] ) -> Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]: """ This function reconstruct a numpy number from it's byte data, dtype by first loading the byte data with the appropiate dtype Args: worker: the worker doing the deserialization np_representation (tuple): a tuple holding the byte representation and dtype of the numpy number Returns: numpy.float or numpy.int: a numpy number Examples: nb = _detail_numpy_number(nb_representation) """ nb_dtype = serde._detail(worker, nb_representation[1]) nb = numpy.frombuffer(nb_representation[0], dtype=nb_dtype)[0] assert type(nb) in [numpy.float32, numpy.float64, numpy.int32, numpy.int64] return nb