def _simplify_dictionary(worker: AbstractWorker,
my_dict: Dict,
shallow: bool = False) -> Tuple:
"""
This function is designed to search a dict for any objects
which may need to be simplified (i.e., torch tensors). It iterates
through each key, value in the dict and calls _simplify on it. Finally,
it returns the output tuple of tuples containing key/value pairs. The
reverse function to this function is _detail_dictionary, which undoes
the functionality of this function.
Args:
my_dict: A dictionary of python objects.
Returns:
Tuple: Tuple containing tuples of simplified key/value pairs from the
input dictionary.
"""
pieces = list()
# for dictionaries we want to simplify both the key and the value
for key, value in my_dict.items():
pieces.append(
(serde._simplify(worker, key),
serde._simplify(worker, value) if not shallow else value))
return tuple(pieces)
def test_ellipsis_simplify():
"""Make sure ellipsis simplifies correctly."""
# the id indicating an ellipsis is here
assert _simplify(Ellipsis)[0] == 9
# the simplified ellipsis (empty object)
assert _simplify(Ellipsis)[1] == b""
def test_torch_device_simplify():
"""Test the simplification of torch.device"""
device = torch.device("cpu")
# the id indicating an torch.device is here
assert _simplify(device)[0] == 10
# the simplified torch.device
assert _simplify(device)[1] == "cpu"
def test_set_simplify():
"""This tests our ability to simplify set objects.
This test is pretty simple since sets just serialize to
lists, with a tuple wrapper with the correct ID (3)
for sets so that the detailer knows how to interpret it."""
input = set(["hello", "world"])
target = (4, ["hello", "world"])
assert _simplify(input)[0] == target[0]
assert set(_simplify(input)[1]) == set(target[1])
def _simplify_collection(my_collection: Collection) -> Collection:
"""
This function is designed to search a collection for any objects
which may need to be simplified (i.e., torch tensors). It iterates
through each object in the collection and calls _simplify on it. Finally,
it returns the output collection as the same type as the input collection
so that the consuming serialization step knows the correct type info. The
reverse function to this function is _detail_collection, which undoes
the functionality of this function.
Args:
my_collection (Collection): a collection of python objects
Returns:
Collection: a collection of the same type as the input of simplified
objects.
"""
# Step 0: get collection type for later use and itialize empty list
my_type = type(my_collection)
pieces = list()
# Step 1: serialize each part of the collection
for part in my_collection:
pieces.append(serde._simplify(part))
# Step 2: convert back to original type and return serialization
if my_type == set:
return pieces
return my_type(pieces)
def test_torch_tensor_simplify():
"""This tests our ability to simplify torch.Tensor objects
At the time of writing, tensors simplify to a tuple where the
first value in the tuple is the tensor's ID and the second
value is a serialized version of the Tensor (serialized
by PyTorch's torch.save method)
"""
# create a tensor
input = Tensor(numpy.random.random((100, 100)))
# simplify the tnesor
output = _simplify(input)
# make sure outer type is correct
assert type(output) == tuple
# make sure the object type ID is correct
# (0 for torch.Tensor)
assert output[0] == 0
# make sure inner type is correct
assert type(output[1]) == tuple
# make sure ID is correctly encoded
assert output[1][0] == input.id
# make sure tensor data type is correct
assert type(output[1][1]) == bytes
def _simplify_collection(my_collection: Collection) -> Tuple:
"""
This function is designed to search a collection for any objects
which may need to be simplified (i.e., torch tensors). It iterates
through each object in the collection and calls _simplify on it. Finally,
it returns the output as the tuple of simplified items of the input collection.
This function is used to simplify list, set, and tuple. The reverse function,
which undoes the functionality of this function is different for each of these types:
_detail_collection_list, _detail_collection_set, _detail_collection_tuple.
Args:
my_collection (Collection): a collection of python objects
Returns:
Tuple: a tuple with simplified objects.
"""
# Step 0: initialize empty list
pieces = list()
# Step 1: serialize each part of the collection
for part in my_collection:
pieces.append(serde._simplify(part))
# Step 2: return serialization as tuple of simplified items
return tuple(pieces)
def test_float_simplify():
"""This tests our ability to simplify float objects.
This test is pretty simple since floats just serialize to
themselves, with no tuple/id necessary."""
input = 5.6
target = 5.6
assert serde._simplify(input) == target
def test_int_simplify():
"""This tests our ability to simplify int objects.
This test is pretty simple since ints just serialize to
themselves, with no tuple/id necessary."""
input = 5
target = 5
assert _simplify(input) == target
def test_string_simplify():
"""This tests our ability to simplify string objects.
This test is pretty simple since strings just serialize to
themselves, with no tuple/id necessary."""
input = "hello"
target = "hello"
assert _simplify(input) == target
def test_dict_simplify():
"""This tests our ability to simplify dict objects.
This test is pretty simple since dicts just serialize to
themselves, with a tuple wrapper with the correct ID (4)
for dicts so that the detailer knows how to interpret it."""
input = {"hello": "world"}
target = (5, {"hello": "world"})
assert _simplify(input) == target
def test_list_simplify():
"""This tests our ability to simplify list types.
This test is pretty simple since lists just serialize to
themselves, with a tuple wrapper with the correct ID (2)
for lists so that the detailer knows how to interpret it."""
input = ["hello", "world"]
target = (3, ["hello", "world"])
assert _simplify(input) == target
def test_tuple_simplify():
"""This tests our ability to simplify tuple types.
This test is pretty simple since tuples just serialize to
themselves, with a tuple wrapper with the correct ID (1)
for tuples so that the detailer knows how to interpret it."""
input = ("hello", "world")
target = (2, ("hello", "world"))
assert _simplify(input) == target
def test_range_simplify():
"""This tests our ability to simplify range objects.
This test is pretty simple since range objs just serialize to
themselves, with a tuple wrapper with the correct ID (5)
for dicts so that the detailer knows how to interpret it."""
input = range(1, 3, 4)
target = (6, (1, 3, 4))
assert _simplify(input) == target
def test_pointer_tensor_simplify():
"""Test the simplification of PointerTensor"""
alice = syft.VirtualWorker(syft.torch.hook, id="alice")
input_tensor = PointerTensor(id=1000, location=alice, owner=alice)
output = _simplify(input_tensor)
assert output[1][0] == input_tensor.id
assert output[1][1] == input_tensor.id_at_location
assert output[1][2] == input_tensor.owner.id
def _simplify_ndarray(worker: AbstractWorker,
my_array: numpy.ndarray) -> Tuple[bin, Tuple, Tuple]:
"""
This function gets the byte representation of the array
and stores the dtype and shape for reconstruction
Args:
my_array (numpy.ndarray): a numpy array
Returns:
list: a list holding the byte representation, shape and dtype of the array
Examples:
arr_representation = _simplify_ndarray(numpy.random.random([1000, 1000])))
"""
arr_bytes = my_array.tobytes()
arr_shape = serde._simplify(worker, my_array.shape)
arr_dtype = serde._simplify(worker, my_array.dtype.name)
return (arr_bytes, arr_shape, arr_dtype)
def _simplify_dictionary(my_dict: Dict) -> Dict:
"""
This function is designed to search a dict for any objects
which may need to be simplified (i.e., torch tensors). It iterates
through each key, value in the dict and calls _simplify on it. Finally,
it returns the output dict as the same type as the input dict
so that the consuming serialization step knows the correct type info. The
reverse function to this function is _detail_dictionary, which undoes
the functionality of this function.
Args:
my_dict: A dictionary of python objects.
Returns:
Dict: A dictionary of the same type as the input of simplified
objects.
"""
pieces = list()
# for dictionaries we want to simplify both the key and the value
for key, value in my_dict.items():
pieces.append((serde._simplify(key), serde._simplify(value)))
return pieces
def test_ndarray_simplify():
"""This tests our ability to simplify numpy.array objects
At the time of writing, arrays simplify to an object inside
of a tuple which specifies the ID for the np.array type (6) so
that the detailer knows to turn the simplifed form to a np.array
"""
input = numpy.random.random((100, 100))
output = _simplify(input)
# make sure simplified type ID is correct
assert output[0] == 7
# make sure serialized form is correct
assert type(output[1][0]) == bytes
assert output[1][1] == input.shape
assert output[1][2] == input.dtype.name
def _simplify_numpy_number(
worker: AbstractWorker, numpy_nb: Union[numpy.int32, numpy.int64,
numpy.float32, numpy.float64]
) -> Tuple[bin, Tuple]:
"""
This function gets the byte representation of the numpy number
and stores the dtype for reconstruction
Args:
numpy_nb (e.g numpy.float64): a numpy number
Returns:
list: a list holding the byte representation, dtype of the numpy number
Examples:
np_representation = _simplify_numpy_number(worker, numpy.float64(2.3)))
"""
nb_bytes = numpy_nb.tobytes()
nb_dtype = serde._simplify(worker, numpy_nb.dtype.name)
return (nb_bytes, nb_dtype)
