def _simplify_dictionary(worker: AbstractWorker, my_dict: Dict, shallow: bool = False) -> Tuple: """ This function is designed to search a dict for any objects which may need to be simplified (i.e., torch tensors). It iterates through each key, value in the dict and calls _simplify on it. Finally, it returns the output tuple of tuples containing key/value pairs. The reverse function to this function is _detail_dictionary, which undoes the functionality of this function. Args: my_dict: A dictionary of python objects. Returns: Tuple: Tuple containing tuples of simplified key/value pairs from the input dictionary. """ pieces = list() # for dictionaries we want to simplify both the key and the value for key, value in my_dict.items(): pieces.append( (serde._simplify(worker, key), serde._simplify(worker, value) if not shallow else value)) return tuple(pieces)
def test_ellipsis_simplify(): """Make sure ellipsis simplifies correctly.""" # the id indicating an ellipsis is here assert _simplify(Ellipsis)[0] == 9 # the simplified ellipsis (empty object) assert _simplify(Ellipsis)[1] == b""
def test_torch_device_simplify(): """Test the simplification of torch.device""" device = torch.device("cpu") # the id indicating an torch.device is here assert _simplify(device)[0] == 10 # the simplified torch.device assert _simplify(device)[1] == "cpu"
def test_set_simplify(): """This tests our ability to simplify set objects. This test is pretty simple since sets just serialize to lists, with a tuple wrapper with the correct ID (3) for sets so that the detailer knows how to interpret it.""" input = set(["hello", "world"]) target = (4, ["hello", "world"]) assert _simplify(input)[0] == target[0] assert set(_simplify(input)[1]) == set(target[1])
def _simplify_collection(my_collection: Collection) -> Collection: """ This function is designed to search a collection for any objects which may need to be simplified (i.e., torch tensors). It iterates through each object in the collection and calls _simplify on it. Finally, it returns the output collection as the same type as the input collection so that the consuming serialization step knows the correct type info. The reverse function to this function is _detail_collection, which undoes the functionality of this function. Args: my_collection (Collection): a collection of python objects Returns: Collection: a collection of the same type as the input of simplified objects. """ # Step 0: get collection type for later use and itialize empty list my_type = type(my_collection) pieces = list() # Step 1: serialize each part of the collection for part in my_collection: pieces.append(serde._simplify(part)) # Step 2: convert back to original type and return serialization if my_type == set: return pieces return my_type(pieces)
def test_torch_tensor_simplify(): """This tests our ability to simplify torch.Tensor objects At the time of writing, tensors simplify to a tuple where the first value in the tuple is the tensor's ID and the second value is a serialized version of the Tensor (serialized by PyTorch's torch.save method) """ # create a tensor input = Tensor(numpy.random.random((100, 100))) # simplify the tnesor output = _simplify(input) # make sure outer type is correct assert type(output) == tuple # make sure the object type ID is correct # (0 for torch.Tensor) assert output[0] == 0 # make sure inner type is correct assert type(output[1]) == tuple # make sure ID is correctly encoded assert output[1][0] == input.id # make sure tensor data type is correct assert type(output[1][1]) == bytes
def _simplify_collection(my_collection: Collection) -> Tuple: """ This function is designed to search a collection for any objects which may need to be simplified (i.e., torch tensors). It iterates through each object in the collection and calls _simplify on it. Finally, it returns the output as the tuple of simplified items of the input collection. This function is used to simplify list, set, and tuple. The reverse function, which undoes the functionality of this function is different for each of these types: _detail_collection_list, _detail_collection_set, _detail_collection_tuple. Args: my_collection (Collection): a collection of python objects Returns: Tuple: a tuple with simplified objects. """ # Step 0: initialize empty list pieces = list() # Step 1: serialize each part of the collection for part in my_collection: pieces.append(serde._simplify(part)) # Step 2: return serialization as tuple of simplified items return tuple(pieces)
def test_float_simplify(): """This tests our ability to simplify float objects. This test is pretty simple since floats just serialize to themselves, with no tuple/id necessary.""" input = 5.6 target = 5.6 assert serde._simplify(input) == target
def test_int_simplify(): """This tests our ability to simplify int objects. This test is pretty simple since ints just serialize to themselves, with no tuple/id necessary.""" input = 5 target = 5 assert _simplify(input) == target
def test_string_simplify(): """This tests our ability to simplify string objects. This test is pretty simple since strings just serialize to themselves, with no tuple/id necessary.""" input = "hello" target = "hello" assert _simplify(input) == target
def test_dict_simplify(): """This tests our ability to simplify dict objects. This test is pretty simple since dicts just serialize to themselves, with a tuple wrapper with the correct ID (4) for dicts so that the detailer knows how to interpret it.""" input = {"hello": "world"} target = (5, {"hello": "world"}) assert _simplify(input) == target
def test_list_simplify(): """This tests our ability to simplify list types. This test is pretty simple since lists just serialize to themselves, with a tuple wrapper with the correct ID (2) for lists so that the detailer knows how to interpret it.""" input = ["hello", "world"] target = (3, ["hello", "world"]) assert _simplify(input) == target
def test_tuple_simplify(): """This tests our ability to simplify tuple types. This test is pretty simple since tuples just serialize to themselves, with a tuple wrapper with the correct ID (1) for tuples so that the detailer knows how to interpret it.""" input = ("hello", "world") target = (2, ("hello", "world")) assert _simplify(input) == target
def test_range_simplify(): """This tests our ability to simplify range objects. This test is pretty simple since range objs just serialize to themselves, with a tuple wrapper with the correct ID (5) for dicts so that the detailer knows how to interpret it.""" input = range(1, 3, 4) target = (6, (1, 3, 4)) assert _simplify(input) == target
def test_pointer_tensor_simplify(): """Test the simplification of PointerTensor""" alice = syft.VirtualWorker(syft.torch.hook, id="alice") input_tensor = PointerTensor(id=1000, location=alice, owner=alice) output = _simplify(input_tensor) assert output[1][0] == input_tensor.id assert output[1][1] == input_tensor.id_at_location assert output[1][2] == input_tensor.owner.id
def _simplify_ndarray(worker: AbstractWorker, my_array: numpy.ndarray) -> Tuple[bin, Tuple, Tuple]: """ This function gets the byte representation of the array and stores the dtype and shape for reconstruction Args: my_array (numpy.ndarray): a numpy array Returns: list: a list holding the byte representation, shape and dtype of the array Examples: arr_representation = _simplify_ndarray(numpy.random.random([1000, 1000]))) """ arr_bytes = my_array.tobytes() arr_shape = serde._simplify(worker, my_array.shape) arr_dtype = serde._simplify(worker, my_array.dtype.name) return (arr_bytes, arr_shape, arr_dtype)
def _simplify_dictionary(my_dict: Dict) -> Dict: """ This function is designed to search a dict for any objects which may need to be simplified (i.e., torch tensors). It iterates through each key, value in the dict and calls _simplify on it. Finally, it returns the output dict as the same type as the input dict so that the consuming serialization step knows the correct type info. The reverse function to this function is _detail_dictionary, which undoes the functionality of this function. Args: my_dict: A dictionary of python objects. Returns: Dict: A dictionary of the same type as the input of simplified objects. """ pieces = list() # for dictionaries we want to simplify both the key and the value for key, value in my_dict.items(): pieces.append((serde._simplify(key), serde._simplify(value))) return pieces
def test_ndarray_simplify(): """This tests our ability to simplify numpy.array objects At the time of writing, arrays simplify to an object inside of a tuple which specifies the ID for the np.array type (6) so that the detailer knows to turn the simplifed form to a np.array """ input = numpy.random.random((100, 100)) output = _simplify(input) # make sure simplified type ID is correct assert output[0] == 7 # make sure serialized form is correct assert type(output[1][0]) == bytes assert output[1][1] == input.shape assert output[1][2] == input.dtype.name
def _simplify_numpy_number( worker: AbstractWorker, numpy_nb: Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64] ) -> Tuple[bin, Tuple]: """ This function gets the byte representation of the numpy number and stores the dtype for reconstruction Args: numpy_nb (e.g numpy.float64): a numpy number Returns: list: a list holding the byte representation, dtype of the numpy number Examples: np_representation = _simplify_numpy_number(worker, numpy.float64(2.3))) """ nb_bytes = numpy_nb.tobytes() nb_dtype = serde._simplify(worker, numpy_nb.dtype.name) return (nb_bytes, nb_dtype)