def test_serde_object_wrapper_traced_module(): data = torch.tensor([[-1, 2.0], [0, 1.1], [-1, 2.1], [0, 1.2]]) class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.fc1 = torch.nn.Linear(2, 3) def forward(self, x): x = torch.nn.functional.relu(self.fc1(x)) return x obj = torch.jit.trace(Net(), data) obj_wrapper = ObjectWrapper(obj, id=200) msg = serde.serialize(obj_wrapper) obj_wrapper_received = serde.deserialize(msg) pred_before = obj(data) pred_after = obj_wrapper_received.obj(data) assert (pred_before == pred_after).all() assert obj_wrapper.id == obj_wrapper_received.id
def test_float(compress): if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression x = 0.5 y = 1.5 x_serialized = serde.serialize(x) x_serialized_deserialized = serde.deserialize(x_serialized) y_serialized = serde.serialize(y) y_serialized_deserialized = serde.deserialize(y_serialized) assert x_serialized_deserialized == x assert y_serialized_deserialized == y
def test_serde_object_wrapper_int(): obj = 4 obj_wrapper = ObjectWrapper(obj, id=100) msg = serde.serialize(obj_wrapper) obj_wrapper_received = serde.deserialize(msg) assert obj_wrapper.obj == obj_wrapper_received.obj assert obj_wrapper.id == obj_wrapper_received.id
def test_torch_jit_script_module_serde(): # pragma: no cover @torch.jit.script def foo(x): return x + 2 msg = serde.serialize(foo) foo_received = serde.deserialize(msg) assert foo.code == foo_received.code
def test_pointer_tensor(hook, workers): serde._apply_compress_scheme = serde.apply_no_compression t = PointerTensor( id=1000, location=workers["alice"], owner=workers["alice"], id_at_location=12345 ) t_serialized = serde.serialize(t) t_serialized_deserialized = serde.deserialize(t_serialized) assert t.id == t_serialized_deserialized.id assert t.location.id == t_serialized_deserialized.location.id assert t.id_at_location == t_serialized_deserialized.id_at_location
def test_torch_Tensor(compress): if compress: syft.serde._apply_compress_scheme = serde.apply_lz4_compression else: syft.serde._apply_compress_scheme = serde.apply_no_compression t = Tensor(numpy.random.random((100, 100))) t_serialized = serde.serialize(t) t_serialized_deserialized = serde.deserialize(t_serialized) assert (t == t_serialized_deserialized).all()
def test_serde_virtual_worker(hook): virtual_worker = syft.VirtualWorker(hook=hook, id="deserialized_worker1") # Populate worker tensor1, tensor2 = torch.tensor([1.0, 2.0]), torch.tensor([0.0]) ptr1, ptr2 = tensor1.send(virtual_worker), tensor2.send(virtual_worker) serialized_worker = serde.serialize(virtual_worker, force_full_simplification=False) deserialized_worker = serde.deserialize(serialized_worker) assert virtual_worker.id == deserialized_worker.id
def test_ndarray_serde(compress): if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression arr = numpy.random.random((100, 100)) arr_serialized = serde.serialize(arr) arr_serialized_deserialized = serde.deserialize(arr_serialized) assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_invalid_decompression_scheme(compress_scheme): # using numpy.ones because numpy.random.random is not compressed. arr = numpy.ones((100, 100)) def some_other_compression_scheme(decompressed_input): # Simulate compression by removing some values return decompressed_input[:10], compress_scheme serde._apply_compress_scheme = some_other_compression_scheme arr_serialized = serde.serialize(arr) with pytest.raises(CompressionNotFoundException): _ = serde.deserialize(arr_serialized)
def test_range_serde(compress): if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression _range = range(1, 2, 3) range_serialized = serde.serialize(_range) range_serialized_deserialized = serde.deserialize(range_serialized) assert _range == range_serialized_deserialized
def test_slice(compress): if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression s = slice(0, 100, 2) x = numpy.random.rand(100) s_serialized = serde.serialize(s) s_serialized_deserialized = serde.deserialize(s_serialized) assert type(s) == type(s_serialized_deserialized) assert (x[s] == x[s_serialized_deserialized]).all() s = slice(40, 50) x = numpy.random.rand(100) s_serialized = serde.serialize(s) s_serialized_deserialized = serde.deserialize(s_serialized) assert type(s) == type(s_serialized_deserialized) assert (x[s] == x[s_serialized_deserialized]).all()
def test_plan_serde(hook): @sy.func2plan(args_shape=[(1, 3)]) def my_plan(data): x = data * 2 y = (x - 2) * 10 return x + y serialized_plan = serialize(my_plan) deserialized_plan = deserialize(serialized_plan) x = th.tensor([-1, 2, 3]) assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
def test_compressed_serde(compress_scheme): if compress_scheme == LZ4: syft.serde.serde._apply_compress_scheme = apply_lz4_compression elif compress_scheme == ZSTD: syft.serde.serde._apply_compress_scheme = apply_zstd_compression else: syft.serde.serde._apply_compress_scheme = apply_no_compression arr = numpy.random.random((100, 100)) arr_serialized = serialize(arr) arr_serialized_deserialized = deserialize(arr_serialized) assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_func_plan_can_be_translated_to_tfjs(hook, workers): Plan._build_translators = [] @sy.func2plan(args_shape=[(3, 3)]) def plan(x): x = x * 2 x = x.abs() return x orig_plan = plan.copy() plan_js = plan.copy() plan_js.add_translation(PlanTranslatorTfjs) plan_js.base_framework = TranslationTarget.TENSORFLOW_JS.value assert plan_js.role.actions[0].name == "tf.mul" assert len(plan_js.role.actions[0].args) == 2 assert len(plan_js.role.input_placeholders()) == len( orig_plan.role.input_placeholders()) assert len(plan_js.role.output_placeholders()) == len( orig_plan.role.output_placeholders()) # Test plan caching plan_js2 = plan_js.copy() plan_js2.add_translation(PlanTranslatorTfjs) plan_js2.base_framework = TranslationTarget.TENSORFLOW_JS.value assert plan_js2.role.actions[0].name == "tf.mul" assert len(plan_js2.role.actions[0].args) == 2 # check that translation can be done after serde serde_plan = deserialize(serialize(orig_plan)) serde_plan.add_translation(PlanTranslatorTfjs) serde_plan.base_framework = TranslationTarget.TENSORFLOW_JS.value assert serde_plan.role.actions[0].name == "tf.mul" assert len(serde_plan.role.actions[0].args) == 2 # check that translation is not lost after serde serde_plan_full = deserialize(serialize(plan_js)) assert serde_plan_full.role.actions[0].name == "tf.mul" assert len(serde_plan_full.role.actions[0].args) == 2
def test_numpy_tensor_serde(): syft.serde.torch_serde._serialize_tensor = syft.serde.torch_serde.numpy_tensor_serializer syft.serde.torch_serde._deserialize_tensor = syft.serde.torch_serde.numpy_tensor_deserializer tensor = torch.tensor(numpy.random.random((10, 10)), requires_grad=False) tensor_serialized = serialize(tensor) tensor_deserialized = deserialize(tensor_serialized) # Back to Pytorch serializer syft.serde.torch_serde._serialize_tensor = syft.serde.torch_serde.torch_tensor_serializer syft.serde.torch_serde._deserialize_tensor = syft.serde.torch_serde.torch_tensor_deserializer assert torch.eq(tensor_deserialized, tensor).all()
def test_plan_serde(hook): with hook.local_worker.registration_enabled(): @sy.func2plan(args_shape=[(1, 3)]) def my_plan(data): x = data * 2 y = (x - 2) * 10 return x + y serialized_plan = serialize(my_plan) deserialized_plan = deserialize(serialized_plan) x = th.tensor([-1, 2, 3]) assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
def test_tuple(compress): # Test with a simple datatype if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression tuple = (1, 2) tuple_serialized = serde.serialize(tuple) tuple_serialized_deserialized = serde.deserialize(tuple_serialized) assert tuple == tuple_serialized_deserialized # Test with a complex data structure tensor_one = Tensor(numpy.random.random((100, 100))) tensor_two = Tensor(numpy.random.random((100, 100))) tuple = (tensor_one, tensor_two) tuple_serialized = serde.serialize(tuple) tuple_serialized_deserialized = serde.deserialize(tuple_serialized) # `assert tuple_serialized_deserialized == tuple` does not work, therefore it's split # into 3 assertions assert type(tuple_serialized_deserialized) == type(tuple) assert (tuple_serialized_deserialized[0] == tensor_one).all() assert (tuple_serialized_deserialized[1] == tensor_two).all()
def test_cls_plan_can_be_translated_to_torchscript(hook, workers): # Disable build time auto translation Plan._build_translators = [] class Net(sy.Plan): def __init__(self): super(Net, self).__init__() self.fc1 = nn.Linear(2, 3) self.fc2 = nn.Linear(3, 1) def forward(self, x): x = self.fc1(x) x = F.relu(x) x = self.fc2(x) return x plan = Net() plan.build(th.zeros(10, 2)) orig_plan = plan.copy() inp = th.randn(10, 2) res1 = plan(inp) plan.add_translation(PlanTranslatorTorchscript) res2 = plan.torchscript(inp, plan.parameters()) assert (res1 == res2).all() # check that translation can be done after serde serde_plan = deserialize(serialize(orig_plan)) serde_plan.add_translation(PlanTranslatorTorchscript) res3 = serde_plan.torchscript(inp, serde_plan.parameters()) assert (res1 == res3).all() # check that translation is not lost after serde serde_plan_full = deserialize(serialize(plan)) res4 = serde_plan_full.torchscript(inp, serde_plan_full.parameters()) assert (res1 == res4).all()
def test_hooked_tensor(compress, compress_scheme): if compress: if compress_scheme == LZ4: syft.serde.serde._apply_compress_scheme = apply_lz4_compression elif compress_scheme == ZSTD: syft.serde.serde._apply_compress_scheme = apply_zstd_compression else: syft.serde.serde._apply_compress_scheme = apply_no_compression else: syft.serde.serde._apply_compress_scheme = apply_no_compression t = Tensor(numpy.random.random((100, 100))) t_serialized = serialize(t) t_serialized_deserialized = deserialize(t_serialized) assert (t == t_serialized_deserialized).all()
def test_compressed_serde(compress_scheme): if compress_scheme == serde.LZ4: serde._apply_compress_scheme = serde.apply_lz4_compression elif compress_scheme == serde.ZSTD: serde._apply_compress_scheme = serde.apply_zstd_compression else: serde._apply_compress_scheme = serde.apply_no_compression # using numpy.ones because numpy.random.random is not compressed. arr = numpy.ones((100, 100)) arr_serialized = serde.serialize(arr) arr_serialized_deserialized = serde.deserialize(arr_serialized) assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_full_serde_virtual_worker(hook): virtual_worker = syft.VirtualWorker(hook=hook, id="deserialized_worker2") # Populate worker tensor1, tensor2 = torch.tensor([1.0, 2.0]), torch.tensor([0.0]) ptr1, ptr2 = tensor1.send(virtual_worker), tensor2.send(virtual_worker) serialized_worker = serde.serialize(virtual_worker, force_full_simplification=True) deserialized_worker = serde.deserialize(serialized_worker) assert virtual_worker.id == deserialized_worker.id assert virtual_worker.auto_add == deserialized_worker.auto_add assert len(deserialized_worker._objects) == 2 assert tensor1.id in deserialized_worker._objects assert tensor2.id in deserialized_worker._objects
def test_set(compress): if compress: serde._apply_compress_scheme = serde.apply_lz4_compression else: serde._apply_compress_scheme = serde.apply_no_compression # Test with integers _set = set([1, 2]) set_serialized = serde.serialize(_set) set_serialized_deserialized = serde.deserialize(set_serialized) assert _set == set_serialized_deserialized # Test with strings _set = set(["hello", "world"]) set_serialized = serde.serialize(_set) set_serialized_deserialized = serde.deserialize(set_serialized) assert _set == set_serialized_deserialized # Test with a complex data structure tensor_one = Tensor(numpy.ones((100, 100))) tensor_two = Tensor(numpy.ones((100, 100)) * 2) _set = (tensor_one, tensor_two) set_serialized = serde.serialize(_set) if compress: assert set_serialized[0] == serde.LZ4 else: assert set_serialized[0] == serde.NO_COMPRESSION set_serialized_deserialized = serde.deserialize(set_serialized) # `assert set_serialized_deserialized == _set` does not work, therefore it's split # into 3 assertions assert type(set_serialized_deserialized) == type(_set) assert (set_serialized_deserialized[0] == tensor_one).all() assert (set_serialized_deserialized[1] == tensor_two).all()
def test_plan_serde(hook): hook.local_worker.is_client_worker = False @sy.func2plan(args_shape=[(1, 3)]) def my_plan(data): x = data * 2 y = (x - 2) * 10 return x + y serialized_plan = serialize(my_plan) deserialized_plan = deserialize(serialized_plan) x = th.tensor([-1, 2, 3]) assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all() hook.local_worker.is_client_worker = True
def test_fixed_precision_tensor_serde(compress, workers): alice, bob, james = workers["alice"], workers["bob"], workers["james"] x = (torch.tensor([[3.1, 4.3]]).fix_prec( base=12, precision_fractional=5).share(alice, bob, crypto_provider=james)) serialized_x = serde.serialize(x) deserialied_x = serde.deserialize(serialized_x) assert x.id == deserialied_x.child.id assert x.child.field == deserialied_x.child.field assert x.child.kappa == deserialied_x.child.kappa assert x.child.precision_fractional == deserialied_x.child.precision_fractional assert x.child.base == deserialied_x.child.base
def test_hooked_tensor(compress, compress_scheme): if compress: if compress_scheme == serde.LZ4: serde._apply_compress_scheme = serde.apply_lz4_compression elif compress_scheme == serde.ZSTD: serde._apply_compress_scheme = serde.apply_zstd_compression else: serde._apply_compress_scheme = serde.apply_no_compression else: serde._apply_compress_scheme = serde.apply_no_compression t = Tensor(numpy.ones((100, 100))) t_serialized = serde.serialize(t) assert (t_serialized[0] == compress_scheme if compress else t_serialized[0] == serde.NO_COMPRESSION) t_serialized_deserialized = serde.deserialize(t_serialized) assert (t == t_serialized_deserialized).all()
def test_numpy_tensor_serde(): serde._apply_compress_scheme = serde.apply_lz4_compression serde._serialize_tensor = syft.serde.numpy_tensor_serializer serde._deserialize_tensor = syft.serde.numpy_tensor_deserializer tensor = torch.tensor(numpy.ones((10, 10)), requires_grad=False) tensor_serialized = serde.serialize(tensor) assert tensor_serialized[0] != serde.NO_COMPRESSION tensor_deserialized = serde.deserialize(tensor_serialized) # Back to Pytorch serializer serde._serialize_tensor = syft.serde.torch_tensor_serializer serde._deserialize_tensor = syft.serde.torch_tensor_deserializer assert torch.eq(tensor_deserialized, tensor).all()
def test_plan_serde(hook): hook.local_worker.is_client_worker = False @sy.func2plan def my_plan(data): x = data * 2 y = (x - 2) * 10 return x + y # TODO: remove this line when issue #2062 is fixed # Force to build plan my_plan(th.tensor([1, 2, 3])) serialized_plan = serialize(my_plan) deserialized_plan = deserialize(serialized_plan) x = th.tensor([-1, 2, 3]) assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()