def test_serde_object_wrapper_traced_module():
data = torch.tensor([[-1, 2.0], [0, 1.1], [-1, 2.1], [0, 1.2]])
class Net(torch.nn.Module):
def __init__(self):
super(Net, self).__init__()
self.fc1 = torch.nn.Linear(2, 3)
def forward(self, x):
x = torch.nn.functional.relu(self.fc1(x))
return x
obj = torch.jit.trace(Net(), data)
obj_wrapper = ObjectWrapper(obj, id=200)
msg = serde.serialize(obj_wrapper)
obj_wrapper_received = serde.deserialize(msg)
pred_before = obj(data)
pred_after = obj_wrapper_received.obj(data)
assert (pred_before == pred_after).all()
assert obj_wrapper.id == obj_wrapper_received.id
def test_float(compress):
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
x = 0.5
y = 1.5
x_serialized = serde.serialize(x)
x_serialized_deserialized = serde.deserialize(x_serialized)
y_serialized = serde.serialize(y)
y_serialized_deserialized = serde.deserialize(y_serialized)
assert x_serialized_deserialized == x
assert y_serialized_deserialized == y
def test_serde_object_wrapper_int():
obj = 4
obj_wrapper = ObjectWrapper(obj, id=100)
msg = serde.serialize(obj_wrapper)
obj_wrapper_received = serde.deserialize(msg)
assert obj_wrapper.obj == obj_wrapper_received.obj
assert obj_wrapper.id == obj_wrapper_received.id
def test_torch_jit_script_module_serde(): # pragma: no cover
@torch.jit.script
def foo(x):
return x + 2
msg = serde.serialize(foo)
foo_received = serde.deserialize(msg)
assert foo.code == foo_received.code
def test_pointer_tensor(hook, workers):
serde._apply_compress_scheme = serde.apply_no_compression
t = PointerTensor(
id=1000, location=workers["alice"], owner=workers["alice"], id_at_location=12345
)
t_serialized = serde.serialize(t)
t_serialized_deserialized = serde.deserialize(t_serialized)
assert t.id == t_serialized_deserialized.id
assert t.location.id == t_serialized_deserialized.location.id
assert t.id_at_location == t_serialized_deserialized.id_at_location
def test_torch_Tensor(compress):
if compress:
syft.serde._apply_compress_scheme = serde.apply_lz4_compression
else:
syft.serde._apply_compress_scheme = serde.apply_no_compression
t = Tensor(numpy.random.random((100, 100)))
t_serialized = serde.serialize(t)
t_serialized_deserialized = serde.deserialize(t_serialized)
assert (t == t_serialized_deserialized).all()
def test_serde_virtual_worker(hook):
virtual_worker = syft.VirtualWorker(hook=hook, id="deserialized_worker1")
# Populate worker
tensor1, tensor2 = torch.tensor([1.0, 2.0]), torch.tensor([0.0])
ptr1, ptr2 = tensor1.send(virtual_worker), tensor2.send(virtual_worker)
serialized_worker = serde.serialize(virtual_worker, force_full_simplification=False)
deserialized_worker = serde.deserialize(serialized_worker)
assert virtual_worker.id == deserialized_worker.id
def test_ndarray_serde(compress):
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
arr = numpy.random.random((100, 100))
arr_serialized = serde.serialize(arr)
arr_serialized_deserialized = serde.deserialize(arr_serialized)
assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_invalid_decompression_scheme(compress_scheme):
# using numpy.ones because numpy.random.random is not compressed.
arr = numpy.ones((100, 100))
def some_other_compression_scheme(decompressed_input):
# Simulate compression by removing some values
return decompressed_input[:10], compress_scheme
serde._apply_compress_scheme = some_other_compression_scheme
arr_serialized = serde.serialize(arr)
with pytest.raises(CompressionNotFoundException):
_ = serde.deserialize(arr_serialized)
def test_range_serde(compress):
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
_range = range(1, 2, 3)
range_serialized = serde.serialize(_range)
range_serialized_deserialized = serde.deserialize(range_serialized)
assert _range == range_serialized_deserialized
def test_slice(compress):
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
s = slice(0, 100, 2)
x = numpy.random.rand(100)
s_serialized = serde.serialize(s)
s_serialized_deserialized = serde.deserialize(s_serialized)
assert type(s) == type(s_serialized_deserialized)
assert (x[s] == x[s_serialized_deserialized]).all()
s = slice(40, 50)
x = numpy.random.rand(100)
s_serialized = serde.serialize(s)
s_serialized_deserialized = serde.deserialize(s_serialized)
assert type(s) == type(s_serialized_deserialized)
assert (x[s] == x[s_serialized_deserialized]).all()
def test_plan_serde(hook):
@sy.func2plan(args_shape=[(1, 3)])
def my_plan(data):
x = data * 2
y = (x - 2) * 10
return x + y
serialized_plan = serialize(my_plan)
deserialized_plan = deserialize(serialized_plan)
x = th.tensor([-1, 2, 3])
assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
def test_compressed_serde(compress_scheme):
if compress_scheme == LZ4:
syft.serde.serde._apply_compress_scheme = apply_lz4_compression
elif compress_scheme == ZSTD:
syft.serde.serde._apply_compress_scheme = apply_zstd_compression
else:
syft.serde.serde._apply_compress_scheme = apply_no_compression
arr = numpy.random.random((100, 100))
arr_serialized = serialize(arr)
arr_serialized_deserialized = deserialize(arr_serialized)
assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_func_plan_can_be_translated_to_tfjs(hook, workers):
Plan._build_translators = []
@sy.func2plan(args_shape=[(3, 3)])
def plan(x):
x = x * 2
x = x.abs()
return x
orig_plan = plan.copy()
plan_js = plan.copy()
plan_js.add_translation(PlanTranslatorTfjs)
plan_js.base_framework = TranslationTarget.TENSORFLOW_JS.value
assert plan_js.role.actions[0].name == "tf.mul"
assert len(plan_js.role.actions[0].args) == 2
assert len(plan_js.role.input_placeholders()) == len(
orig_plan.role.input_placeholders())
assert len(plan_js.role.output_placeholders()) == len(
orig_plan.role.output_placeholders())
# Test plan caching
plan_js2 = plan_js.copy()
plan_js2.add_translation(PlanTranslatorTfjs)
plan_js2.base_framework = TranslationTarget.TENSORFLOW_JS.value
assert plan_js2.role.actions[0].name == "tf.mul"
assert len(plan_js2.role.actions[0].args) == 2
# check that translation can be done after serde
serde_plan = deserialize(serialize(orig_plan))
serde_plan.add_translation(PlanTranslatorTfjs)
serde_plan.base_framework = TranslationTarget.TENSORFLOW_JS.value
assert serde_plan.role.actions[0].name == "tf.mul"
assert len(serde_plan.role.actions[0].args) == 2
# check that translation is not lost after serde
serde_plan_full = deserialize(serialize(plan_js))
assert serde_plan_full.role.actions[0].name == "tf.mul"
assert len(serde_plan_full.role.actions[0].args) == 2
def test_numpy_tensor_serde():
syft.serde.torch_serde._serialize_tensor = syft.serde.torch_serde.numpy_tensor_serializer
syft.serde.torch_serde._deserialize_tensor = syft.serde.torch_serde.numpy_tensor_deserializer
tensor = torch.tensor(numpy.random.random((10, 10)), requires_grad=False)
tensor_serialized = serialize(tensor)
tensor_deserialized = deserialize(tensor_serialized)
# Back to Pytorch serializer
syft.serde.torch_serde._serialize_tensor = syft.serde.torch_serde.torch_tensor_serializer
syft.serde.torch_serde._deserialize_tensor = syft.serde.torch_serde.torch_tensor_deserializer
assert torch.eq(tensor_deserialized, tensor).all()
def test_plan_serde(hook):
with hook.local_worker.registration_enabled():
@sy.func2plan(args_shape=[(1, 3)])
def my_plan(data):
x = data * 2
y = (x - 2) * 10
return x + y
serialized_plan = serialize(my_plan)
deserialized_plan = deserialize(serialized_plan)
x = th.tensor([-1, 2, 3])
assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
def test_tuple(compress):
# Test with a simple datatype
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
tuple = (1, 2)
tuple_serialized = serde.serialize(tuple)
tuple_serialized_deserialized = serde.deserialize(tuple_serialized)
assert tuple == tuple_serialized_deserialized
# Test with a complex data structure
tensor_one = Tensor(numpy.random.random((100, 100)))
tensor_two = Tensor(numpy.random.random((100, 100)))
tuple = (tensor_one, tensor_two)
tuple_serialized = serde.serialize(tuple)
tuple_serialized_deserialized = serde.deserialize(tuple_serialized)
# `assert tuple_serialized_deserialized == tuple` does not work, therefore it's split
# into 3 assertions
assert type(tuple_serialized_deserialized) == type(tuple)
assert (tuple_serialized_deserialized[0] == tensor_one).all()
assert (tuple_serialized_deserialized[1] == tensor_two).all()
def test_cls_plan_can_be_translated_to_torchscript(hook, workers):
# Disable build time auto translation
Plan._build_translators = []
class Net(sy.Plan):
def __init__(self):
super(Net, self).__init__()
self.fc1 = nn.Linear(2, 3)
self.fc2 = nn.Linear(3, 1)
def forward(self, x):
x = self.fc1(x)
x = F.relu(x)
x = self.fc2(x)
return x
plan = Net()
plan.build(th.zeros(10, 2))
orig_plan = plan.copy()
inp = th.randn(10, 2)
res1 = plan(inp)
plan.add_translation(PlanTranslatorTorchscript)
res2 = plan.torchscript(inp, plan.parameters())
assert (res1 == res2).all()
# check that translation can be done after serde
serde_plan = deserialize(serialize(orig_plan))
serde_plan.add_translation(PlanTranslatorTorchscript)
res3 = serde_plan.torchscript(inp, serde_plan.parameters())
assert (res1 == res3).all()
# check that translation is not lost after serde
serde_plan_full = deserialize(serialize(plan))
res4 = serde_plan_full.torchscript(inp, serde_plan_full.parameters())
assert (res1 == res4).all()
def test_hooked_tensor(compress, compress_scheme):
if compress:
if compress_scheme == LZ4:
syft.serde.serde._apply_compress_scheme = apply_lz4_compression
elif compress_scheme == ZSTD:
syft.serde.serde._apply_compress_scheme = apply_zstd_compression
else:
syft.serde.serde._apply_compress_scheme = apply_no_compression
else:
syft.serde.serde._apply_compress_scheme = apply_no_compression
t = Tensor(numpy.random.random((100, 100)))
t_serialized = serialize(t)
t_serialized_deserialized = deserialize(t_serialized)
assert (t == t_serialized_deserialized).all()
def test_compressed_serde(compress_scheme):
if compress_scheme == serde.LZ4:
serde._apply_compress_scheme = serde.apply_lz4_compression
elif compress_scheme == serde.ZSTD:
serde._apply_compress_scheme = serde.apply_zstd_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
# using numpy.ones because numpy.random.random is not compressed.
arr = numpy.ones((100, 100))
arr_serialized = serde.serialize(arr)
arr_serialized_deserialized = serde.deserialize(arr_serialized)
assert numpy.array_equal(arr, arr_serialized_deserialized)
def test_full_serde_virtual_worker(hook):
virtual_worker = syft.VirtualWorker(hook=hook, id="deserialized_worker2")
# Populate worker
tensor1, tensor2 = torch.tensor([1.0, 2.0]), torch.tensor([0.0])
ptr1, ptr2 = tensor1.send(virtual_worker), tensor2.send(virtual_worker)
serialized_worker = serde.serialize(virtual_worker, force_full_simplification=True)
deserialized_worker = serde.deserialize(serialized_worker)
assert virtual_worker.id == deserialized_worker.id
assert virtual_worker.auto_add == deserialized_worker.auto_add
assert len(deserialized_worker._objects) == 2
assert tensor1.id in deserialized_worker._objects
assert tensor2.id in deserialized_worker._objects
def test_set(compress):
if compress:
serde._apply_compress_scheme = serde.apply_lz4_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
# Test with integers
_set = set([1, 2])
set_serialized = serde.serialize(_set)
set_serialized_deserialized = serde.deserialize(set_serialized)
assert _set == set_serialized_deserialized
# Test with strings
_set = set(["hello", "world"])
set_serialized = serde.serialize(_set)
set_serialized_deserialized = serde.deserialize(set_serialized)
assert _set == set_serialized_deserialized
# Test with a complex data structure
tensor_one = Tensor(numpy.ones((100, 100)))
tensor_two = Tensor(numpy.ones((100, 100)) * 2)
_set = (tensor_one, tensor_two)
set_serialized = serde.serialize(_set)
if compress:
assert set_serialized[0] == serde.LZ4
else:
assert set_serialized[0] == serde.NO_COMPRESSION
set_serialized_deserialized = serde.deserialize(set_serialized)
# `assert set_serialized_deserialized == _set` does not work, therefore it's split
# into 3 assertions
assert type(set_serialized_deserialized) == type(_set)
assert (set_serialized_deserialized[0] == tensor_one).all()
assert (set_serialized_deserialized[1] == tensor_two).all()
def test_plan_serde(hook):
hook.local_worker.is_client_worker = False
@sy.func2plan(args_shape=[(1, 3)])
def my_plan(data):
x = data * 2
y = (x - 2) * 10
return x + y
serialized_plan = serialize(my_plan)
deserialized_plan = deserialize(serialized_plan)
x = th.tensor([-1, 2, 3])
assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
hook.local_worker.is_client_worker = True
def test_fixed_precision_tensor_serde(compress, workers):
alice, bob, james = workers["alice"], workers["bob"], workers["james"]
x = (torch.tensor([[3.1, 4.3]]).fix_prec(
base=12, precision_fractional=5).share(alice,
bob,
crypto_provider=james))
serialized_x = serde.serialize(x)
deserialied_x = serde.deserialize(serialized_x)
assert x.id == deserialied_x.child.id
assert x.child.field == deserialied_x.child.field
assert x.child.kappa == deserialied_x.child.kappa
assert x.child.precision_fractional == deserialied_x.child.precision_fractional
assert x.child.base == deserialied_x.child.base
def test_hooked_tensor(compress, compress_scheme):
if compress:
if compress_scheme == serde.LZ4:
serde._apply_compress_scheme = serde.apply_lz4_compression
elif compress_scheme == serde.ZSTD:
serde._apply_compress_scheme = serde.apply_zstd_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
else:
serde._apply_compress_scheme = serde.apply_no_compression
t = Tensor(numpy.ones((100, 100)))
t_serialized = serde.serialize(t)
assert (t_serialized[0] == compress_scheme
if compress else t_serialized[0] == serde.NO_COMPRESSION)
t_serialized_deserialized = serde.deserialize(t_serialized)
assert (t == t_serialized_deserialized).all()
def test_numpy_tensor_serde():
serde._apply_compress_scheme = serde.apply_lz4_compression
serde._serialize_tensor = syft.serde.numpy_tensor_serializer
serde._deserialize_tensor = syft.serde.numpy_tensor_deserializer
tensor = torch.tensor(numpy.ones((10, 10)), requires_grad=False)
tensor_serialized = serde.serialize(tensor)
assert tensor_serialized[0] != serde.NO_COMPRESSION
tensor_deserialized = serde.deserialize(tensor_serialized)
# Back to Pytorch serializer
serde._serialize_tensor = syft.serde.torch_tensor_serializer
serde._deserialize_tensor = syft.serde.torch_tensor_deserializer
assert torch.eq(tensor_deserialized, tensor).all()
def test_plan_serde(hook):
hook.local_worker.is_client_worker = False
@sy.func2plan
def my_plan(data):
x = data * 2
y = (x - 2) * 10
return x + y
# TODO: remove this line when issue #2062 is fixed
# Force to build plan
my_plan(th.tensor([1, 2, 3]))
serialized_plan = serialize(my_plan)
deserialized_plan = deserialize(serialized_plan)
x = th.tensor([-1, 2, 3])
assert (deserialized_plan(x) == th.tensor([-42, 24, 46])).all()
