def test_serde_storable_obj() -> None:
id = UID()
data = th.Tensor([-1, -2, -3, -4])
description = "This is a dummy id"
tags = ["dummy", "test"]
obj1 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = th.Tensor([1, 2, 3, 4])
description = "This is a dummy tensor n1"
tags = ["dummy", "test"]
obj2 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = th.Tensor([10, 20, 30, 40])
description = "This is a dummy tensor n2"
tags = ["dummy", "test"]
obj3 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = [obj1, obj2, obj3]
description = "This is a dataset"
tags = ["dummy", "dataset"]
dataset_obj = Dataset(id=id, data=data, description=description, tags=tags)
blob = sy.serialize(obj=dataset_obj)
sy.deserialize(blob=blob)
def test_decode_message() -> None:
"""Tests that SignedMessage serialized_message is not encrypted"""
blob = get_signed_message_bytes()
sig_msg = sy.deserialize(blob=blob, from_bytes=True)
nonveri_msg = sy.deserialize(blob=sig_msg.serialized_message, from_bytes=True)
obj = get_repr_message()
assert nonveri_msg == obj
def test_serde_storable_obj() -> None:
id = UID()
data = th.Tensor([1, 2, 3, 4])
description = "This is a dummy test"
tags = ["dummy", "test"]
obj = StorableObject(id=id, data=data, description=description, tags=tags)
blob = sy.serialize(obj=obj)
sy.deserialize(blob=blob)
def test_get_role_response_serde() -> None:
target = Address(name="Alice")
content = {
"name": "Role Sample",
"can_triage_results": True,
"can_edit_settings": False,
"can_create_users": False,
"can_edit_roles": False,
"can_manage_infrastructure": True,
}
msg = GetRoleResponse(
address=target,
status_code=200,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_update_role_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
content = {
"role_id": "9a4f9dasd6",
"name": "Role Sample",
"can_triage_results": True,
"can_edit_settings": False,
"can_create_users": False,
"can_edit_roles": False,
"can_manage_infrastructure": True,
}
msg = UpdateRoleMessage(
address=target,
content=content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_get_all_data_requests_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"dataset-id": "68a465aer3adf",
"user-id": "user-id7",
"request-type": "read",
},
"a84ew64wq6e": {
"dataset-id": "98w4e54a6d",
"user-id": "user-id9",
"request-type": "write",
},
}
}
msg = GetRequestsResponse(
address=target,
status_code=200,
content=request_content,
)
blob = sy.serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_delete_worker_response_serde() -> None:
target = Address(name="Alice")
content = {
"settings": {
"cloud-admin-token": "d84we35ad3a1d59a84sd9",
"cloud-credentials": "<cloud-credentials.pem>",
"infra": {
"autoscaling": True,
"triggers": {
"memory": "50",
"vCPU": "80"
}
},
}
}
msg = GetSetUpResponse(
status_code=200,
address=target,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_create_initial_setup_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
request_content = {
"settings": {
"cloud-admin-token": "d84we35ad3a1d59a84sd9",
"cloud-credentials": "<cloud-credentials.pem>",
"infra": {
"autoscaling": True,
"triggers": {
"memory": "50",
"vCPU": "80"
}
},
}
}
msg = CreateInitialSetUpMessage(
address=target,
content=request_content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_uid_default_deserialization() -> None:
"""Tests that default UID deserialization works as expected - from Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
blob = _serialize(obj=uid)
obj = sy.deserialize(blob=blob)
assert obj == UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
def test_get_all_tensors_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"tensor": [1, 2, 3, 4, 5, 6],
"description": "Tensor description",
"tags": ["#x", "#data-sample"],
"pointable": True,
},
"a84ew64wq6e": {
"tensor": [9, 8, 2, 3, 5, 6],
"description": "Tensor sample description",
"tags": ["#y", "#label-sample"],
"pointable": True,
},
}
}
msg = GetTensorsResponse(
address=target,
status_code=200,
content=request_content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_proto_deserialization() -> None:
"""Tests that default Address deserialization works as expected - from Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
loc = SpecificLocation(id=uid, name="Test Location")
obj = Address(
network=loc,
domain=loc,
device=loc,
vm=loc,
)
blob = Address.get_protobuf_schema()(
has_network=True,
has_domain=True,
has_device=True,
has_vm=True,
network=loc.serialize(),
domain=loc.serialize(),
device=loc.serialize(),
vm=loc.serialize(),
)
obj2 = sy.deserialize(blob=blob, from_proto=True)
assert obj == obj2
def test_run_function_or_constructor_action_serde() -> None:
alice = sy.VirtualMachine(name="alice")
alice_client = alice.get_client()
args = (
th.tensor([1, 2, 3]).send(alice_client),
th.tensor([4, 5, 5]).send(alice_client),
)
msg = RunFunctionOrConstructorAction(
path="torch.Tensor.add",
args=args,
kwargs={},
id_at_location=UID(),
address=alice_client.address,
msg_id=UID(),
)
blob = msg.serialize()
msg2 = sy.deserialize(blob=blob)
assert msg2.path == msg.path
# FIXME this cannot be checked before we fix the Pointer serde problem (see _proto2object in Pointer)
# assert msg2.args == msg.args
assert msg2.kwargs == msg.kwargs
assert msg2.address == msg.address
assert msg2.id == msg.id
assert msg2.id_at_location == msg.id_at_location
def test_signaling_answer_message_serde() -> None:
bob_vm = sy.VirtualMachine(name="Bob")
target = Address(name="Alice")
target_id = secrets.token_hex(nbytes=16)
host_id = secrets.token_hex(nbytes=16)
msg = SignalingAnswerMessage(
address=target,
payload="SDP",
host_metadata=bob_vm.get_metadata_for_client(),
target_peer=target_id,
host_peer=host_id,
)
msg_metadata = bob_vm.get_metadata_for_client()
blob = msg.serialize()
msg2 = sy.deserialize(blob=blob)
msg2_metadata = msg2.host_metadata
assert msg.id == msg2.id
assert msg.address == target
assert msg.payload == msg2.payload
assert msg2.payload == "SDP"
assert msg2.host_peer == host_id
assert msg2.target_peer == target_id
assert msg == msg2
assert msg_metadata.name == msg2_metadata.name
assert msg_metadata.node == msg2_metadata.node
assert msg_metadata.id == msg2_metadata.id
def syft_multipart_route():
# grid relative
from ...core.node import get_node # TODO: fix circular import
if "file" not in request.files:
response = {"error": "Invalid message!"}
status_code = 403
file_obj = request.files["file"]
msg = file_obj.stream.read()
obj_msg = deserialize(blob=msg, from_bytes=True)
if isinstance(obj_msg, SignedImmediateSyftMessageWithReply):
reply = get_node().recv_immediate_msg_with_reply(msg=obj_msg)
r = Response(response=_serialize(obj=reply, to_bytes=True), status=200)
r.headers["Content-Type"] = "application/octet-stream"
del msg
del obj_msg
return r
elif isinstance(obj_msg, SignedImmediateSyftMessageWithoutReply):
get_node().recv_immediate_msg_without_reply(msg=obj_msg)
else:
get_node().recv_eventual_msg_without_reply(msg=obj_msg)
return ""
def test_get_all_workers_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"environment-name": "Heart Diseases Environment",
"owner": "user-id7",
"deployment-date": "05/12/2021",
},
"a84ew64wq6e": {
"worker-id": "eqw9e4a5d846",
"environment-name": "Brain Diseases Environment",
"owner": "user-id8",
"deployment-date": "15/12/2021",
},
}
}
msg = GetWorkersResponse(
address=target,
status_code=200,
content=request_content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_create_dataset_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
request_content = {
"dataset":
["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"],
"description": "Dataset Description",
"tags": ["#x", "#data-sample"],
"pointable": True,
"read-permission": ["user-id1", "user-id2", "user-id3"],
"write-permission": ["user-id1", "user-id5", "user-id9"],
}
msg = CreateDatasetMessage(
address=target,
content=request_content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_serde_matches() -> None:
"""Tests that the nested serde is reversible at all levels"""
# serial
sig_msg_blob = get_signed_message_bytes()
# deserial should be expected type
sig_msg = sy.deserialize(blob=sig_msg_blob, from_bytes=True)
assert type(sig_msg) == SignedImmediateSyftMessageWithoutReply
# reserial should be same as original fixture
comp_blob = serialize(sig_msg, to_bytes=True)
assert type(comp_blob) == bytes
assert comp_blob == sig_msg_blob
# now try sub message
msg = sig_msg.message
assert type(msg) == ReprMessage
# resign and the result should be the same
signing_key = get_signing_key()
sig_msg_comp = msg.sign(signing_key=signing_key)
assert type(sig_msg_comp) == SignedImmediateSyftMessageWithoutReply
assert type(sig_msg_comp) == type(sig_msg)
# make sure they have the same message id for comparison
sig_msg_comp._id = sig_msg.id
# identical (except the auto generated UID for the envelope)
assert sig_msg_comp == sig_msg
def test_get_dataset_response_serde() -> None:
target = Address(name="Alice")
content = {
"dataset":
["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"],
"description": "Dataset Description",
"tags": ["#x", "#data-sample"],
"pointable": True,
"read-permission": ["user-id1", "user-id2", "user-id3"],
"write-permission": ["user-id1", "user-id5", "user-id9"],
}
msg = GetDatasetResponse(
address=target,
status_code=200,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_update_worker_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
content = {
"worker-id": "eqw9e4a5d846",
"settings": {
"instance-size": "t4g.large",
"vCPU": "2",
"network-bandwith": "5Gbps",
"vGPU": True,
},
}
msg = UpdateWorkerMessage(
address=target,
content=content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_uid_proto_deserialization() -> None:
"""Tests that proto UID deserialization works as expected"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
blob = _serialize(obj=uid)
obj = sy.deserialize(blob=blob, from_proto=True)
assert obj == UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
def message_integrity_test(msg: AbstractMessage, target: Address) -> None:
blob = sy.serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_uid_binary_deserialization() -> None:
"""Test that binary deserialization works as expected"""
blob = (b"\n\x18syft.core.common.uid.UID\x12\x12\n\x10\xfb\x1b\xb0" +
b"g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14")
obj = sy.deserialize(blob=blob, from_bytes=True)
assert obj == UID(value=uuid.UUID(
int=333779996850170035686993356951732753684))
def test_object_with_id_default_deserialization() -> None:
"""Tests that default ObjectWithID deserialization works as expected - from Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = ObjectWithID(id=uid)
blob = ObjectWithID.get_protobuf_schema()(id=uid.serialize())
obj2 = sy.deserialize(blob=blob)
assert obj == obj2
def test_default_deserialization() -> None:
"""Tests that default SpecificLocation deserialization works as expected - from Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = SpecificLocation(id=uid, name="Test")
blob = SpecificLocation.get_protobuf_schema()(id=uid.serialize())
obj2 = sy.deserialize(blob=blob)
assert obj == obj2
def store_bytes_at(self, key: str, obj: bytes) -> None:
bin_obj = self.db.session.query(BinaryObject).get(key)
dataset = deserialize(blob=obj, from_bytes=True)
json_obj = self.db.session.query(JsonObject).get(key)
_json = dataset_to_dict(dataset)
setattr(bin_obj, "binary", obj)
setattr(json_obj, "binary", _json)
self.db.session.commit()
def test_verify_message() -> None:
"""Tests that SignedMessage can be verified"""
blob = get_signed_message_bytes()
sig_msg = sy.deserialize(blob=blob, from_bytes=True)
veri_msg = sig_msg.message
obj = get_repr_message()
assert veri_msg == obj
def test_partial_serialization_and_deserialization() -> None:
"""Test that addresses with only some attributes serialize and deserialize correctly/"""
an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = Address( # network=SpecificLocation(id=an_id),
domain=SpecificLocation(id=an_id),
device=SpecificLocation(id=an_id),
vm=SpecificLocation(id=an_id),
)
assert obj == sy.deserialize(blob=obj.serialize())
obj = Address(
network=SpecificLocation(id=an_id),
# domain=SpecificLocation(id=an_id),
device=SpecificLocation(id=an_id),
vm=SpecificLocation(id=an_id),
)
blob = obj.to_proto()
assert obj == sy.deserialize(blob=blob)
obj = Address(
network=SpecificLocation(id=an_id),
domain=SpecificLocation(id=an_id),
# device=SpecificLocation(id=an_id),
vm=SpecificLocation(id=an_id),
)
blob = obj.to_proto()
assert obj == sy.deserialize(blob=blob)
obj = Address(
network=SpecificLocation(id=an_id),
domain=SpecificLocation(id=an_id),
device=SpecificLocation(id=an_id),
# vm=SpecificLocation(id=an_id)
)
blob = obj.to_proto()
assert obj == sy.deserialize(blob=blob)
def test_get_message() -> None:
"""Tests that SignedMessage verification can be ignored"""
blob = get_signed_message_bytes()
sig_msg = sy.deserialize(blob=blob, from_bytes=True)
sig_msg.signature += b"a"
nonveri_msg = sig_msg.message
obj = get_repr_message()
assert nonveri_msg == obj
def test_get_object_action_serde() -> None:
msg = GetObjectAction(id_at_location=UID(),
address=Address(),
reply_to=Address(),
msg_id=UID())
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.id_at_location == msg2.id_at_location
assert msg.address == msg2.address
assert msg.reply_to == msg2.reply_to
def test_parameter_serde() -> None:
param = th.nn.parameter.Parameter(th.tensor([1.0, 2, 3]), requires_grad=True)
# Setting grad manually to check it is passed through serialization
param.grad = th.randn_like(param)
blob = sy.serialize(param)
param2 = sy.deserialize(blob=blob)
assert (param == param2).all()
assert (param2.grad == param2.grad).all()
assert param2.requires_grad == param2.requires_grad