def test_proto_deserialization() -> None:
"""Tests that default Address deserialization works as expected - from Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
loc = SpecificLocation(id=uid, name="Test Location")
obj = Address(
network=loc,
domain=loc,
device=loc,
vm=loc,
)
blob = Address.get_protobuf_schema()(
has_network=True,
has_domain=True,
has_device=True,
has_vm=True,
network=sy.serialize(loc),
domain=sy.serialize(loc),
device=sy.serialize(loc),
vm=sy.serialize(loc),
)
obj2 = sy.deserialize(blob=blob, from_proto=True)
assert obj == obj2
def _object2proto(self) -> Pointer_PB:
"""Returns a protobuf serialization of self.
As a requirement of all objects which inherit from Serializable,
this method transforms the current object into the corresponding
Protobuf object so that it can be further serialized.
:return: returns a protobuf object
:rtype: Pointer_PB
.. note::
This method is purely an internal method. Please use sy.serialize(object) or one of
the other public serialization methods if you wish to serialize an
object.
"""
return Pointer_PB(
points_to_object_with_path=self.path_and_name,
pointer_name=type(self).__name__,
id_at_location=sy.serialize(self.id_at_location),
location=sy.serialize(self.client.address),
tags=self.tags,
description=self.description,
object_type=self.object_type,
attribute_name=getattr(self, "attribute_name", ""),
)
def test_default_serialization() -> None:
"""Tests that default SpecificLocation serialization works as expected - to Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = SpecificLocation(id=uid, name="Test")
blob = sy.serialize(obj, to_proto=True)
assert sy.serialize(obj) == blob
def test_object_with_id_default_serialization() -> None:
"""Tests that default ObjectWithID serialization works as expected - to Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = ObjectWithID(id=uid)
blob = sy.serialize(obj, to_proto=True)
assert sy.serialize(obj) == blob
def test_uid_proto_serialization() -> None:
"""Tests that proto UID serialization works as expected"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
blob = _serialize(obj=uid)
assert sy.serialize(uid, to_proto=True) == blob
assert sy.serialize(uid, to_proto=True) == blob
assert sy.serialize(uid, to_proto=True) == blob
def test_uid_binary_serialization() -> None:
"""Tests that binary UID serializes as expected"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
blob = (b"\n\x18syft.core.common.uid.UID\x12\x12\n\x10\xfb\x1b\xb0" +
b"g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14")
assert sy.serialize(uid, to_bytes=True) == blob
assert sy.serialize(uid, to_bytes=True) == blob
assert sy.serialize(uid, to_bytes=True) == blob
def test_object_with_id_binary_serialization() -> None:
"""Tests that binary ObjectWithID serializes as expected"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = ObjectWithID(id=uid)
blob = (
b"\n$syft.core.common.object.ObjectWithID\x12\x14\n\x12\n\x10\xfb\x1b\xb0"
+ b"g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14")
assert sy.serialize(obj, to_bytes=True) == blob
assert sy.serialize(obj, to_bytes=True) == blob
assert sy.serialize(obj, to_bytes=True) == blob
def test_binary_serialization() -> None:
"""Tests that binary SpecificLocation serializes as expected"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = SpecificLocation(id=uid, name="Test")
blob = (
b"\n/syft.core.io.location.specific.SpecificLocation\x12\x1a\n\x12\n\x10"
+ b"\xfb\x1b\xb0g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14\x12\x04Test")
assert sy.serialize(obj, to_bytes=True) == blob
assert sy.serialize(obj, to_bytes=True) == blob
assert sy.serialize(obj, to_bytes=True) == blob
def test_get_role_response_serde() -> None:
target = Address(name="Alice")
content = {
"name": "Role Sample",
"can_triage_results": True,
"can_edit_settings": False,
"can_create_users": False,
"can_edit_roles": False,
"can_manage_infrastructure": True,
}
msg = GetRoleResponse(
address=target,
status_code=200,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_update_role_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
content = {
"role_id": "9a4f9dasd6",
"name": "Role Sample",
"can_triage_results": True,
"can_edit_settings": False,
"can_create_users": False,
"can_edit_roles": False,
"can_manage_infrastructure": True,
}
msg = UpdateRoleMessage(
address=target,
content=content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_signaling_answer_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
target_id = secrets.token_hex(nbytes=16)
host_id = secrets.token_hex(nbytes=16)
msg = SignalingAnswerMessage(
address=target,
payload="SDP",
host_metadata=node.get_metadata_for_client(),
target_peer=target_id,
host_peer=host_id,
)
msg_metadata = node.get_metadata_for_client()
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
msg2_metadata = msg2.host_metadata
assert msg.id == msg2.id
assert msg.address == target
assert msg.payload == msg2.payload
assert msg2.payload == "SDP"
assert msg2.host_peer == host_id
assert msg2.target_peer == target_id
assert msg == msg2
assert msg_metadata.name == msg2_metadata.name
assert msg_metadata.node == msg2_metadata.node
assert msg_metadata.id == msg2_metadata.id
def test_uid_default_serialization() -> None:
"""Tests that default UID serialization works as expected - to Protobuf"""
uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
blob = _serialize(obj=uid)
_ = _deserialize(blob=blob)
assert sy.serialize(uid) == blob
def test_get_dataset_response_serde() -> None:
target = Address(name="Alice")
content = {
"dataset":
["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"],
"description": "Dataset Description",
"tags": ["#x", "#data-sample"],
"pointable": True,
"read-permission": ["user-id1", "user-id2", "user-id3"],
"write-permission": ["user-id1", "user-id5", "user-id9"],
}
msg = GetDatasetResponse(
address=target,
status_code=200,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_create_initial_setup_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
request_content = {
"settings": {
"cloud-admin-token": "d84we35ad3a1d59a84sd9",
"cloud-credentials": "<cloud-credentials.pem>",
"infra": {
"autoscaling": True,
"triggers": {
"memory": "50",
"vCPU": "80"
}
},
}
}
msg = CreateInitialSetUpMessage(
address=target,
content=request_content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_default_serialization_and_deserialization() -> None:
"""Tests that default Address serde works as expected - to Protobuf"""
an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684))
obj = Address(
network=SpecificLocation(id=an_id),
domain=SpecificLocation(id=an_id),
device=SpecificLocation(id=an_id),
vm=SpecificLocation(id=an_id),
)
blob = sy.serialize(obj, to_proto=True)
assert sy.serialize(obj) == blob
assert obj == sy.deserialize(blob=blob)
def test_run_function_or_constructor_action_serde(
root_client: sy.VirtualMachineClient, ) -> None:
args = (
th.tensor([1, 2, 3]).send(root_client),
th.tensor([4, 5, 5]).send(root_client),
)
msg = RunFunctionOrConstructorAction(
path="torch.Tensor.add",
args=args,
kwargs={},
id_at_location=UID(),
address=root_client.address,
msg_id=UID(),
)
blob = sy.serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg2.path == msg.path
# FIXME this cannot be checked before we fix the Pointer serde problem (see _proto2object in Pointer)
# assert msg2.args == msg.args
assert msg2.kwargs == msg.kwargs
assert msg2.address == msg.address
assert msg2.id == msg.id
assert msg2.id_at_location == msg.id_at_location
def test_delete_worker_response_serde() -> None:
target = Address(name="Alice")
content = {
"settings": {
"cloud-admin-token": "d84we35ad3a1d59a84sd9",
"cloud-credentials": "<cloud-credentials.pem>",
"infra": {
"autoscaling": True,
"triggers": {
"memory": "50",
"vCPU": "80"
}
},
}
}
msg = GetSetUpResponse(
status_code=200,
address=target,
content=content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_create_dataset_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
request_content = {
"dataset":
["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"],
"description": "Dataset Description",
"tags": ["#x", "#data-sample"],
"pointable": True,
"read-permission": ["user-id1", "user-id2", "user-id3"],
"write-permission": ["user-id1", "user-id5", "user-id9"],
}
msg = CreateDatasetMessage(
address=target,
content=request_content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_get_all_data_requests_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"dataset-id": "68a465aer3adf",
"user-id": "user-id7",
"request-type": "read",
},
"a84ew64wq6e": {
"dataset-id": "98w4e54a6d",
"user-id": "user-id9",
"request-type": "write",
},
}
}
msg = GetRequestsResponse(
address=target,
status_code=200,
content=request_content,
)
blob = sy.serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def process_network_msgs() -> flask.Response:
data = flask.request.get_data()
obj_msg = _deserialize(blob=data, from_bytes=True)
if isinstance(obj_msg, SignedImmediateSyftMessageWithReply):
info(
f"Signaling server SignedImmediateSyftMessageWithReply: {obj_msg.message} watch"
)
reply = network.recv_immediate_msg_with_reply(msg=obj_msg)
r = Response(response=serialize(reply, to_bytes=True), status=200)
r.headers["Content-Type"] = "application/octet-stream"
return r
elif isinstance(obj_msg, SignedImmediateSyftMessageWithoutReply):
info(
f"Signaling server SignedImmediateSyftMessageWithoutReply: {obj_msg.message} watch"
)
network.recv_immediate_msg_without_reply(msg=obj_msg)
r = Response(status=200)
return r
else:
info(
f"Signaling server SignedImmediateSyftMessageWithoutReply: {obj_msg.message} watch"
)
network.recv_eventual_msg_without_reply(msg=obj_msg)
r = Response(status=200)
return r
def test_get_all_tensors_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"tensor": [1, 2, 3, 4, 5, 6],
"description": "Tensor description",
"tags": ["#x", "#data-sample"],
"pointable": True,
},
"a84ew64wq6e": {
"tensor": [9, 8, 2, 3, 5, 6],
"description": "Tensor sample description",
"tags": ["#y", "#label-sample"],
"pointable": True,
},
}
}
msg = GetTensorsResponse(
address=target,
status_code=200,
content=request_content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_serde_storable_obj() -> None:
id = UID()
data = th.Tensor([-1, -2, -3, -4])
description = "This is a dummy id"
tags = ["dummy", "test"]
obj1 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = th.Tensor([1, 2, 3, 4])
description = "This is a dummy tensor n1"
tags = ["dummy", "test"]
obj2 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = th.Tensor([10, 20, 30, 40])
description = "This is a dummy tensor n2"
tags = ["dummy", "test"]
obj3 = StorableObject(id=id, data=data, description=description, tags=tags)
id = UID()
data = [obj1, obj2, obj3]
description = "This is a dataset"
tags = ["dummy", "dataset"]
dataset_obj = Dataset(id=id, data=data, description=description, tags=tags)
blob = sy.serialize(obj=dataset_obj)
sy.deserialize(blob=blob)
def test_serde_matches() -> None:
"""Tests that the nested serde is reversible at all levels"""
# serial
sig_msg_blob = get_signed_message_bytes()
# deserial should be expected type
sig_msg = sy.deserialize(blob=sig_msg_blob, from_bytes=True)
assert type(sig_msg) == SignedImmediateSyftMessageWithoutReply
# reserial should be same as original fixture
comp_blob = serialize(sig_msg, to_bytes=True)
assert type(comp_blob) == bytes
assert comp_blob == sig_msg_blob
# now try sub message
msg = sig_msg.message
assert type(msg) == ReprMessage
# resign and the result should be the same
signing_key = get_signing_key()
sig_msg_comp = msg.sign(signing_key=signing_key)
assert type(sig_msg_comp) == SignedImmediateSyftMessageWithoutReply
assert type(sig_msg_comp) == type(sig_msg)
# make sure they have the same message id for comparison
sig_msg_comp._id = sig_msg.id
# identical (except the auto generated UID for the envelope)
assert sig_msg_comp == sig_msg
def metadata_route():
response_body = {
"metadata": serialize(get_node().get_metadata_for_client())
.SerializeToString()
.decode("ISO-8859-1")
}
return Response(json.dumps(response_body), status=200, mimetype="application/json")
def test_get_all_workers_response_serde() -> None:
target = Address(name="Alice")
request_content = {
"workers": {
"626sadaf631": {
"environment-name": "Heart Diseases Environment",
"owner": "user-id7",
"deployment-date": "05/12/2021",
},
"a84ew64wq6e": {
"worker-id": "eqw9e4a5d846",
"environment-name": "Brain Diseases Environment",
"owner": "user-id8",
"deployment-date": "15/12/2021",
},
}
}
msg = GetWorkersResponse(
address=target,
status_code=200,
content=request_content,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_update_worker_message_serde(node: sy.VirtualMachine) -> None:
target = Address(name="Alice")
content = {
"worker-id": "eqw9e4a5d846",
"settings": {
"instance-size": "t4g.large",
"vCPU": "2",
"network-bandwith": "5Gbps",
"vGPU": True,
},
}
msg = UpdateWorkerMessage(
address=target,
content=content,
reply_to=node.address,
)
blob = serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def message_integrity_test(msg: AbstractMessage, target: Address) -> None:
blob = sy.serialize(msg)
msg2 = sy.deserialize(blob=blob)
assert msg.id == msg2.id
assert msg.address == target
assert msg.content == msg2.content
assert msg == msg2
def test_plan_deterministic_bytes(
root_client: sy.VirtualMachineClient) -> None:
# TODO: https://github.com/OpenMined/PySyft/issues/5292
@make_plan
def add_plan(inp=th.zeros((3))) -> th.Tensor: # type: ignore
return inp + inp
@make_plan
def add_plan2(inp=th.zeros((3))) -> th.Tensor: # type: ignore
return inp + inp
plan_pointer = add_plan.send(root_client)
plan2_pointer = add_plan2.send(root_client)
plan1 = serialize(plan_pointer, to_bytes=True)
plan2 = serialize(plan2_pointer, to_bytes=True)
assert plan1 == plan2
def metadata() -> flask.Response:
metadata = network.get_metadata_for_client()
metadata_proto = serialize(metadata)
r = Response(
response=metadata_proto.SerializeToString(),
status=200,
)
r.headers["Content-Type"] = "application/octet-stream"
return r
def _object2proto(self) -> DeleteRequestResponse_PB:
"""Returns a protobuf serialization of self.
As a requirement of all objects which inherit from Serializable,
this method transforms the current object into the corresponding
Protobuf object so that it can be further serialized.
:return: returns a protobuf object
:rtype: SignalingOfferMessage_PB
.. note::
This method is purely an internal method. Please use serialize(object) or one of
the other public serialization methods if you wish to serialize an
object.
"""
return DeleteRequestResponse_PB(
msg_id=serialize(self.id),
address=serialize(self.address),
status_code=self.status_code,
content=json.dumps(self.content),
)
