def test_proto_deserialization() -> None: """Tests that default Address deserialization works as expected - from Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) loc = SpecificLocation(id=uid, name="Test Location") obj = Address( network=loc, domain=loc, device=loc, vm=loc, ) blob = Address.get_protobuf_schema()( has_network=True, has_domain=True, has_device=True, has_vm=True, network=sy.serialize(loc), domain=sy.serialize(loc), device=sy.serialize(loc), vm=sy.serialize(loc), ) obj2 = sy.deserialize(blob=blob, from_proto=True) assert obj == obj2
def _object2proto(self) -> Pointer_PB: """Returns a protobuf serialization of self. As a requirement of all objects which inherit from Serializable, this method transforms the current object into the corresponding Protobuf object so that it can be further serialized. :return: returns a protobuf object :rtype: Pointer_PB .. note:: This method is purely an internal method. Please use sy.serialize(object) or one of the other public serialization methods if you wish to serialize an object. """ return Pointer_PB( points_to_object_with_path=self.path_and_name, pointer_name=type(self).__name__, id_at_location=sy.serialize(self.id_at_location), location=sy.serialize(self.client.address), tags=self.tags, description=self.description, object_type=self.object_type, attribute_name=getattr(self, "attribute_name", ""), )
def test_default_serialization() -> None: """Tests that default SpecificLocation serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid, name="Test") blob = sy.serialize(obj, to_proto=True) assert sy.serialize(obj) == blob
def test_object_with_id_default_serialization() -> None: """Tests that default ObjectWithID serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = ObjectWithID(id=uid) blob = sy.serialize(obj, to_proto=True) assert sy.serialize(obj) == blob
def test_uid_proto_serialization() -> None: """Tests that proto UID serialization works as expected""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) blob = _serialize(obj=uid) assert sy.serialize(uid, to_proto=True) == blob assert sy.serialize(uid, to_proto=True) == blob assert sy.serialize(uid, to_proto=True) == blob
def test_uid_binary_serialization() -> None: """Tests that binary UID serializes as expected""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) blob = (b"\n\x18syft.core.common.uid.UID\x12\x12\n\x10\xfb\x1b\xb0" + b"g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14") assert sy.serialize(uid, to_bytes=True) == blob assert sy.serialize(uid, to_bytes=True) == blob assert sy.serialize(uid, to_bytes=True) == blob
def test_object_with_id_binary_serialization() -> None: """Tests that binary ObjectWithID serializes as expected""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = ObjectWithID(id=uid) blob = ( b"\n$syft.core.common.object.ObjectWithID\x12\x14\n\x12\n\x10\xfb\x1b\xb0" + b"g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14") assert sy.serialize(obj, to_bytes=True) == blob assert sy.serialize(obj, to_bytes=True) == blob assert sy.serialize(obj, to_bytes=True) == blob
def test_binary_serialization() -> None: """Tests that binary SpecificLocation serializes as expected""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid, name="Test") blob = ( b"\n/syft.core.io.location.specific.SpecificLocation\x12\x1a\n\x12\n\x10" + b"\xfb\x1b\xb0g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14\x12\x04Test") assert sy.serialize(obj, to_bytes=True) == blob assert sy.serialize(obj, to_bytes=True) == blob assert sy.serialize(obj, to_bytes=True) == blob
def test_get_role_response_serde() -> None: target = Address(name="Alice") content = { "name": "Role Sample", "can_triage_results": True, "can_edit_settings": False, "can_create_users": False, "can_edit_roles": False, "can_manage_infrastructure": True, } msg = GetRoleResponse( address=target, status_code=200, content=content, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_update_role_message_serde(node: sy.VirtualMachine) -> None: target = Address(name="Alice") content = { "role_id": "9a4f9dasd6", "name": "Role Sample", "can_triage_results": True, "can_edit_settings": False, "can_create_users": False, "can_edit_roles": False, "can_manage_infrastructure": True, } msg = UpdateRoleMessage( address=target, content=content, reply_to=node.address, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_signaling_answer_message_serde(node: sy.VirtualMachine) -> None: target = Address(name="Alice") target_id = secrets.token_hex(nbytes=16) host_id = secrets.token_hex(nbytes=16) msg = SignalingAnswerMessage( address=target, payload="SDP", host_metadata=node.get_metadata_for_client(), target_peer=target_id, host_peer=host_id, ) msg_metadata = node.get_metadata_for_client() blob = serialize(msg) msg2 = sy.deserialize(blob=blob) msg2_metadata = msg2.host_metadata assert msg.id == msg2.id assert msg.address == target assert msg.payload == msg2.payload assert msg2.payload == "SDP" assert msg2.host_peer == host_id assert msg2.target_peer == target_id assert msg == msg2 assert msg_metadata.name == msg2_metadata.name assert msg_metadata.node == msg2_metadata.node assert msg_metadata.id == msg2_metadata.id
def test_uid_default_serialization() -> None: """Tests that default UID serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) blob = _serialize(obj=uid) _ = _deserialize(blob=blob) assert sy.serialize(uid) == blob
def test_get_dataset_response_serde() -> None: target = Address(name="Alice") content = { "dataset": ["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"], "description": "Dataset Description", "tags": ["#x", "#data-sample"], "pointable": True, "read-permission": ["user-id1", "user-id2", "user-id3"], "write-permission": ["user-id1", "user-id5", "user-id9"], } msg = GetDatasetResponse( address=target, status_code=200, content=content, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_create_initial_setup_message_serde(node: sy.VirtualMachine) -> None: target = Address(name="Alice") request_content = { "settings": { "cloud-admin-token": "d84we35ad3a1d59a84sd9", "cloud-credentials": "<cloud-credentials.pem>", "infra": { "autoscaling": True, "triggers": { "memory": "50", "vCPU": "80" } }, } } msg = CreateInitialSetUpMessage( address=target, content=request_content, reply_to=node.address, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_default_serialization_and_deserialization() -> None: """Tests that default Address serde works as expected - to Protobuf""" an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = Address( network=SpecificLocation(id=an_id), domain=SpecificLocation(id=an_id), device=SpecificLocation(id=an_id), vm=SpecificLocation(id=an_id), ) blob = sy.serialize(obj, to_proto=True) assert sy.serialize(obj) == blob assert obj == sy.deserialize(blob=blob)
def test_run_function_or_constructor_action_serde( root_client: sy.VirtualMachineClient, ) -> None: args = ( th.tensor([1, 2, 3]).send(root_client), th.tensor([4, 5, 5]).send(root_client), ) msg = RunFunctionOrConstructorAction( path="torch.Tensor.add", args=args, kwargs={}, id_at_location=UID(), address=root_client.address, msg_id=UID(), ) blob = sy.serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg2.path == msg.path # FIXME this cannot be checked before we fix the Pointer serde problem (see _proto2object in Pointer) # assert msg2.args == msg.args assert msg2.kwargs == msg.kwargs assert msg2.address == msg.address assert msg2.id == msg.id assert msg2.id_at_location == msg.id_at_location
def test_delete_worker_response_serde() -> None: target = Address(name="Alice") content = { "settings": { "cloud-admin-token": "d84we35ad3a1d59a84sd9", "cloud-credentials": "<cloud-credentials.pem>", "infra": { "autoscaling": True, "triggers": { "memory": "50", "vCPU": "80" } }, } } msg = GetSetUpResponse( status_code=200, address=target, content=content, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_create_dataset_message_serde(node: sy.VirtualMachine) -> None: target = Address(name="Alice") request_content = { "dataset": ["<tensor_id>", "<tensor_id>", "<tensor_id>", "<tensor_id>"], "description": "Dataset Description", "tags": ["#x", "#data-sample"], "pointable": True, "read-permission": ["user-id1", "user-id2", "user-id3"], "write-permission": ["user-id1", "user-id5", "user-id9"], } msg = CreateDatasetMessage( address=target, content=request_content, reply_to=node.address, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_get_all_data_requests_response_serde() -> None: target = Address(name="Alice") request_content = { "workers": { "626sadaf631": { "dataset-id": "68a465aer3adf", "user-id": "user-id7", "request-type": "read", }, "a84ew64wq6e": { "dataset-id": "98w4e54a6d", "user-id": "user-id9", "request-type": "write", }, } } msg = GetRequestsResponse( address=target, status_code=200, content=request_content, ) blob = sy.serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def process_network_msgs() -> flask.Response: data = flask.request.get_data() obj_msg = _deserialize(blob=data, from_bytes=True) if isinstance(obj_msg, SignedImmediateSyftMessageWithReply): info( f"Signaling server SignedImmediateSyftMessageWithReply: {obj_msg.message} watch" ) reply = network.recv_immediate_msg_with_reply(msg=obj_msg) r = Response(response=serialize(reply, to_bytes=True), status=200) r.headers["Content-Type"] = "application/octet-stream" return r elif isinstance(obj_msg, SignedImmediateSyftMessageWithoutReply): info( f"Signaling server SignedImmediateSyftMessageWithoutReply: {obj_msg.message} watch" ) network.recv_immediate_msg_without_reply(msg=obj_msg) r = Response(status=200) return r else: info( f"Signaling server SignedImmediateSyftMessageWithoutReply: {obj_msg.message} watch" ) network.recv_eventual_msg_without_reply(msg=obj_msg) r = Response(status=200) return r
def test_get_all_tensors_response_serde() -> None: target = Address(name="Alice") request_content = { "workers": { "626sadaf631": { "tensor": [1, 2, 3, 4, 5, 6], "description": "Tensor description", "tags": ["#x", "#data-sample"], "pointable": True, }, "a84ew64wq6e": { "tensor": [9, 8, 2, 3, 5, 6], "description": "Tensor sample description", "tags": ["#y", "#label-sample"], "pointable": True, }, } } msg = GetTensorsResponse( address=target, status_code=200, content=request_content, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_serde_storable_obj() -> None: id = UID() data = th.Tensor([-1, -2, -3, -4]) description = "This is a dummy id" tags = ["dummy", "test"] obj1 = StorableObject(id=id, data=data, description=description, tags=tags) id = UID() data = th.Tensor([1, 2, 3, 4]) description = "This is a dummy tensor n1" tags = ["dummy", "test"] obj2 = StorableObject(id=id, data=data, description=description, tags=tags) id = UID() data = th.Tensor([10, 20, 30, 40]) description = "This is a dummy tensor n2" tags = ["dummy", "test"] obj3 = StorableObject(id=id, data=data, description=description, tags=tags) id = UID() data = [obj1, obj2, obj3] description = "This is a dataset" tags = ["dummy", "dataset"] dataset_obj = Dataset(id=id, data=data, description=description, tags=tags) blob = sy.serialize(obj=dataset_obj) sy.deserialize(blob=blob)
def test_serde_matches() -> None: """Tests that the nested serde is reversible at all levels""" # serial sig_msg_blob = get_signed_message_bytes() # deserial should be expected type sig_msg = sy.deserialize(blob=sig_msg_blob, from_bytes=True) assert type(sig_msg) == SignedImmediateSyftMessageWithoutReply # reserial should be same as original fixture comp_blob = serialize(sig_msg, to_bytes=True) assert type(comp_blob) == bytes assert comp_blob == sig_msg_blob # now try sub message msg = sig_msg.message assert type(msg) == ReprMessage # resign and the result should be the same signing_key = get_signing_key() sig_msg_comp = msg.sign(signing_key=signing_key) assert type(sig_msg_comp) == SignedImmediateSyftMessageWithoutReply assert type(sig_msg_comp) == type(sig_msg) # make sure they have the same message id for comparison sig_msg_comp._id = sig_msg.id # identical (except the auto generated UID for the envelope) assert sig_msg_comp == sig_msg
def metadata_route(): response_body = { "metadata": serialize(get_node().get_metadata_for_client()) .SerializeToString() .decode("ISO-8859-1") } return Response(json.dumps(response_body), status=200, mimetype="application/json")
def test_get_all_workers_response_serde() -> None: target = Address(name="Alice") request_content = { "workers": { "626sadaf631": { "environment-name": "Heart Diseases Environment", "owner": "user-id7", "deployment-date": "05/12/2021", }, "a84ew64wq6e": { "worker-id": "eqw9e4a5d846", "environment-name": "Brain Diseases Environment", "owner": "user-id8", "deployment-date": "15/12/2021", }, } } msg = GetWorkersResponse( address=target, status_code=200, content=request_content, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_update_worker_message_serde(node: sy.VirtualMachine) -> None: target = Address(name="Alice") content = { "worker-id": "eqw9e4a5d846", "settings": { "instance-size": "t4g.large", "vCPU": "2", "network-bandwith": "5Gbps", "vGPU": True, }, } msg = UpdateWorkerMessage( address=target, content=content, reply_to=node.address, ) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def message_integrity_test(msg: AbstractMessage, target: Address) -> None: blob = sy.serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.address == target assert msg.content == msg2.content assert msg == msg2
def test_plan_deterministic_bytes( root_client: sy.VirtualMachineClient) -> None: # TODO: https://github.com/OpenMined/PySyft/issues/5292 @make_plan def add_plan(inp=th.zeros((3))) -> th.Tensor: # type: ignore return inp + inp @make_plan def add_plan2(inp=th.zeros((3))) -> th.Tensor: # type: ignore return inp + inp plan_pointer = add_plan.send(root_client) plan2_pointer = add_plan2.send(root_client) plan1 = serialize(plan_pointer, to_bytes=True) plan2 = serialize(plan2_pointer, to_bytes=True) assert plan1 == plan2
def metadata() -> flask.Response: metadata = network.get_metadata_for_client() metadata_proto = serialize(metadata) r = Response( response=metadata_proto.SerializeToString(), status=200, ) r.headers["Content-Type"] = "application/octet-stream" return r
def _object2proto(self) -> DeleteRequestResponse_PB: """Returns a protobuf serialization of self. As a requirement of all objects which inherit from Serializable, this method transforms the current object into the corresponding Protobuf object so that it can be further serialized. :return: returns a protobuf object :rtype: SignalingOfferMessage_PB .. note:: This method is purely an internal method. Please use serialize(object) or one of the other public serialization methods if you wish to serialize an object. """ return DeleteRequestResponse_PB( msg_id=serialize(self.id), address=serialize(self.address), status_code=self.status_code, content=json.dumps(self.content), )