def create_dataset(df_json: dict) -> dict: _json = deepcopy(df_json) storage = DiskObjectStore(db) mapping = [] # Separate CSV from metadata for el in _json["tensors"].copy(): _id = UID() _json["tensors"][el]["id"] = str(_id.value) mapping.append((el, _id, _json["tensors"][el].pop("content", None))) # Ensure we have same ID in metadata and dataset df_id = UID() _json["id"] = str(df_id.value) # Create storables from UID/CSV. Update metadata storables = [] for idx, (name, _id, raw_file) in enumerate(mapping): _tensor = pd.read_csv(StringIO(raw_file)) _tensor = th.tensor(_tensor.values.astype(np.float32)) _json["tensors"][name]["shape"] = [int(x) for x in _tensor.size()] _json["tensors"][name]["dtype"] = "{}".format(_tensor.dtype) storage.__setitem__(_id, StorableObject(id=_id, data=_tensor)) # Ensure we have same ID in metadata and dataset db.session.add( DatasetGroup(bin_object=str(_id.value), dataset=str(df_id.value))) json_obj = JsonObject(id=_json["id"], binary=_json) metadata = get_metadata(db) metadata.length += 1 db.session.add(json_obj) db.session.commit() return _json
def test_run_function_or_constructor_action_serde() -> None: alice = sy.VirtualMachine(name="alice") alice_client = alice.get_client() args = ( th.tensor([1, 2, 3]).send(alice_client), th.tensor([4, 5, 5]).send(alice_client), ) msg = RunFunctionOrConstructorAction( path="torch.Tensor.add", args=args, kwargs={}, id_at_location=UID(), address=alice_client.address, msg_id=UID(), ) blob = msg.serialize() msg2 = sy.deserialize(blob=blob) assert msg2.path == msg.path # FIXME this cannot be checked before we fix the Pointer serde problem (see _proto2object in Pointer) # assert msg2.args == msg.args assert msg2.kwargs == msg.kwargs assert msg2.address == msg.address assert msg2.id == msg.id assert msg2.id_at_location == msg.id_at_location
def accept_or_deny_request(msg: AcceptOrDenyRequestMessage, node: AbstractNode, verify_key: VerifyKey): if verify_key is None: raise ValueError( "Can't process AcceptOrDenyRequestService without a specified verification key" ) _msg: AcceptOrDenyRequestMessage = validate_type( msg, AcceptOrDenyRequestMessage) current_user = node.users.first(verify_key=verify_key.encode( encoder=HexEncoder).decode("utf-8")) _req = node.data_requests.first(id=str(_msg.request_id.value)) _can_triage_request = node.users.can_triage_requests( user_id=current_user.id) if _msg.accept: if _req and _can_triage_request: tmp_obj = node.store[UID.from_string(_req.object_id)] tmp_obj.read_permissions[VerifyKey(_req.verify_key.encode("utf-8"), encoder=HexEncoder)] = _req.id node.store[UID.from_string(_req.object_id)] = tmp_obj node.data_requests.set(request_id=_req.id, status="accepted") else: _req_owner = current_user.verify_key == _req.verify_key if _req and (_can_triage_request or _req_owner): node.data_requests.set(request_id=_req.id, status="denied")
def _construct_address() -> Address: """Helper method to construct an Address""" return Address( network=SpecificLocation(id=UID()), domain=SpecificLocation(id=UID()), device=SpecificLocation(id=UID()), vm=SpecificLocation(id=UID()), )
def test_default_deserialization() -> None: """Tests that default SpecificLocation deserialization works as expected - from Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid, name="Test") blob = SpecificLocation.get_protobuf_schema()(id=uid.serialize()) obj2 = sy.deserialize(blob=blob) assert obj == obj2
def update_request_msg( msg: DeleteRequestMessage, node: AbstractNode, verify_key: VerifyKey, ) -> DeleteRequestResponse: # Get Payload Content request_id = msg.content.get("request_id", None) status = msg.content.get("status", None) current_user_id = msg.content.get("current_user", None) users = node.users if not current_user_id: current_user_id = users.first(verify_key=verify_key.encode( encoder=HexEncoder).decode("utf-8")).id # Check if status field is empty missing_paramaters = not status if missing_paramaters: raise MissingRequestKeyError( message="Invalid request payload, empty fields (status)!") _req = node.data_requests.first(id=request_id) if not _req: raise RequestError if status not in ["accepted", "denied"]: raise InvalidParameterValueError( message='Request status should be either "accepted" or "denied"') _can_triage_request = node.users.can_triage_requests( user_id=current_user_id) _current_user_key = verify_key.encode(encoder=HexEncoder).decode("utf-8") _req_owner = _current_user_key == _req.verify_key if status == "accepted" and _can_triage_request: tmp_obj = node.store[UID.from_string(_req.object_id)] tmp_obj.read_permissions[VerifyKey(_req.verify_key.encode("utf-8"), encoder=HexEncoder)] = _req.id node.store[UID.from_string(_req.object_id)] = tmp_obj node.data_requests.set(request_id=_req.id, status=status) elif status == "denied" and (_can_triage_request or _req_owner): node.data_requests.set(request_id=_req.id, status=status) else: raise AuthorizationError( "You're not allowed to update Request information!") return DeleteRequestResponse( address=msg.reply_to, status_code=200, content={"msg": "Request Updated!"}, )
def test_get_object_action_serde() -> None: msg = GetObjectAction(id_at_location=UID(), address=Address(), reply_to=Address(), msg_id=UID()) blob = serialize(msg) msg2 = sy.deserialize(blob=blob) assert msg.id == msg2.id assert msg.id_at_location == msg2.id_at_location assert msg.address == msg2.address assert msg.reply_to == msg2.reply_to
def test_proto_serialization() -> None: """Tests that default SpecificLocation serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid, name="Test") blob = SpecificLocation.get_protobuf_schema()(id=uid.serialize(), name="Test") assert obj.proto() == blob assert obj.to_proto() == blob assert obj.serialize(to_proto=True) == blob
def test_vm_and_vm_id_property_methods() -> None: """Unit test for Address.vm and Address.vm_id methods""" # Test getter vm = SpecificLocation(id=UID()) address_with_vm = Address( network=SpecificLocation(id=UID()), domain=SpecificLocation(id=UID()), device=SpecificLocation(id=UID()), vm=vm, ) # Test device getter assert address_with_vm.vm == vm # Test device setter an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) new_vm = SpecificLocation(id=an_id) address_with_vm.vm = new_vm assert address_with_vm.vm == new_vm # Test domain_id getter address_without_vm = Address( network=SpecificLocation(id=UID()), domain=SpecificLocation(id=UID()), device=SpecificLocation(id=UID()), ) assert address_with_vm.vm_id == an_id assert address_without_vm.vm_id is None
def test_domain_and_domain_id_property_methods() -> None: """Unit test for Address.domain and Address.domain_id methods""" # Test getter domain = SpecificLocation(id=UID()) address_with_domain = Address( network=SpecificLocation(id=UID()), domain=domain, device=SpecificLocation(id=UID()), vm=SpecificLocation(id=UID()), ) # Test domain getter assert address_with_domain.domain == domain # Test domain setter an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) new_domain = SpecificLocation(id=an_id) address_with_domain.domain = new_domain assert address_with_domain.domain == new_domain # Test domain_id getter address_without_domain = Address( network=SpecificLocation(id=UID()), device=SpecificLocation(id=UID()), vm=SpecificLocation(id=UID()), ) assert address_with_domain.domain_id == an_id assert address_without_domain.domain_id is None
def test_keys(client, database, cleanup): disk_store = DiskObjectStore(database) a_id = UID() id1 = UID() id2 = UID() storable1 = StorableObject(id=id1, data=tensor1) disk_store.__setitem__(id1, storable1) storable2 = StorableObject(id=id2, data=tensor2) disk_store.__setitem__(id2, storable2) keys = disk_store.keys() assert any(id1 == k for k in keys) assert any(id2 == k for k in keys) assert len(keys) == 2
def test_address_property_method() -> None: """Unit tests for Address.address property method""" address = Address( network=SpecificLocation(id=UID()), domain=SpecificLocation(id=UID()), device=SpecificLocation(id=UID()), vm=SpecificLocation(id=UID()), ) returned_address = address.address assert isinstance(returned_address, Address) assert returned_address.network == address.network assert returned_address.domain == address.domain assert returned_address.device == address.device assert returned_address.vm == address.vm
def create_request( self, user_id, user_name, object_id, reason, request_type, verify_key=None, tags=[], object_type="", ): date = datetime.now() return self.register( id=str(UID().value), user_id=user_id, user_name=user_name, object_id=object_id, date=date, reason=reason, request_type=request_type, verify_key=verify_key, tags=tags, object_type=object_type, )
def test_delete(client, database, cleanup): disk_store = DiskObjectStore(database) _id = UID() storable = StorableObject(id=_id, data=tensor1) disk_store.__setitem__(_id, storable) bin_obj = database.session.query(BinObject).get(str(_id.value)) metadata = (database.session.query(ObjectMetadata).filter_by( obj=str(_id.value)).one()) assert bin_obj is not None assert th.all(th.eq(bin_obj.object, tensor1)) assert metadata is not None assert metadata.tags == [] assert metadata.description == "" assert metadata.read_permissions == {} assert metadata.search_permissions == {} disk_store.delete(_id) assert database.session.query(BinObject).get(str(_id.value)) is None with pytest.raises(NoResultFound) as e_info: database.session.query(ObjectMetadata).filter_by( obj=str(_id.value)).one()
def test_clear(client, database, cleanup): disk_store = DiskObjectStore(database) _id = UID() storable = StorableObject(id=_id, data=tensor1) disk_store.__setitem__(_id, storable) bin_obj = database.session.query(BinObject).get(str(_id.value)) metadata = (database.session.query(ObjectMetadata).filter_by( obj=str(_id.value)).one()) assert bin_obj is not None assert th.all(th.eq(bin_obj.object, tensor1)) assert metadata is not None assert metadata.tags == [] assert metadata.description == "" assert metadata.read_permissions == {} assert metadata.search_permissions == {} retrieved = disk_store.get_object(_id) assert th.all(th.eq(retrieved.data, tensor1)) assert retrieved.id == _id disk_store.clear() assert database.session.query(BinObject).count() == 0 assert database.session.query(ObjectMetadata).count() == 0
def test_proto_serialization() -> None: """Tests that default Address serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) loc = SpecificLocation(id=uid, name="Test Location") obj = Address( name="Test Address", network=loc, domain=loc, device=loc, vm=loc, ) blob = Address.get_protobuf_schema()( name="Test Address", has_network=True, has_domain=True, has_device=True, has_vm=True, network=sy.serialize(loc), domain=sy.serialize(loc), device=sy.serialize(loc), vm=sy.serialize(loc), ) assert sy.serialize(obj, to_proto=True) == blob assert sy.serialize(obj, to_proto=True) == blob assert sy.serialize(obj, to_proto=True) == blob
def test_dict_creation() -> None: d1 = {String("t1"): 1, String("t2"): 2} dict1 = Dict(d1) assert type(getattr(dict1, "id", None)) is UID d2 = dict({"t1": 1, "t2": 2}) dict2 = Dict(d2) dict2._id = UID() assert type(getattr(dict2, "id", None)) is UID d3 = UserDict({"t1": 1, "t2": 2}) dict3 = Dict(**d3) assert type(getattr(dict3, "id", None)) is UID assert dict1.keys() == dict2.keys() assert dict1.keys() == dict3.keys() # ValuesView uses object.__eq__ # https://stackoverflow.com/questions/34312674/why-are-the-values-of-an-ordereddict-not-equal assert dict1.values() != dict2.values() assert dict1.values() != dict3.values() assert dict1.items() == dict2.items() assert dict1.items() == dict3.items() it = list(iter(dict2.values())) assert len(it) == 2 assert type(it) is list
def get_tensor_msg( msg: GetTensorMessage, node: AbstractNode, ) -> GetTensorResponse: try: payload = msg.content # Retrieve the dataset from node.store key = UID.from_string(value=payload["tensor_id"]) tensor = node.store[key] return GetTensorResponse( address=msg.reply_to, status_code=200, content={ "tensor": { "id": payload["tensor_id"], "tags": tensor.tags, "description": tensor.description, } }, ) except Exception as e: return GetTensorResponse( address=msg.reply_to, status_code=200, content={"error": str(e)}, )
def test_proto_deserialization() -> None: """Tests that default Address deserialization works as expected - from Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) loc = SpecificLocation(id=uid, name="Test Location") obj = Address( network=loc, domain=loc, device=loc, vm=loc, ) blob = Address.get_protobuf_schema()( has_network=True, has_domain=True, has_device=True, has_vm=True, network=loc.serialize(), domain=loc.serialize(), device=loc.serialize(), vm=loc.serialize(), ) obj2 = sy.deserialize(blob=blob, from_proto=True) assert obj == obj2
def test_get_objects_of_type(client, database, cleanup): disk_store = DiskObjectStore(database) a_id = UID() id1 = UID() id2 = UID() storable1 = StorableObject(id=id1, data=tensor1) disk_store.__setitem__(id1, storable1) storable2 = StorableObject(id=id2, data=tensor2) disk_store.__setitem__(id2, storable2) selected = disk_store.get_objects_of_type(th.Tensor) selected_data = [x.data for x in selected] assert any(th.all(th.eq(tensor1, d_)) for d_ in selected_data) assert any(th.all(th.eq(tensor2, d_)) for d_ in selected_data) assert len(selected_data) == 2
def test_values(client, database, cleanup): disk_store = DiskObjectStore(database) a_id = UID() id1 = UID() id2 = UID() storable1 = StorableObject(id=id1, data=tensor1) disk_store.__setitem__(id1, storable1) storable2 = StorableObject(id=id2, data=tensor2) disk_store.__setitem__(id2, storable2) values = disk_store.values() values_data = [v.data for v in values] assert any(th.all(th.eq(tensor1, v)) for v in values_data) assert any(th.all(th.eq(tensor2, v)) for v in values_data) assert len(values_data) == 2
def test_to_string() -> None: """Tests that SpecificLocation generates an intuitive string.""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid) assert str(obj) == "<SpecificLocation: fb1bb0675bb74c49becee700ab0a1514>" assert obj.__repr__( ) == "<SpecificLocation: fb1bb0675bb74c49becee700ab0a1514>"
def test_specific_location_init_with_specific_id() -> None: """Test that SpecificLocation will use the ID you pass into the constructor""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) loc = SpecificLocation(id=uid) assert loc.id == uid
def test_default_serialization() -> None: """Tests that default SpecificLocation serialization works as expected - to Protobuf""" uid = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) obj = SpecificLocation(id=uid, name="Test") blob = sy.serialize(obj, to_proto=True) assert sy.serialize(obj) == blob
def test_update(): OrderedDict = SyOrderedDict with pytest.raises(TypeError): OrderedDict().update([("a", 1), ("b", 2)], None) # too many args pairs = [("a", 1), ("b", 2), ("c", 3), ("d", 4), ("e", 5)] od = OrderedDict() od.update(dict(pairs)) assertEqual(sorted(od.items()), pairs) # dict input od = OrderedDict() od.update(**dict(pairs)) assertEqual(sorted(od.items()), pairs) # kwds input od = OrderedDict() od.update(pairs) assertEqual(list(od.items()), pairs) # pairs input od = OrderedDict() od.update([("a", 1), ("b", 2), ("c", 9), ("d", 4)], c=3, e=5) assertEqual(list(od.items()), pairs) # mixed input # Issue 9137: Named argument called 'other' or '' # shouldn't be treated specially. od = OrderedDict() od = OrderedDict() od.update(other={}) assertEqual(list(od.items()), [("other", {})]) # Make sure that direct calls to update do not clear previous contents # add that updates items are not moved to the end d = OrderedDict([("a", 1), ("b", 2), ("c", 3), ("d", 44), ("e", 55)]) d.update([("e", 5), ("f", 6)], g=7, d=4) assertEqual( list(d.items()), [("a", 1), ("b", 2), ("c", 3), ("d", 4), ("e", 5), ("f", 6), ("g", 7)], ) pytest.raises(TypeError, OrderedDict().update, 42) pytest.raises(TypeError, OrderedDict().update, (), ()) d = OrderedDict( [("a", 1), ("b", 2), ("c", 3), ("d", 44), ("e", 55)], _id=UID.from_string(value="{12345678-1234-5678-1234-567812345678}"), ) assert d.id.__eq__( UID.from_string(value="{12345678-1234-5678-1234-567812345678}"))
def test_target_id_property_method_with_a_return() -> None: """Unit test for Address.target_id method""" network = SpecificLocation(id=UID()) domain = SpecificLocation(id=UID()) device = SpecificLocation(id=UID()) vm = SpecificLocation(id=UID()) address = Address( network=network, domain=domain, device=device, vm=vm, ) assert address.target_id == vm address.vm = None assert address.target_id == device address.device = None assert address.target_id == domain address.domain = None assert address.target_id == network
def test_plan_execution(client: sy.VirtualMachineClient) -> None: tensor_pointer1 = th.tensor([1, 2, 3]).send(client) tensor_pointer2 = th.tensor([4, 5, 6]).send(client) tensor_pointer3 = th.tensor([7, 8, 9]).send(client) result_tensor_pointer1 = th.tensor([0, 0, 0]).send(client) result_tensor_pointer2 = th.tensor([0, 0, 0]).send(client) result1_uid = result_tensor_pointer1.id_at_location result2_uid = result_tensor_pointer2.id_at_location a1 = RunClassMethodAction( path="torch.Tensor.add", _self=tensor_pointer1, args=[tensor_pointer2], kwargs={}, id_at_location=result1_uid, address=Address(), msg_id=UID(), ) a2 = RunClassMethodAction( path="torch.Tensor.add", _self=result_tensor_pointer1, args=[tensor_pointer3], kwargs={}, id_at_location=result2_uid, address=Address(), msg_id=UID(), ) plan = Plan([a1, a2]) plan_pointer = plan.send(client) plan_pointer() expected_tensor1 = th.tensor([5, 7, 9]) expected_tensor2 = th.tensor([12, 15, 18]) assert all(expected_tensor1 == result_tensor_pointer1.get()) assert all(expected_tensor2 == result_tensor_pointer2.get())
def test_target_emoji_method() -> None: """Unit test for Address.target_emoji method""" an_id = UID(value=uuid.UUID(int=333779996850170035686993356951732753684)) address = Address( network=SpecificLocation(id=an_id), domain=SpecificLocation(id=an_id), device=SpecificLocation(id=an_id), vm=SpecificLocation(id=an_id), ) assert address.target_emoji() == "@<UID:🙍🛖>"
def _gen_address_kwargs() -> list: """ Helper method to generate pre-ordered arguments for initializing an Address instance. There are at least 3 arguments, all taken from 'vm', 'device', 'domain', 'network'. """ # the order matches the _gen_icons below all_combos = [] for combination in combinations(ARGUMENTS, 3): all_combos.append(list(combination)) all_combos.append(ARGUMENTS) return [{key: SpecificLocation(id=UID()) for key in combo} for combo in all_combos]
def test_binary_deserialization() -> None: """Test that binary SpecificLocation deserialization works as expected""" blob = ( b"\n/syft.core.io.location.specific.SpecificLocation\x12\x1a\n\x12\n\x10" + b"\xfb\x1b\xb0g[\xb7LI\xbe\xce\xe7\x00\xab\n\x15\x14\x12\x04Test") obj = sy.deserialize(blob=blob, from_bytes=True) assert obj == SpecificLocation( id=UID(value=uuid.UUID(int=333779996850170035686993356951732753684)), name="Test", )