def detail(worker: BaseWorker, simple_obj: tuple): """Takes a simplified SimpleTagger object, details it and returns a SimpleTagger object. Args: worker (BaseWorker): The worker on which the detail operation is carried out. simple_obj (tuple): the simplified SubPipeline object. Returns: (SimpleTagger): The SimpleTagger object. """ # Unpack the simplified object attribute, lookups, tag, default_tag, case_sensitive = simple_obj # Detail each property attribute = serde._detail(worker, attribute) lookups = serde._detail(worker, lookups) tag = serde._detail(worker, tag) default_tag = serde._detail(worker, default_tag) case_sensitive = serde._detail(worker, case_sensitive) # Instantiate a SimpleTagger object simple_tagger = SimpleTagger( attribute=attribute, lookups=lookups, tag=tag, default_tag=default_tag, case_sensitive=case_sensitive, ) return simple_tagger
def _detail_ndarray( worker: AbstractWorker, arr_representation: Tuple[bin, Tuple, str]) -> numpy.ndarray: """ This function reconstruct a numpy array from it's byte data, the shape and the dtype by first loading the byte data with the appropiate dtype and then reshaping it into the original shape Args: worker: the worker doing the deserialization arr_representation (tuple): a tuple holding the byte representation, shape and dtype of the array Returns: numpy.ndarray: a numpy array Examples: arr = _detail_ndarray(arr_representation) """ arr_shape = serde._detail(worker, arr_representation[1]) arr_dtype = serde._detail(worker, arr_representation[2]) res = numpy.frombuffer(arr_representation[0], dtype=arr_dtype).reshape(arr_shape) assert type(res) == numpy.ndarray return res
def _detail_dictionary(worker: AbstractWorker, my_dict: Tuple, shallow: bool = False) -> Dict: """ This function is designed to operate in the opposite direction of _simplify_dictionary. It takes a dictionary of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. Args: worker: the worker doing the deserialization my_dict (Tuple): a simplified dictionary of simple python objects (including binary). Returns: Dict: a collection of the same type as the input where the objects in the collection have been detailed. """ pieces = {} # for dictionaries we want to detail both the key and the value for key, value in my_dict: detailed_key = serde._detail(worker, key) if shallow: pieces[detailed_key] = value else: detailed_value = serde._detail(worker, value) pieces[detailed_key] = detailed_value return pieces
def _detail_torch_parameter(worker: AbstractWorker, param_tuple: tuple) -> torch.nn.Parameter: """ This function converts a serialized torch Parameter into a torch Parameter. Args: param_tuple (tuple): serialized obj of torch tensor. It's a tuple where the first value is the ID and the second value is the binary for the PyTorch data attribute et and third value is the requires_grad attr. Returns: torch.Parameter: a torch Parameter that was serialized """ param_id, tensor_ser, requires_grad, grad_ser = param_tuple tensor = serde._detail(worker, tensor_ser) if grad_ser is not None: grad = _detail_torch_tensor(worker, grad_ser) grad.garbage_collect_data = False elif hasattr(tensor, "child") and isinstance(tensor.child, PointerTensor): grad = tensor.attr("grad") else: grad = None param = torch.nn.Parameter(tensor, requires_grad) param.id = param_id param.grad = grad param.is_wrapper = isinstance(tensor, AbstractTensor) or tensor.is_wrapper return param
def _detail_collection_set(worker: AbstractWorker, my_collection: Tuple, shallow: bool = False) -> Collection: """ This function is designed to operate in the opposite direction of _simplify_collection. It takes a tuple of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. Args: worker: the worker doing the deserialization my_collection (Tuple): a tuple of simple python objects (including binary). Returns: Collection: a collection of the same type as the input where the objects in the collection have been detailed. """ # Don't detail contents if shallow: return set(my_collection) pieces = [] # Step 1: deserialize each part of the collection for part in my_collection: detailed = serde._detail(worker, part) pieces.append(detailed) return set(pieces)
def _detail_numpy_number( worker: AbstractWorker, nb_representation: Tuple[bin, Tuple, str] ) -> Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]: """ This function reconstruct a numpy number from it's byte data, dtype by first loading the byte data with the appropiate dtype Args: worker: the worker doing the deserialization np_representation (tuple): a tuple holding the byte representation and dtype of the numpy number Returns: numpy.float or numpy.int: a numpy number Examples: nb = _detail_numpy_number(nb_representation) """ nb_dtype = serde._detail(worker, nb_representation[1]) nb = numpy.frombuffer(nb_representation[0], dtype=nb_dtype)[0] assert type(nb) in [numpy.float32, numpy.float64, numpy.int32, numpy.int64] return nb
def _detail_collection_tuple(worker: AbstractWorker, my_tuple: Tuple, shallow: bool = False) -> Tuple: """ This function is designed to operate in the opposite direction of _simplify_collection. It takes a tuple of simple python objects and iterates through it to determine whether objects in the collection need to be converted into more advanced types. In particular, it converts binary objects into torch Tensors where appropriate. This is only applicable to tuples. They need special handling because `msgpack` is encoding a tuple as a list. Args: worker: the worker doing the deserialization my_tuple (Tuple): a collection of simple python objects (including binary). Returns: tuple: a collection of the same type as the input where the objects in the collection have been detailed. """ # Don't detail contents if shallow: return my_tuple pieces = [] # Step 1: deserialize each part of the collection for part in my_tuple: pieces.append(serde._detail(worker, part)) return tuple(pieces)
def simplified_tensor_deserializer(worker: AbstractWorker, tensor_tuple: tuple) -> torch.Tensor: """"Strategy to deserialize a simplified tensor into a Torch tensor""" size, dtype, data_arr = serde._detail(worker, tensor_tuple) tensor = torch.tensor(data_arr, dtype=TORCH_STR_DTYPE[dtype]).reshape(size) return tensor
def test_plan_torch_function_no_args(workers): bob, alice = workers["bob"], workers["alice"] from syft.serde.msgpack import serde @sy.func2plan(args_shape=[(1, )]) def serde_plan(x): y = th.tensor([-1]) z = x + y return z serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([1.0]) expected = serde_plan(t) actual = serde_plan_detailed(t) assert actual == expected == th.tensor([0.0]) @sy.func2plan(args_shape=[(1, )]) def serde_plan(x): y = th.arange(3) z = y + x return z serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([1.0]) expected = serde_plan(t) actual = serde_plan_detailed(t) assert (actual == expected).all() assert (actual == th.tensor([1, 2, 3])).all() @sy.func2plan(args_shape=[(1, )]) def serde_plan(x): th.manual_seed(14) y = th.randint(2, size=(1, ), dtype=th.uint8) y = y + 10 return y serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([1.0]) expected = serde_plan(t) actual = serde_plan_detailed(t) assert actual == expected and actual >= 10
def detail(worker: BaseWorker, simple_obj: tuple) -> "SubPipeline": """Takes a simplified SubPipeline object, details it along with every pipe included in it and returns a SubPipeline object. Args: worker (BaseWorker): The worker on which the detail operation is carried out. Returns: (SubPipeline): The SubPipeline object. """ # Unpack the simplified object id, client_id, simple_pipe_names, simple_pipes = simple_obj # Detail the client ID and the pipe names id = serde._detail(worker, id) client_id = serde._detail(worker, client_id) pipe_names = serde._detail(worker, simple_pipe_names) # Initialize a list of pipes pipes = [] # Detail the pipes with the help of PySyft serde module for simple_pipe in simple_pipes: # Get the proto id of the pipe proto_id, simple_pipe = simple_pipe # Detail the simple_pipe to retriev the pipe object pipe = msgpack_global_state.detailers[proto_id](worker, simple_pipe) pipes.append(pipe) # Create the subpipeline object and set the client ID subpipeline = SubPipeline(id=id, pipes=pipes) # Set some key properties subpipeline.client_id = client_id subpipeline.owner = worker subpipeline.pipe_names = pipe_names return subpipeline
def _detail_torch_tensor(worker: AbstractWorker, tensor_tuple: tuple) -> torch.Tensor: """ This function converts a serialized torch tensor into a torch tensor using pickle. Args: tensor_tuple (bin): serialized obj of torch tensor. It's a tuple where the first value is the ID, the second vlaue is the binary for the PyTorch object, the third value is the chain of tensor abstractions, and the fourth object is the chain of gradients (.grad.grad, etc.) Returns: torch.Tensor: a torch tensor that was serialized """ tensor_id, tensor_bin, chain, grad_chain, tags, description, serializer = tensor_tuple tensor = _deserialize_tensor(worker, serde._detail(worker, serializer), tensor_bin) # note we need to do this explicitly because torch.load does not # include .grad informatino if grad_chain is not None: tensor.grad = _detail_torch_tensor(worker, grad_chain) initialize_tensor(hook=syft.torch.hook, obj=tensor, owner=worker, id=tensor_id, init_args=[], init_kwargs={}) if chain is not None: chain = serde._detail(worker, chain) tensor.child = chain tensor.is_wrapper = True tensor.tags = serde._detail(worker, tags) tensor.description = serde._detail(worker, description) return tensor
def test_plan_execute_locally_ambiguous_output(workers): bob, alice = workers["bob"], workers["alice"] @sy.func2plan(args_shape=[(1, )]) def serde_plan(x): x = x + x y = x * 2 return x serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([2.3]) expected = serde_plan(t) actual = serde_plan_detailed(t) assert actual == expected
def test_plan_several_output_action(workers): bob, alice = workers["bob"], workers["alice"] @sy.func2plan(args_shape=[(4, )]) def serde_plan(x, torch=th): y, z = torch.split(x, 2) return y + z serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([1, 2, 3, 4]) expected = serde_plan_detailed(t) actual = serde_plan_detailed(t) assert (actual == th.tensor([4, 6])).all() assert (actual == expected).all()
def test_plan_fixed_len_loop(workers): bob, alice = workers["bob"], workers["alice"] @sy.func2plan(args_shape=[(1, )]) def serde_plan(x): for i in range(10): x = x + 1 return x serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t = th.tensor([1.0]) expected = serde_plan_detailed(t) actual = serde_plan_detailed(t) assert actual == expected
def test_plan_with_comp(workers): bob, alice = workers["bob"], workers["alice"] @sy.func2plan(args_shape=[(2, ), (2, )]) def serde_plan(x, y): z = x > y return z serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t1 = th.tensor([2.0, 0.0]) t2 = th.tensor([1.0, 1.0]) expected = serde_plan_detailed(t1, t2) actual = serde_plan_detailed(t1, t2) assert (actual == expected).all()
def test_plan_execute_locally_ambiguous_input(workers): bob, alice = workers["bob"], workers["alice"] @sy.func2plan(args_shape=[(1, ), (1, ), (1, )]) def serde_plan(x, y, z): a = x + x # 2 b = x + z # 4 c = y + z # 5 return c, b, a # 5, 4, 2 serde_plan_simplified = serde._simplify(bob, serde_plan) serde_plan_detailed = serde._detail(bob, serde_plan_simplified) t1, t2, t3 = th.tensor([1]), th.tensor([2]), th.tensor([3]) expected = serde_plan(t1, t2, t3) actual = serde_plan_detailed(t1, t2, t3) assert actual == expected
def _detail_torch_parameter(worker: AbstractWorker, param_tuple: tuple) -> torch.nn.Parameter: """ This function converts a serialized torch Parameter into a torch Parameter. Args: param_tuple (tuple): serialized obj of torch tensor. It's a tuple where the first value is the ID and the second value is the binary for the PyTorch data attribute et and third value is the requires_grad attr. Returns: torch.Parameter: a torch Parameter that was serialized """ param_id, tensor_ser, requires_grad, grad_ser = param_tuple tensor = serde._detail(worker, tensor_ser) if grad_ser is not None: grad = _detail_torch_tensor(worker, grad_ser) grad.garbage_collect_data = False elif hasattr(tensor, "child") and isinstance(tensor.child, PointerTensor): grad = tensor.attr("grad") else: grad = None param = torch.nn.Parameter(tensor, requires_grad) param.id = param_id param.grad = grad param.is_wrapper = isinstance(tensor, AbstractTensor) or tensor.is_wrapper # Note: should be # param.origin = tensor.origin # param.id_at_origin = tensor.id_at_origin # but the wrapper is lost at serialisation because of the way we hook parameter.data # TODO: fix serialisation of parameters (check in particular .child & .data) See #3214 # Below is just a fix: param.origin = tensor.origin if hasattr(tensor, "origin") else None param.id_at_origin = tensor.id_at_origin if hasattr( tensor, "id_at_origin") else None return param
def _detail_torch_device(worker: AbstractWorker, device_type: tuple) -> torch.device: return torch.device(type=serde._detail(worker, device_type[0]))