def detail(worker: BaseWorker, simple_obj: tuple):
"""Takes a simplified SimpleTagger object, details it
and returns a SimpleTagger object.
Args:
worker (BaseWorker): The worker on which the
detail operation is carried out.
simple_obj (tuple): the simplified SubPipeline object.
Returns:
(SimpleTagger): The SimpleTagger object.
"""
# Unpack the simplified object
attribute, lookups, tag, default_tag, case_sensitive = simple_obj
# Detail each property
attribute = serde._detail(worker, attribute)
lookups = serde._detail(worker, lookups)
tag = serde._detail(worker, tag)
default_tag = serde._detail(worker, default_tag)
case_sensitive = serde._detail(worker, case_sensitive)
# Instantiate a SimpleTagger object
simple_tagger = SimpleTagger(
attribute=attribute,
lookups=lookups,
tag=tag,
default_tag=default_tag,
case_sensitive=case_sensitive,
)
return simple_tagger
def _detail_ndarray(
worker: AbstractWorker,
arr_representation: Tuple[bin, Tuple, str]) -> numpy.ndarray:
"""
This function reconstruct a numpy array from it's byte data, the shape and the dtype
by first loading the byte data with the appropiate dtype and then reshaping it into the
original shape
Args:
worker: the worker doing the deserialization
arr_representation (tuple): a tuple holding the byte representation, shape
and dtype of the array
Returns:
numpy.ndarray: a numpy array
Examples:
arr = _detail_ndarray(arr_representation)
"""
arr_shape = serde._detail(worker, arr_representation[1])
arr_dtype = serde._detail(worker, arr_representation[2])
res = numpy.frombuffer(arr_representation[0],
dtype=arr_dtype).reshape(arr_shape)
assert type(res) == numpy.ndarray
return res
def _detail_dictionary(worker: AbstractWorker,
my_dict: Tuple,
shallow: bool = False) -> Dict:
"""
This function is designed to operate in the opposite direction of
_simplify_dictionary. It takes a dictionary of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
Args:
worker: the worker doing the deserialization
my_dict (Tuple): a simplified dictionary of simple python objects (including binary).
Returns:
Dict: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
pieces = {}
# for dictionaries we want to detail both the key and the value
for key, value in my_dict:
detailed_key = serde._detail(worker, key)
if shallow:
pieces[detailed_key] = value
else:
detailed_value = serde._detail(worker, value)
pieces[detailed_key] = detailed_value
return pieces
def _detail_torch_parameter(worker: AbstractWorker,
param_tuple: tuple) -> torch.nn.Parameter:
"""
This function converts a serialized torch Parameter into a torch Parameter.
Args:
param_tuple (tuple): serialized obj of torch tensor. It's a tuple where
the first value is the ID and the second value is the binary for the
PyTorch data attribute et and third value is the requires_grad attr.
Returns:
torch.Parameter: a torch Parameter that was serialized
"""
param_id, tensor_ser, requires_grad, grad_ser = param_tuple
tensor = serde._detail(worker, tensor_ser)
if grad_ser is not None:
grad = _detail_torch_tensor(worker, grad_ser)
grad.garbage_collect_data = False
elif hasattr(tensor, "child") and isinstance(tensor.child, PointerTensor):
grad = tensor.attr("grad")
else:
grad = None
param = torch.nn.Parameter(tensor, requires_grad)
param.id = param_id
param.grad = grad
param.is_wrapper = isinstance(tensor, AbstractTensor) or tensor.is_wrapper
return param
def _detail_collection_set(worker: AbstractWorker,
my_collection: Tuple,
shallow: bool = False) -> Collection:
"""
This function is designed to operate in the opposite direction of
_simplify_collection. It takes a tuple of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
Args:
worker: the worker doing the deserialization
my_collection (Tuple): a tuple of simple python objects (including binary).
Returns:
Collection: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
# Don't detail contents
if shallow:
return set(my_collection)
pieces = []
# Step 1: deserialize each part of the collection
for part in my_collection:
detailed = serde._detail(worker, part)
pieces.append(detailed)
return set(pieces)
def _detail_numpy_number(
worker: AbstractWorker, nb_representation: Tuple[bin, Tuple, str]
) -> Union[numpy.int32, numpy.int64, numpy.float32, numpy.float64]:
"""
This function reconstruct a numpy number from it's byte data, dtype
by first loading the byte data with the appropiate dtype
Args:
worker: the worker doing the deserialization
np_representation (tuple): a tuple holding the byte representation
and dtype of the numpy number
Returns:
numpy.float or numpy.int: a numpy number
Examples:
nb = _detail_numpy_number(nb_representation)
"""
nb_dtype = serde._detail(worker, nb_representation[1])
nb = numpy.frombuffer(nb_representation[0], dtype=nb_dtype)[0]
assert type(nb) in [numpy.float32, numpy.float64, numpy.int32, numpy.int64]
return nb
def _detail_collection_tuple(worker: AbstractWorker,
my_tuple: Tuple,
shallow: bool = False) -> Tuple:
"""
This function is designed to operate in the opposite direction of
_simplify_collection. It takes a tuple of simple python objects
and iterates through it to determine whether objects in the collection
need to be converted into more advanced types. In particular, it
converts binary objects into torch Tensors where appropriate.
This is only applicable to tuples. They need special handling because
`msgpack` is encoding a tuple as a list.
Args:
worker: the worker doing the deserialization
my_tuple (Tuple): a collection of simple python objects (including binary).
Returns:
tuple: a collection of the same type as the input where the objects
in the collection have been detailed.
"""
# Don't detail contents
if shallow:
return my_tuple
pieces = []
# Step 1: deserialize each part of the collection
for part in my_tuple:
pieces.append(serde._detail(worker, part))
return tuple(pieces)
def simplified_tensor_deserializer(worker: AbstractWorker,
tensor_tuple: tuple) -> torch.Tensor:
""""Strategy to deserialize a simplified tensor into a Torch tensor"""
size, dtype, data_arr = serde._detail(worker, tensor_tuple)
tensor = torch.tensor(data_arr, dtype=TORCH_STR_DTYPE[dtype]).reshape(size)
return tensor
def test_plan_torch_function_no_args(workers):
bob, alice = workers["bob"], workers["alice"]
from syft.serde.msgpack import serde
@sy.func2plan(args_shape=[(1, )])
def serde_plan(x):
y = th.tensor([-1])
z = x + y
return z
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([1.0])
expected = serde_plan(t)
actual = serde_plan_detailed(t)
assert actual == expected == th.tensor([0.0])
@sy.func2plan(args_shape=[(1, )])
def serde_plan(x):
y = th.arange(3)
z = y + x
return z
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([1.0])
expected = serde_plan(t)
actual = serde_plan_detailed(t)
assert (actual == expected).all()
assert (actual == th.tensor([1, 2, 3])).all()
@sy.func2plan(args_shape=[(1, )])
def serde_plan(x):
th.manual_seed(14)
y = th.randint(2, size=(1, ), dtype=th.uint8)
y = y + 10
return y
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([1.0])
expected = serde_plan(t)
actual = serde_plan_detailed(t)
assert actual == expected and actual >= 10
def detail(worker: BaseWorker, simple_obj: tuple) -> "SubPipeline":
"""Takes a simplified SubPipeline object, details it along with
every pipe included in it and returns a SubPipeline object.
Args:
worker (BaseWorker): The worker on which the
detail operation is carried out.
Returns:
(SubPipeline): The SubPipeline object.
"""
# Unpack the simplified object
id, client_id, simple_pipe_names, simple_pipes = simple_obj
# Detail the client ID and the pipe names
id = serde._detail(worker, id)
client_id = serde._detail(worker, client_id)
pipe_names = serde._detail(worker, simple_pipe_names)
# Initialize a list of pipes
pipes = []
# Detail the pipes with the help of PySyft serde module
for simple_pipe in simple_pipes:
# Get the proto id of the pipe
proto_id, simple_pipe = simple_pipe
# Detail the simple_pipe to retriev the pipe object
pipe = msgpack_global_state.detailers[proto_id](worker,
simple_pipe)
pipes.append(pipe)
# Create the subpipeline object and set the client ID
subpipeline = SubPipeline(id=id, pipes=pipes)
# Set some key properties
subpipeline.client_id = client_id
subpipeline.owner = worker
subpipeline.pipe_names = pipe_names
return subpipeline
def _detail_torch_tensor(worker: AbstractWorker,
tensor_tuple: tuple) -> torch.Tensor:
"""
This function converts a serialized torch tensor into a torch tensor
using pickle.
Args:
tensor_tuple (bin): serialized obj of torch tensor. It's a tuple where
the first value is the ID, the second vlaue is the binary for the
PyTorch object, the third value is the chain of tensor abstractions,
and the fourth object is the chain of gradients (.grad.grad, etc.)
Returns:
torch.Tensor: a torch tensor that was serialized
"""
tensor_id, tensor_bin, chain, grad_chain, tags, description, serializer = tensor_tuple
tensor = _deserialize_tensor(worker, serde._detail(worker, serializer),
tensor_bin)
# note we need to do this explicitly because torch.load does not
# include .grad informatino
if grad_chain is not None:
tensor.grad = _detail_torch_tensor(worker, grad_chain)
initialize_tensor(hook=syft.torch.hook,
obj=tensor,
owner=worker,
id=tensor_id,
init_args=[],
init_kwargs={})
if chain is not None:
chain = serde._detail(worker, chain)
tensor.child = chain
tensor.is_wrapper = True
tensor.tags = serde._detail(worker, tags)
tensor.description = serde._detail(worker, description)
return tensor
def test_plan_execute_locally_ambiguous_output(workers):
bob, alice = workers["bob"], workers["alice"]
@sy.func2plan(args_shape=[(1, )])
def serde_plan(x):
x = x + x
y = x * 2
return x
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([2.3])
expected = serde_plan(t)
actual = serde_plan_detailed(t)
assert actual == expected
def test_plan_several_output_action(workers):
bob, alice = workers["bob"], workers["alice"]
@sy.func2plan(args_shape=[(4, )])
def serde_plan(x, torch=th):
y, z = torch.split(x, 2)
return y + z
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([1, 2, 3, 4])
expected = serde_plan_detailed(t)
actual = serde_plan_detailed(t)
assert (actual == th.tensor([4, 6])).all()
assert (actual == expected).all()
def test_plan_fixed_len_loop(workers):
bob, alice = workers["bob"], workers["alice"]
@sy.func2plan(args_shape=[(1, )])
def serde_plan(x):
for i in range(10):
x = x + 1
return x
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t = th.tensor([1.0])
expected = serde_plan_detailed(t)
actual = serde_plan_detailed(t)
assert actual == expected
def test_plan_with_comp(workers):
bob, alice = workers["bob"], workers["alice"]
@sy.func2plan(args_shape=[(2, ), (2, )])
def serde_plan(x, y):
z = x > y
return z
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t1 = th.tensor([2.0, 0.0])
t2 = th.tensor([1.0, 1.0])
expected = serde_plan_detailed(t1, t2)
actual = serde_plan_detailed(t1, t2)
assert (actual == expected).all()
def test_plan_execute_locally_ambiguous_input(workers):
bob, alice = workers["bob"], workers["alice"]
@sy.func2plan(args_shape=[(1, ), (1, ), (1, )])
def serde_plan(x, y, z):
a = x + x # 2
b = x + z # 4
c = y + z # 5
return c, b, a # 5, 4, 2
serde_plan_simplified = serde._simplify(bob, serde_plan)
serde_plan_detailed = serde._detail(bob, serde_plan_simplified)
t1, t2, t3 = th.tensor([1]), th.tensor([2]), th.tensor([3])
expected = serde_plan(t1, t2, t3)
actual = serde_plan_detailed(t1, t2, t3)
assert actual == expected
def _detail_torch_parameter(worker: AbstractWorker,
param_tuple: tuple) -> torch.nn.Parameter:
"""
This function converts a serialized torch Parameter into a torch Parameter.
Args:
param_tuple (tuple): serialized obj of torch tensor. It's a tuple where
the first value is the ID and the second value is the binary for the
PyTorch data attribute et and third value is the requires_grad attr.
Returns:
torch.Parameter: a torch Parameter that was serialized
"""
param_id, tensor_ser, requires_grad, grad_ser = param_tuple
tensor = serde._detail(worker, tensor_ser)
if grad_ser is not None:
grad = _detail_torch_tensor(worker, grad_ser)
grad.garbage_collect_data = False
elif hasattr(tensor, "child") and isinstance(tensor.child, PointerTensor):
grad = tensor.attr("grad")
else:
grad = None
param = torch.nn.Parameter(tensor, requires_grad)
param.id = param_id
param.grad = grad
param.is_wrapper = isinstance(tensor, AbstractTensor) or tensor.is_wrapper
# Note: should be
# param.origin = tensor.origin
# param.id_at_origin = tensor.id_at_origin
# but the wrapper is lost at serialisation because of the way we hook parameter.data
# TODO: fix serialisation of parameters (check in particular .child & .data) See #3214
# Below is just a fix:
param.origin = tensor.origin if hasattr(tensor, "origin") else None
param.id_at_origin = tensor.id_at_origin if hasattr(
tensor, "id_at_origin") else None
return param
def _detail_torch_device(worker: AbstractWorker, device_type: tuple) -> torch.device:
return torch.device(type=serde._detail(worker, device_type[0]))
