def python_encode(self, obj, private_local): # Case of basic types if isinstance(obj, (int, float, str)) or obj is None: return obj elif isinstance(obj, type(...)): return "..." elif isinstance(obj, np.ndarray): return obj.ser(private=private_local, to_json=False) # Dict elif isinstance(obj, dict): return { k: self.python_encode(v, private_local) for k, v in obj.items() } # Variable elif torch_utils.is_variable(obj): tail_object = torch_utils.find_tail_of_chain(obj) if self.retrieve_pointers and isinstance(tail_object, sy._PointerTensor): self.found_pointers.append(tail_object) return obj.ser(private=private_local, is_head=True) # Tensors elif torch_utils.is_tensor(obj): tail_object = torch_utils.find_tail_of_chain(obj) if self.retrieve_pointers and isinstance(tail_object, sy._PointerTensor): self.found_pointers.append(tail_object) return obj.ser(private=private_local) # sy._SyftTensor (Pointer, Local) # [Note: shouldn't be called on regular chain with end=tensorvar] elif torch_utils.is_syft_tensor(obj): tail_object = torch_utils.find_tail_of_chain(obj) if self.retrieve_pointers and isinstance(tail_object, sy._PointerTensor): self.found_pointers.append(tail_object) return obj.ser(private=private_local) # List elif isinstance(obj, list): return [self.python_encode(i, private_local) for i in obj] # np array elif isinstance(obj, np.ndarray): return obj.ser(private=private_local, to_json=False) # Iterables non json-serializable elif isinstance(obj, (tuple, set, bytearray, range)): key = get_serialized_key(obj) return {key: [self.python_encode(i, private_local) for i in obj]} # Slice elif isinstance(obj, slice): key = get_serialized_key(obj) return {key: {'args': [obj.start, obj.stop, obj.step]}} # Generator elif isinstance(obj, types.GeneratorType): logging.warning("Generator args can't be transmitted") return [] # worker elif isinstance(obj, (sy.SocketWorker, sy.VirtualWorker)): return {'__worker__': obj.id} # Else log the error else: raise ValueError('Unhandled type', type(obj))
def python_decode(self, dct): """ Is called on every dict found. We check if some keys correspond to special keywords referring to a type we need to re-cast (e.g. tuple, or torch Variable). """ # PLAN A: See if the dct object is not actually a dictionary and address # each case. if isinstance(dct, (int, str, float)): # a very strange special case if (dct == '...'): return ... return dct if isinstance(dct, (list, )): return [self.python_decode(o) for o in dct] if dct is None: return None if not isinstance(dct, dict): print(type(dct)) raise TypeError('Type not handled', dct) # PLAN B: If the dct object IS a dictionary, check to see if it has a "type" key if ('type' in dct): if dct['type'] == "numpy.array": # at first glance, the following if statement might seem a bit confusing # since the dct object is identical for both. Basically, the pointer object # is created here (on the receiving end of a message) as opposed to on the sending # side. We decide whether to use the dictionary to construct a pointer or the # actual tensor based on wehther self.acquire is true. Note that this changes # how dct['id'] is used. If creating an actual tensor, the tensor id is set to dct['id] # otherwise, id_at_location is set to be dct['id']. Similarly with dct['owner']. # if we intend to receive the tensor itself, construct an array if (self.acquire): return array(dct['data'], id=dct['id'], owner=self.worker) # if we intend to create a pointer, construct a pointer. Note that else: return array_ptr(dct['data'], owner=self.worker, location=self.worker.get_worker( dct['owner']), id_at_location=dct['id']) elif dct['type'] == 'numpy.array_ptr': return self.worker.get_obj(dct['id_at_location']) # Plan C: As a last resort, use a Regex to try to find a type somewhere. # TODO: Plan C should never be called - but is used extensively in PySyft's PyTorch integratio pat = re.compile('__(.+)__') for key, obj in dct.items(): if pat.search(key) is not None: obj_type = pat.search(key).group(1) # Case of a tensor if torch_utils.is_tensor(obj_type): o = torch.guard['syft.' + obj_type].deser({key: obj}, self.worker, self.acquire) return o # Case of a Variable elif torch_utils.is_variable(obj_type): return sy.Variable.deser({key: obj}, self.worker, self.acquire, is_head=True) # Case of a Syft tensor elif torch_utils.is_syft_tensor(obj_type): return sy._SyftTensor.deser_routing({key: obj}, self.worker, self.acquire) # Case of a iter type non json serializable elif obj_type in ('tuple', 'set', 'bytearray', 'range'): return eval(obj_type)([self.python_decode(o) for o in obj]) # Case of a slice elif obj_type == 'slice': return slice(*obj['args']) # Case of a worker elif obj_type == 'worker': return self.worker.get_worker(obj) else: raise TypeError('The special object type', obj_type, 'is not supported') else: dct[key] = self.python_decode(obj) return dct