def test_itemgetter(self): from operator import itemgetter ser = CloudPickleSerializer() d = range(10) getter = itemgetter(1) getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d)) getter = itemgetter(0, 3) getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d))
def to_pytorch(self): """ Convert to pytorch model :return: a pytorch model """ new_weight = self.get_weights() assert (len(new_weight) == 1, "TorchModel's weights should be one tensor") # set weights m = CloudPickleSerializer.loads(CloudPickleSerializer, self.module_bytes) w = torch.Tensor(new_weight[0]) torch.nn.utils.vector_to_parameters(w, trainable_param(m)) # set named buffers new_extra_params = callZooFunc(self.bigdl_type, "getModuleExtraParameters", self.value) if len(new_extra_params) != 0: idx = 0 for named_buffer in m.named_buffers(): named_buffer[1].copy_( torch.reshape( torch.Tensor(new_extra_params[idx].to_ndarray()), named_buffer[1].size())) idx += 1 return m
def test_function_module_name(self): ser = CloudPickleSerializer() def func(x): return x func2 = ser.loads(ser.dumps(func)) self.assertEqual(func.__module__, func2.__module__)
def test_pickling_file_handles(self): # to be corrected with SPARK-11160 try: import xmlrunner # type: ignore[import] # noqa: F401 except ImportError: ser = CloudPickleSerializer() out1 = sys.stderr out2 = ser.loads(ser.dumps(out1)) self.assertEqual(out1, out2)
def test_pickling_file_handles(self): # to be corrected with SPARK-11160 try: import xmlrunner except ImportError: ser = CloudPickleSerializer() out1 = sys.stderr out2 = ser.loads(ser.dumps(out1)) self.assertEqual(out1, out2)
def test_attrgetter(self): from operator import attrgetter ser = CloudPickleSerializer() class C(object): def __getattr__(self, item): return item d = C() getter = attrgetter("a") getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d)) getter = attrgetter("a", "b") getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d)) d.e = C() getter = attrgetter("e.a") getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d)) getter = attrgetter("e.a", "e.b") getter2 = ser.loads(ser.dumps(getter)) self.assertEqual(getter(d), getter2(d))
def to_pytorch(self): """ Convert to pytorch model :return: a pytorch model """ new_weight = self.get_weights() assert (len(new_weight) == 1, "TorchModel's weights should be one tensor") m = CloudPickleSerializer.loads(CloudPickleSerializer, self.module_bytes) w = torch.Tensor(new_weight[0]) torch.nn.utils.vector_to_parameters(w, m.parameters()) return m
def test_function_module_name(self): ser = CloudPickleSerializer() func = lambda x: x func2 = ser.loads(ser.dumps(func)) self.assertEqual(func.__module__, func2.__module__)