def test_itemgetter(self):
from operator import itemgetter
ser = CloudPickleSerializer()
d = range(10)
getter = itemgetter(1)
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = itemgetter(0, 3)
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
def test_itemgetter(self):
from operator import itemgetter
ser = CloudPickleSerializer()
d = range(10)
getter = itemgetter(1)
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = itemgetter(0, 3)
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
def to_pytorch(self):
"""
Convert to pytorch model
:return: a pytorch model
"""
new_weight = self.get_weights()
assert (len(new_weight) == 1,
"TorchModel's weights should be one tensor")
# set weights
m = CloudPickleSerializer.loads(CloudPickleSerializer,
self.module_bytes)
w = torch.Tensor(new_weight[0])
torch.nn.utils.vector_to_parameters(w, trainable_param(m))
# set named buffers
new_extra_params = callZooFunc(self.bigdl_type,
"getModuleExtraParameters", self.value)
if len(new_extra_params) != 0:
idx = 0
for named_buffer in m.named_buffers():
named_buffer[1].copy_(
torch.reshape(
torch.Tensor(new_extra_params[idx].to_ndarray()),
named_buffer[1].size()))
idx += 1
return m
def test_function_module_name(self):
ser = CloudPickleSerializer()
def func(x):
return x
func2 = ser.loads(ser.dumps(func))
self.assertEqual(func.__module__, func2.__module__)
def test_pickling_file_handles(self):
# to be corrected with SPARK-11160
try:
import xmlrunner # type: ignore[import] # noqa: F401
except ImportError:
ser = CloudPickleSerializer()
out1 = sys.stderr
out2 = ser.loads(ser.dumps(out1))
self.assertEqual(out1, out2)
def test_pickling_file_handles(self):
# to be corrected with SPARK-11160
try:
import xmlrunner
except ImportError:
ser = CloudPickleSerializer()
out1 = sys.stderr
out2 = ser.loads(ser.dumps(out1))
self.assertEqual(out1, out2)
def test_attrgetter(self):
from operator import attrgetter
ser = CloudPickleSerializer()
class C(object):
def __getattr__(self, item):
return item
d = C()
getter = attrgetter("a")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = attrgetter("a", "b")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
d.e = C()
getter = attrgetter("e.a")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = attrgetter("e.a", "e.b")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
def to_pytorch(self):
"""
Convert to pytorch model
:return: a pytorch model
"""
new_weight = self.get_weights()
assert (len(new_weight) == 1,
"TorchModel's weights should be one tensor")
m = CloudPickleSerializer.loads(CloudPickleSerializer,
self.module_bytes)
w = torch.Tensor(new_weight[0])
torch.nn.utils.vector_to_parameters(w, m.parameters())
return m
def test_attrgetter(self):
from operator import attrgetter
ser = CloudPickleSerializer()
class C(object):
def __getattr__(self, item):
return item
d = C()
getter = attrgetter("a")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = attrgetter("a", "b")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
d.e = C()
getter = attrgetter("e.a")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
getter = attrgetter("e.a", "e.b")
getter2 = ser.loads(ser.dumps(getter))
self.assertEqual(getter(d), getter2(d))
def test_function_module_name(self):
ser = CloudPickleSerializer()
func = lambda x: x
func2 = ser.loads(ser.dumps(func))
self.assertEqual(func.__module__, func2.__module__)
def test_function_module_name(self):
ser = CloudPickleSerializer()
func = lambda x: x
func2 = ser.loads(ser.dumps(func))
self.assertEqual(func.__module__, func2.__module__)