def test_register_naive_bayes_serialization():
"""
Assuming here that the Dask serializers are well-tested.
This test-case is only validating that register_serialization
actually provides the expected serializers on the expected
objects.
"""
mnb = MultinomialNB()
X = cupyx.scipy.sparse.random(1, 5)
y = cp.array([0])
mnb.fit(X, y)
# Unfortunately, Dask has no `unregister` function and Pytest
# shares the same process so cannot test the base-state here.
stype, sbytes = serialize(mnb, serializers=['cuda'])
assert stype['serializer'] == 'cuda'
stype, sbytes = serialize(mnb, serializers=['dask'])
assert stype['serializer'] == 'dask'
def __setattr__(self, name, value):
if name == "value":
try:
serialize(value)
except:
# TODO: narrow except
msg.logMessage(f"Value '{value}'on input '{name}' could not be cloudpickled.", level=msg.WARNING)
super().__setattr__(name, value)
else:
super().__setattr__(name, value)
def serialize_sparse(x):
coords_header, coords_frames = serialize(x.coords)
data_header, data_frames = serialize(x.data)
header = {
"coords-header": coords_header,
"data-header": data_header,
"shape": x.shape,
"nframes": [len(coords_frames), len(data_frames)],
}
return header, coords_frames + data_frames
def test_sparse_cumlarray_serialization():
X = cupyx.scipy.sparse.random(10, 5, format='csr', density=0.9)
X_m = SparseCumlArray(X)
stype, sbytes = serialize(X_m, serializers=['cuda'])
assert stype['serializer'] == 'cuda'
stype, sbytes = serialize(X_m, serializers=['dask'])
assert stype['serializer'] == 'dask'
def serialize_torch_Tensor(t):
requires_grad_ = t.requires_grad
if requires_grad_:
sub_header, frames = serialize(t.detach().numpy())
else:
sub_header, frames = serialize(t.numpy())
header = {"sub-header": sub_header}
if t.grad is not None:
grad_header, grad_frames = serialize(t.grad.numpy())
header["grad"] = {"header": grad_header, "start": len(frames)}
frames += grad_frames
header["requires_grad"] = requires_grad_
header["device"] = t.device.type
return header, frames
def test_sfmData_roundtrip(self):
"""Test for equality after serializing and then de-serializing an SfmData instance."""
expected = EXAMPLE_DATA
header, frames = serialize(expected)
recovered = deserialize(header, frames)
# comparing tracks in an order-sensitive fashion.
self.assertTrue(recovered.equals(expected, 1e-9))
def get_prep_value(self, value):
"""
Called to convert Python objects back to query values.
"""
try:
serialize_tuple = serialize(value)
return str(serialize_tuple)
except Exception:
raise ValidationError('Unable to serialize value: {}'.format(value))
def test_sfmResult_roundtrip(self):
"""Test for equality after serializing and then de-serializing an SfmResult instance."""
expected = SfmResult(EXAMPLE_DATA, total_reproj_error=1.5)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
# comparing cameras and total reprojection error
self.assertEqual(recovered.total_reproj_error,
expected.total_reproj_error)
self.assertTrue(recovered.sfm_data.equals(expected.sfm_data, 1e-9))
def get_prep_value(self, value):
"""
Called to convert Python objects back to query values.
"""
try:
serialize_tuple = serialize(value)
return str(serialize_tuple)
except Exception:
raise ValidationError(
'Unable to serialize value: {}'.format(value))
def test_pinholeCameraCal3Bundler_roundtrip(self):
"""Test the round-trip on Unit3 object."""
expected = PinholeCameraCal3Bundler(
Pose3(Rot3.RzRyRx(0, 0.1, -0.05), np.random.randn(3, 1)),
Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70),
)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def test_serialize_of_proxied_cudf(proxy_serializers, dask_serializers):
"""Check that we can serialize a proxied cudf dataframe, which might
be serialized already.
"""
cudf = pytest.importorskip("cudf")
df = cudf.DataFrame({"a": range(10)})
pxy = proxy_object.asproxy(df, serializers=proxy_serializers)
header, frames = serialize(pxy,
serializers=dask_serializers,
on_error="raise")
pxy = deserialize(header, frames)
assert_frame_equal(df.to_pandas(), pxy.to_pandas())
def test_dumps_loads_Serialized():
msg = {"x": 1, "data": Serialized(*serialize(123))}
frames = dumps(msg)
assert len(frames) > 2
result = loads(frames)
assert result == {"x": 1, "data": 123}
result2 = loads(frames, deserialize=False)
assert result2 == msg
frames2 = dumps(result2)
assert all(map(eq_frames, frames, frames2))
result3 = loads(frames2)
assert result == result3
def test_dumps_loads_Serialized():
msg = {'x': 1,
'data': Serialized(*serialize(123)),
}
frames = dumps(msg)
assert len(frames) > 2
result = loads(frames)
assert result == {'x': 1, 'data': 123}
result2 = loads(frames, deserialize=False)
assert result2 == msg
frames2 = dumps(result2)
assert all(map(eq_frames, frames, frames2))
result3 = loads(frames2)
assert result == result3
def test_preserve_header(serializers):
"""
Test that a serialization family doesn't overwrite the headers
of the underlying registered dumps/loads functions.
"""
class MyObj:
pass
@cuda_serialize.register(MyObj)
@dask_serialize.register(MyObj)
def _(x):
return {}, []
@cuda_deserialize.register(MyObj)
@dask_deserialize.register(MyObj)
def _(header, frames):
assert header == {}
assert frames == []
return MyObj()
header, frames = serialize(MyObj(), serializers=serializers)
o = deserialize(header, frames)
assert isinstance(o, MyObj)
def test_gds(gds_enabled, cuda_lib):
lib = pytest.importorskip(cuda_lib)
if cuda_lib == "cupy":
data_create = lambda: lib.arange(10)
data_compare = lambda x, y: all(x == y)
elif cuda_lib == "cudf":
data_create = lambda: lib.Series(range(10))
data_compare = lambda x, y: all((x == y).values_host)
elif cuda_lib == "numba.cuda":
data_create = lambda: lib.to_device(range(10))
data_compare = lambda x, y: all(x.copy_to_host() == y.copy_to_host())
try:
ProxifyHostFile.register_disk_spilling()
if gds_enabled and not ProxifyHostFile._gds_enabled:
pytest.skip("GDS not available")
a = data_create()
header, frames = serialize(a, serializers=("disk", ))
b = deserialize(header, frames)
assert type(a) == type(b)
assert data_compare(a, b)
finally:
ProxifyHostFile.register_disk_spilling() # Reset disk spilling options
def test_rot3_roundtrip(self):
"""Test the round-trip on Rot3 object."""
expected = Rot3.RzRyRx(0, 0.05, 0.1)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def set_up(self):
self.field = DaskSerializedField(blank=True, null=True)
self.value = 'foo'
self.s_value = str(serialize(self.value))
def serialize(imar):
meta = imar.get_meta_dict()
ar = np.array(imar)
header, frames = serialize.serialize([meta, ar])
return header, frames
def serialize_netcdf4_variable(x):
header, _ = serialize(x.group())
header["parent-type"] = header["type"]
header["parent-type-serialized"] = header["type-serialized"]
header["name"] = x.name
return header, []
def set_up(self):
self.field = DaskSerializedField(blank=True, null=True)
self.value = 'foo'
self.s_value = str(serialize(self.value))
def test_cal3Bundler_roundtrip(self):
"""Test the round-trip on Cal3Bundler object."""
expected = Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def test_point3_roundtrip(self):
"""Test the round-trip on Point3 object."""
expected = Point3(np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
np.testing.assert_allclose(expected, recovered)
def test_pose3_roundtrip(self):
"""Test the round-trip on Point3 object."""
expected = Pose3(Rot3.RzRyRx(0, 0.1, 0.2), np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(recovered.equals(expected, 1e-5))
def serialize_torch_Parameters(p):
sub_header, frames = serialize(p.detach())
header = {"sub-header": sub_header}
header["requires_grad"] = p.requires_grad
return header, frames
def test_unit3_roundtrip(self):
"""Test the round-trip on Unit3 object."""
expected = Unit3(np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))