def test_register_naive_bayes_serialization(): """ Assuming here that the Dask serializers are well-tested. This test-case is only validating that register_serialization actually provides the expected serializers on the expected objects. """ mnb = MultinomialNB() X = cupyx.scipy.sparse.random(1, 5) y = cp.array([0]) mnb.fit(X, y) # Unfortunately, Dask has no `unregister` function and Pytest # shares the same process so cannot test the base-state here. stype, sbytes = serialize(mnb, serializers=['cuda']) assert stype['serializer'] == 'cuda' stype, sbytes = serialize(mnb, serializers=['dask']) assert stype['serializer'] == 'dask'
def __setattr__(self, name, value): if name == "value": try: serialize(value) except: # TODO: narrow except msg.logMessage(f"Value '{value}'on input '{name}' could not be cloudpickled.", level=msg.WARNING) super().__setattr__(name, value) else: super().__setattr__(name, value)
def serialize_sparse(x): coords_header, coords_frames = serialize(x.coords) data_header, data_frames = serialize(x.data) header = { "coords-header": coords_header, "data-header": data_header, "shape": x.shape, "nframes": [len(coords_frames), len(data_frames)], } return header, coords_frames + data_frames
def test_sparse_cumlarray_serialization(): X = cupyx.scipy.sparse.random(10, 5, format='csr', density=0.9) X_m = SparseCumlArray(X) stype, sbytes = serialize(X_m, serializers=['cuda']) assert stype['serializer'] == 'cuda' stype, sbytes = serialize(X_m, serializers=['dask']) assert stype['serializer'] == 'dask'
def serialize_torch_Tensor(t): requires_grad_ = t.requires_grad if requires_grad_: sub_header, frames = serialize(t.detach().numpy()) else: sub_header, frames = serialize(t.numpy()) header = {"sub-header": sub_header} if t.grad is not None: grad_header, grad_frames = serialize(t.grad.numpy()) header["grad"] = {"header": grad_header, "start": len(frames)} frames += grad_frames header["requires_grad"] = requires_grad_ header["device"] = t.device.type return header, frames
def test_sfmData_roundtrip(self): """Test for equality after serializing and then de-serializing an SfmData instance.""" expected = EXAMPLE_DATA header, frames = serialize(expected) recovered = deserialize(header, frames) # comparing tracks in an order-sensitive fashion. self.assertTrue(recovered.equals(expected, 1e-9))
def get_prep_value(self, value): """ Called to convert Python objects back to query values. """ try: serialize_tuple = serialize(value) return str(serialize_tuple) except Exception: raise ValidationError('Unable to serialize value: {}'.format(value))
def test_sfmResult_roundtrip(self): """Test for equality after serializing and then de-serializing an SfmResult instance.""" expected = SfmResult(EXAMPLE_DATA, total_reproj_error=1.5) header, frames = serialize(expected) recovered = deserialize(header, frames) # comparing cameras and total reprojection error self.assertEqual(recovered.total_reproj_error, expected.total_reproj_error) self.assertTrue(recovered.sfm_data.equals(expected.sfm_data, 1e-9))
def get_prep_value(self, value): """ Called to convert Python objects back to query values. """ try: serialize_tuple = serialize(value) return str(serialize_tuple) except Exception: raise ValidationError( 'Unable to serialize value: {}'.format(value))
def test_pinholeCameraCal3Bundler_roundtrip(self): """Test the round-trip on Unit3 object.""" expected = PinholeCameraCal3Bundler( Pose3(Rot3.RzRyRx(0, 0.1, -0.05), np.random.randn(3, 1)), Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70), ) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def test_serialize_of_proxied_cudf(proxy_serializers, dask_serializers): """Check that we can serialize a proxied cudf dataframe, which might be serialized already. """ cudf = pytest.importorskip("cudf") df = cudf.DataFrame({"a": range(10)}) pxy = proxy_object.asproxy(df, serializers=proxy_serializers) header, frames = serialize(pxy, serializers=dask_serializers, on_error="raise") pxy = deserialize(header, frames) assert_frame_equal(df.to_pandas(), pxy.to_pandas())
def test_dumps_loads_Serialized(): msg = {"x": 1, "data": Serialized(*serialize(123))} frames = dumps(msg) assert len(frames) > 2 result = loads(frames) assert result == {"x": 1, "data": 123} result2 = loads(frames, deserialize=False) assert result2 == msg frames2 = dumps(result2) assert all(map(eq_frames, frames, frames2)) result3 = loads(frames2) assert result == result3
def test_dumps_loads_Serialized(): msg = {'x': 1, 'data': Serialized(*serialize(123)), } frames = dumps(msg) assert len(frames) > 2 result = loads(frames) assert result == {'x': 1, 'data': 123} result2 = loads(frames, deserialize=False) assert result2 == msg frames2 = dumps(result2) assert all(map(eq_frames, frames, frames2)) result3 = loads(frames2) assert result == result3
def test_preserve_header(serializers): """ Test that a serialization family doesn't overwrite the headers of the underlying registered dumps/loads functions. """ class MyObj: pass @cuda_serialize.register(MyObj) @dask_serialize.register(MyObj) def _(x): return {}, [] @cuda_deserialize.register(MyObj) @dask_deserialize.register(MyObj) def _(header, frames): assert header == {} assert frames == [] return MyObj() header, frames = serialize(MyObj(), serializers=serializers) o = deserialize(header, frames) assert isinstance(o, MyObj)
def test_gds(gds_enabled, cuda_lib): lib = pytest.importorskip(cuda_lib) if cuda_lib == "cupy": data_create = lambda: lib.arange(10) data_compare = lambda x, y: all(x == y) elif cuda_lib == "cudf": data_create = lambda: lib.Series(range(10)) data_compare = lambda x, y: all((x == y).values_host) elif cuda_lib == "numba.cuda": data_create = lambda: lib.to_device(range(10)) data_compare = lambda x, y: all(x.copy_to_host() == y.copy_to_host()) try: ProxifyHostFile.register_disk_spilling() if gds_enabled and not ProxifyHostFile._gds_enabled: pytest.skip("GDS not available") a = data_create() header, frames = serialize(a, serializers=("disk", )) b = deserialize(header, frames) assert type(a) == type(b) assert data_compare(a, b) finally: ProxifyHostFile.register_disk_spilling() # Reset disk spilling options
def test_rot3_roundtrip(self): """Test the round-trip on Rot3 object.""" expected = Rot3.RzRyRx(0, 0.05, 0.1) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def set_up(self): self.field = DaskSerializedField(blank=True, null=True) self.value = 'foo' self.s_value = str(serialize(self.value))
def serialize(imar): meta = imar.get_meta_dict() ar = np.array(imar) header, frames = serialize.serialize([meta, ar]) return header, frames
def serialize_netcdf4_variable(x): header, _ = serialize(x.group()) header["parent-type"] = header["type"] header["parent-type-serialized"] = header["type-serialized"] header["name"] = x.name return header, []
def set_up(self): self.field = DaskSerializedField(blank=True, null=True) self.value = 'foo' self.s_value = str(serialize(self.value))
def test_cal3Bundler_roundtrip(self): """Test the round-trip on Cal3Bundler object.""" expected = Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def test_point3_roundtrip(self): """Test the round-trip on Point3 object.""" expected = Point3(np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) np.testing.assert_allclose(expected, recovered)
def test_pose3_roundtrip(self): """Test the round-trip on Point3 object.""" expected = Pose3(Rot3.RzRyRx(0, 0.1, 0.2), np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(recovered.equals(expected, 1e-5))
def serialize_torch_Parameters(p): sub_header, frames = serialize(p.detach()) header = {"sub-header": sub_header} header["requires_grad"] = p.requires_grad return header, frames
def test_unit3_roundtrip(self): """Test the round-trip on Unit3 object.""" expected = Unit3(np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))