def deserialize_sparse(header, frames):
coords_frames = frames[:header["nframes"][0]]
data_frames = frames[header["nframes"][0]:]
coords = deserialize(header["coords-header"], coords_frames)
data = deserialize(header["data-header"], data_frames)
shape = header["shape"]
return sparse.COO(coords, data, shape=shape)
def test_sfmData_roundtrip(self):
"""Test for equality after serializing and then de-serializing an SfmData instance."""
expected = EXAMPLE_DATA
header, frames = serialize(expected)
recovered = deserialize(header, frames)
# comparing tracks in an order-sensitive fashion.
self.assertTrue(recovered.equals(expected, 1e-9))
def test_sfmResult_roundtrip(self):
"""Test for equality after serializing and then de-serializing an SfmResult instance."""
expected = SfmResult(EXAMPLE_DATA, total_reproj_error=1.5)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
# comparing cameras and total reprojection error
self.assertEqual(recovered.total_reproj_error,
expected.total_reproj_error)
self.assertTrue(recovered.sfm_data.equals(expected.sfm_data, 1e-9))
def deserialize_torch_Tensor(header, frames):
if header.get("grad", False):
i = header["grad"]["start"]
frames, grad_frames = frames[:i], frames[i:]
grad = deserialize(header["grad"]["header"], grad_frames)
else:
grad = None
x = deserialize(header["sub-header"], frames)
if header["device"] == "cpu":
t = torch.from_numpy(x)
if header["requires_grad"]:
t = t.requires_grad_(True)
else:
t = torch.tensor(
data=x, device=header["device"], requires_grad=header["requires_grad"]
)
if grad is not None:
t.grad = torch.from_numpy(grad)
return t
def test_loads_deserialize_False():
frames = dumps({"data": Serialize(123), "status": "OK"})
msg = loads(frames)
assert msg == {"data": 123, "status": "OK"}
msg = loads(frames, deserialize=False)
assert msg["status"] == "OK"
assert isinstance(msg["data"], Serialized)
result = deserialize(msg["data"].header, msg["data"].frames)
assert result == 123
def test_loads_deserialize_False():
frames = dumps({'data': Serialize(123), 'status': 'OK'})
msg = loads(frames)
assert msg == {'data': 123, 'status': 'OK'}
msg = loads(frames, deserialize=False)
assert msg['status'] == 'OK'
assert isinstance(msg['data'], Serialized)
result = deserialize(msg['data'].header, msg['data'].frames)
assert result == 123
def test_loads_deserialize_False():
frames = dumps({'data': Serialize(123), 'status': 'OK'})
msg = loads(frames)
assert msg == {'data': 123, 'status': 'OK'}
msg = loads(frames, deserialize=False)
assert msg['status'] == 'OK'
assert isinstance(msg['data'], Serialized)
result = deserialize(msg['data'].header, msg['data'].frames)
assert result == 123
def to_python(self, value):
"""
Called by deserialization and during the clean() method used on forms.
"""
if value is None or not isinstance(value, str):
return value
try:
return deserialize(*make_tuple(value))
except Exception:
raise ValidationError('Unable to deserialize value: {}'.format(value))
def from_db_value(self, value, *args, **kwargs):
"""
Called in all circumstances when data is loaded from the database, including in aggregates and values() calls.
"""
if value is None:
return value
try:
return deserialize(*make_tuple(value))
except Exception:
raise ValidationError('Unable to deserialize value: {}'.format(str(value)))
def test_pinholeCameraCal3Bundler_roundtrip(self):
"""Test the round-trip on Unit3 object."""
expected = PinholeCameraCal3Bundler(
Pose3(Rot3.RzRyRx(0, 0.1, -0.05), np.random.randn(3, 1)),
Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70),
)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def from_db_value(self, value, *args, **kwargs):
"""
Called in all circumstances when data is loaded from the database, including in aggregates and values() calls.
"""
if value is None:
return value
try:
return deserialize(*make_tuple(value))
except Exception:
raise ValidationError('Unable to deserialize value: {}'.format(
str(value)))
def to_python(self, value):
"""
Called by deserialization and during the clean() method used on forms.
"""
if value is None or not isinstance(value, str):
return value
try:
return deserialize(*make_tuple(value))
except Exception:
raise ValidationError(
'Unable to deserialize value: {}'.format(value))
def deserialize_keras_model(header, frames):
from keras.models import model_from_config
n = 0
weights = []
for head, length in zip(header["headers"], header["nframes"]):
x = deserialize(head, frames[n:n + length])
weights.append(x)
n += length
model = model_from_config(header)
model.set_weights(weights)
return model
def test_serialize_of_proxied_cudf(proxy_serializers, dask_serializers):
"""Check that we can serialize a proxied cudf dataframe, which might
be serialized already.
"""
cudf = pytest.importorskip("cudf")
df = cudf.DataFrame({"a": range(10)})
pxy = proxy_object.asproxy(df, serializers=proxy_serializers)
header, frames = serialize(pxy,
serializers=dask_serializers,
on_error="raise")
pxy = deserialize(header, frames)
assert_frame_equal(df.to_pandas(), pxy.to_pandas())
def test_preserve_header(serializers):
"""
Test that a serialization family doesn't overwrite the headers
of the underlying registered dumps/loads functions.
"""
class MyObj:
pass
@cuda_serialize.register(MyObj)
@dask_serialize.register(MyObj)
def _(x):
return {}, []
@cuda_deserialize.register(MyObj)
@dask_deserialize.register(MyObj)
def _(header, frames):
assert header == {}
assert frames == []
return MyObj()
header, frames = serialize(MyObj(), serializers=serializers)
o = deserialize(header, frames)
assert isinstance(o, MyObj)
def test_gds(gds_enabled, cuda_lib):
lib = pytest.importorskip(cuda_lib)
if cuda_lib == "cupy":
data_create = lambda: lib.arange(10)
data_compare = lambda x, y: all(x == y)
elif cuda_lib == "cudf":
data_create = lambda: lib.Series(range(10))
data_compare = lambda x, y: all((x == y).values_host)
elif cuda_lib == "numba.cuda":
data_create = lambda: lib.to_device(range(10))
data_compare = lambda x, y: all(x.copy_to_host() == y.copy_to_host())
try:
ProxifyHostFile.register_disk_spilling()
if gds_enabled and not ProxifyHostFile._gds_enabled:
pytest.skip("GDS not available")
a = data_create()
header, frames = serialize(a, serializers=("disk", ))
b = deserialize(header, frames)
assert type(a) == type(b)
assert data_compare(a, b)
finally:
ProxifyHostFile.register_disk_spilling() # Reset disk spilling options
def test_rot3_roundtrip(self):
"""Test the round-trip on Rot3 object."""
expected = Rot3.RzRyRx(0, 0.05, 0.1)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def deserialize(header, frames):
[meta, ar] = serialize.deserialize(header, frames)
return ImageArray(ar, meta=meta)
def deserialize_netcdf4_variable(header, frames):
header["type"] = header["parent-type"]
header["type-serialized"] = header["parent-type-serialized"]
parent = deserialize(header, frames)
return parent.variables[header["name"]]
def test_point3_roundtrip(self):
"""Test the round-trip on Point3 object."""
expected = Point3(np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
np.testing.assert_allclose(expected, recovered)
def test_pose3_roundtrip(self):
"""Test the round-trip on Point3 object."""
expected = Pose3(Rot3.RzRyRx(0, 0.1, 0.2), np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(recovered.equals(expected, 1e-5))
def test_cal3Bundler_roundtrip(self):
"""Test the round-trip on Cal3Bundler object."""
expected = Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70)
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
def deserialize_torch_Parameters(header, frames):
t = deserialize(header["sub-header"], frames)
return torch.nn.Parameter(data=t, requires_grad=header["requires_grad"])
def test_unit3_roundtrip(self):
"""Test the round-trip on Unit3 object."""
expected = Unit3(np.random.randn(3))
header, frames = serialize(expected)
recovered = deserialize(header, frames)
self.assertTrue(expected.equals(recovered, 1e-5))
