def deserialize_sparse(header, frames): coords_frames = frames[:header["nframes"][0]] data_frames = frames[header["nframes"][0]:] coords = deserialize(header["coords-header"], coords_frames) data = deserialize(header["data-header"], data_frames) shape = header["shape"] return sparse.COO(coords, data, shape=shape)
def test_sfmData_roundtrip(self): """Test for equality after serializing and then de-serializing an SfmData instance.""" expected = EXAMPLE_DATA header, frames = serialize(expected) recovered = deserialize(header, frames) # comparing tracks in an order-sensitive fashion. self.assertTrue(recovered.equals(expected, 1e-9))
def test_sfmResult_roundtrip(self): """Test for equality after serializing and then de-serializing an SfmResult instance.""" expected = SfmResult(EXAMPLE_DATA, total_reproj_error=1.5) header, frames = serialize(expected) recovered = deserialize(header, frames) # comparing cameras and total reprojection error self.assertEqual(recovered.total_reproj_error, expected.total_reproj_error) self.assertTrue(recovered.sfm_data.equals(expected.sfm_data, 1e-9))
def deserialize_torch_Tensor(header, frames): if header.get("grad", False): i = header["grad"]["start"] frames, grad_frames = frames[:i], frames[i:] grad = deserialize(header["grad"]["header"], grad_frames) else: grad = None x = deserialize(header["sub-header"], frames) if header["device"] == "cpu": t = torch.from_numpy(x) if header["requires_grad"]: t = t.requires_grad_(True) else: t = torch.tensor( data=x, device=header["device"], requires_grad=header["requires_grad"] ) if grad is not None: t.grad = torch.from_numpy(grad) return t
def test_loads_deserialize_False(): frames = dumps({"data": Serialize(123), "status": "OK"}) msg = loads(frames) assert msg == {"data": 123, "status": "OK"} msg = loads(frames, deserialize=False) assert msg["status"] == "OK" assert isinstance(msg["data"], Serialized) result = deserialize(msg["data"].header, msg["data"].frames) assert result == 123
def test_loads_deserialize_False(): frames = dumps({'data': Serialize(123), 'status': 'OK'}) msg = loads(frames) assert msg == {'data': 123, 'status': 'OK'} msg = loads(frames, deserialize=False) assert msg['status'] == 'OK' assert isinstance(msg['data'], Serialized) result = deserialize(msg['data'].header, msg['data'].frames) assert result == 123
def test_loads_deserialize_False(): frames = dumps({'data': Serialize(123), 'status': 'OK'}) msg = loads(frames) assert msg == {'data': 123, 'status': 'OK'} msg = loads(frames, deserialize=False) assert msg['status'] == 'OK' assert isinstance(msg['data'], Serialized) result = deserialize(msg['data'].header, msg['data'].frames) assert result == 123
def to_python(self, value): """ Called by deserialization and during the clean() method used on forms. """ if value is None or not isinstance(value, str): return value try: return deserialize(*make_tuple(value)) except Exception: raise ValidationError('Unable to deserialize value: {}'.format(value))
def from_db_value(self, value, *args, **kwargs): """ Called in all circumstances when data is loaded from the database, including in aggregates and values() calls. """ if value is None: return value try: return deserialize(*make_tuple(value)) except Exception: raise ValidationError('Unable to deserialize value: {}'.format(str(value)))
def test_pinholeCameraCal3Bundler_roundtrip(self): """Test the round-trip on Unit3 object.""" expected = PinholeCameraCal3Bundler( Pose3(Rot3.RzRyRx(0, 0.1, -0.05), np.random.randn(3, 1)), Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70), ) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def from_db_value(self, value, *args, **kwargs): """ Called in all circumstances when data is loaded from the database, including in aggregates and values() calls. """ if value is None: return value try: return deserialize(*make_tuple(value)) except Exception: raise ValidationError('Unable to deserialize value: {}'.format( str(value)))
def to_python(self, value): """ Called by deserialization and during the clean() method used on forms. """ if value is None or not isinstance(value, str): return value try: return deserialize(*make_tuple(value)) except Exception: raise ValidationError( 'Unable to deserialize value: {}'.format(value))
def deserialize_keras_model(header, frames): from keras.models import model_from_config n = 0 weights = [] for head, length in zip(header["headers"], header["nframes"]): x = deserialize(head, frames[n:n + length]) weights.append(x) n += length model = model_from_config(header) model.set_weights(weights) return model
def test_serialize_of_proxied_cudf(proxy_serializers, dask_serializers): """Check that we can serialize a proxied cudf dataframe, which might be serialized already. """ cudf = pytest.importorskip("cudf") df = cudf.DataFrame({"a": range(10)}) pxy = proxy_object.asproxy(df, serializers=proxy_serializers) header, frames = serialize(pxy, serializers=dask_serializers, on_error="raise") pxy = deserialize(header, frames) assert_frame_equal(df.to_pandas(), pxy.to_pandas())
def test_preserve_header(serializers): """ Test that a serialization family doesn't overwrite the headers of the underlying registered dumps/loads functions. """ class MyObj: pass @cuda_serialize.register(MyObj) @dask_serialize.register(MyObj) def _(x): return {}, [] @cuda_deserialize.register(MyObj) @dask_deserialize.register(MyObj) def _(header, frames): assert header == {} assert frames == [] return MyObj() header, frames = serialize(MyObj(), serializers=serializers) o = deserialize(header, frames) assert isinstance(o, MyObj)
def test_gds(gds_enabled, cuda_lib): lib = pytest.importorskip(cuda_lib) if cuda_lib == "cupy": data_create = lambda: lib.arange(10) data_compare = lambda x, y: all(x == y) elif cuda_lib == "cudf": data_create = lambda: lib.Series(range(10)) data_compare = lambda x, y: all((x == y).values_host) elif cuda_lib == "numba.cuda": data_create = lambda: lib.to_device(range(10)) data_compare = lambda x, y: all(x.copy_to_host() == y.copy_to_host()) try: ProxifyHostFile.register_disk_spilling() if gds_enabled and not ProxifyHostFile._gds_enabled: pytest.skip("GDS not available") a = data_create() header, frames = serialize(a, serializers=("disk", )) b = deserialize(header, frames) assert type(a) == type(b) assert data_compare(a, b) finally: ProxifyHostFile.register_disk_spilling() # Reset disk spilling options
def test_rot3_roundtrip(self): """Test the round-trip on Rot3 object.""" expected = Rot3.RzRyRx(0, 0.05, 0.1) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def deserialize(header, frames): [meta, ar] = serialize.deserialize(header, frames) return ImageArray(ar, meta=meta)
def deserialize_netcdf4_variable(header, frames): header["type"] = header["parent-type"] header["type-serialized"] = header["parent-type-serialized"] parent = deserialize(header, frames) return parent.variables[header["name"]]
def test_point3_roundtrip(self): """Test the round-trip on Point3 object.""" expected = Point3(np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) np.testing.assert_allclose(expected, recovered)
def test_pose3_roundtrip(self): """Test the round-trip on Point3 object.""" expected = Pose3(Rot3.RzRyRx(0, 0.1, 0.2), np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(recovered.equals(expected, 1e-5))
def test_cal3Bundler_roundtrip(self): """Test the round-trip on Cal3Bundler object.""" expected = Cal3Bundler(fx=100, k1=0.1, k2=0.2, u0=100, v0=70) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))
def deserialize_torch_Parameters(header, frames): t = deserialize(header["sub-header"], frames) return torch.nn.Parameter(data=t, requires_grad=header["requires_grad"])
def test_unit3_roundtrip(self): """Test the round-trip on Unit3 object.""" expected = Unit3(np.random.randn(3)) header, frames = serialize(expected) recovered = deserialize(header, frames) self.assertTrue(expected.equals(recovered, 1e-5))