Esempio n. 1
0
def write_blosc(stream, data, compression='lz4', level=5, shuffle=True):
    if isinstance(compression, six.string_types) and compression.startswith('blosc:'):
        compression = compression[6:]
    data = np.asanyarray(data)
    if data.dtype == np.dtype('O'):
        raise ValueError('unable to serialize: invalid dtype')
    if not data.flags.contiguous:
        raise ValueError('expected contiguous array')
    payload = blosc.compress_ptr(
        data.__array_interface__['data'][0],
        data.size,
        data.dtype.itemsize,
        cname=compression,
        clevel=level,
        shuffle=shuffle
    )
    meta = {
        'size': data.size * data.dtype.itemsize,
        'length': len(payload),
        'comp': (compression, level, int(shuffle)),
        'shape': data.shape,
        'dtype': flatten_dtype(data.dtype)
    }
    meta_length = write_json(stream, meta)
    stream.write(payload)
    return len(payload) + meta_length
Esempio n. 2
0
def test_flatten_restore_dtype(dtype, flattened):
    dtype = np.dtype(dtype)
    assert flatten_dtype(dtype) == flattened
    assert restore_dtype(flatten_dtype(dtype)) == dtype
    assert restore_dtype(json.loads(json.dumps(flatten_dtype(dtype)))) == dtype