def _read_index(self):
try:
self._handle.seek(-8, os.SEEK_END)
self._handle.seek(read_i64(self._handle))
self._index = read_json(self._handle)
except:
raise IOError('unable to read index')
def test_metadata(self, array, compression, shuffle, stream, length):
array = np.asanyarray(array)
stream.seek(0)
meta = read_json(stream)
assert meta['comp'] == [compression, 5, shuffle]
assert meta['size'] == array.size * array.dtype.itemsize
assert np.dtype(meta['dtype']) == array.dtype
assert meta['shape'] == list(array.shape)
assert stream.tell() == length - meta['length']
def test_read_write_json(data):
stream = BytesIO()
total = write_json(stream, data)
assert stream.tell() == total
stream.seek(0)
length = read_i64(stream)
assert stream.tell() == total - length
assert json.loads(stream.read(length).decode('utf-8')) == data
stream.seek(0)
assert read_json(stream) == data
def read(self, key, out=None):
self._check_handle()
self._check_key(key)
is_array, offset = self._index[key]
if not is_array and out is not None:
raise ValueError('can only specify output for array values')
self._handle.seek(offset)
if is_array:
return read_blosc(self._handle, out=out)
else:
return read_json(self._handle)
def info(self, key):
self._check_handle()
self._check_key(key)
is_array, offset = self._index[key]
self._handle.seek(offset)
if is_array:
meta = read_json(self._handle)
return {
'type': 'array',
'shape': tuple(meta['shape']),
'dtype': restore_dtype(meta['dtype']),
'compression': tuple(meta['comp'])
}
else:
return {'type': 'json'}
def read_blosc(stream, out=None):
meta = read_json(stream)
shape = tuple(meta['shape'])
dtype = restore_dtype(meta['dtype'])
if out is None:
out = np.empty(shape, dtype)
elif not isinstance(out, np.ndarray):
raise TypeError('expected ndarray, got {}'.format(type(out).__name__))
elif out.shape != shape:
raise ValueError('incompatible shape: expected {}, got {}'.format(shape, out.shape))
elif out.dtype != dtype:
raise ValueError('incompatible dtype: expected {}, got {}'.format(dtype, out.dtype))
elif not out.flags.contiguous:
raise ValueError('expected contiguous array')
blosc.decompress_ptr(
stream.read(meta['length']),
out.__array_interface__['data'][0]
)
if out.dtype.type is np.record:
out = out.view(np.recarray)
return out