def _chunk_getitem(self, cidx, item, dest):
"""Obtain part or whole of a chunk.
Parameters
----------
cidx : tuple of ints
Indices of the chunk.
item : tuple of slices
Location of region within the chunk.
dest : ndarray
Numpy array to store result in.
"""
try:
# obtain compressed data for chunk
ckey = self._chunk_key(cidx)
cdata = self._chunk_store[ckey]
except KeyError:
# chunk not initialized
if self._fill_value is not None:
dest.fill(self._fill_value)
else:
if is_total_slice(item, self._chunks) and \
not self._filters and \
((self._order == 'C' and dest.flags.c_contiguous) or
(self._order == 'F' and dest.flags.f_contiguous)):
# optimization: we want the whole chunk, and the destination is
# contiguous, so we can decompress directly from the chunk
# into the destination array
if self._compressor:
self._compressor.decode(cdata, dest)
else:
arr = np.frombuffer(cdata, dtype=self._dtype)
arr = arr.reshape(self._chunks, order=self._order)
np.copyto(dest, arr)
else:
# decode chunk
chunk = self._decode_chunk(cdata)
# set data in output array
# (split into two lines for profiling)
tmp = chunk[item]
if dest.shape:
dest[:] = tmp
else:
dest[()] = tmp
def _chunk_getitem(self, cidx, item, dest):
"""Obtain part or whole of a chunk.
Parameters
----------
cidx : tuple of ints
Indices of the chunk.
item : tuple of slices
Location of region within the chunk.
dest : ndarray
Numpy array to store result in.
"""
try:
# obtain compressed data for chunk
ckey = self._chunk_key(cidx)
cdata = self._chunk_store[ckey]
except KeyError:
# chunk not initialized
if self._fill_value is not None:
dest.fill(self._fill_value)
else:
if is_total_slice(item, self._chunks) and \
not self._filters and \
((self._order == 'C' and dest.flags.c_contiguous) or
(self._order == 'F' and dest.flags.f_contiguous)):
# optimization: we want the whole chunk, and the destination is
# contiguous, so we can decompress directly from the chunk
# into the destination array
if self._compressor:
self._compressor.decode(cdata, dest)
else:
arr = np.frombuffer(cdata, dtype=self._dtype)
arr = arr.reshape(self._chunks, order=self._order)
np.copyto(dest, arr)
else:
# decode chunk
chunk = self._decode_chunk(cdata)
# set data in output array
# (split into two lines for profiling)
tmp = chunk[item]
if dest.shape:
dest[:] = tmp
else:
dest[()] = tmp
def test_is_total_slice():
# 1D
assert_true(is_total_slice(Ellipsis, (100, )))
assert_true(is_total_slice(slice(None), (100, )))
assert_true(is_total_slice(slice(0, 100), (100, )))
assert_false(is_total_slice(slice(0, 50), (100, )))
assert_false(is_total_slice(slice(0, 100, 2), (100, )))
# 2D
assert_true(is_total_slice(Ellipsis, (100, 100)))
assert_true(is_total_slice(slice(None), (100, 100)))
assert_true(is_total_slice((slice(None), slice(None)), (100, 100)))
assert_true(is_total_slice((slice(0, 100), slice(0, 100)), (100, 100)))
assert_false(is_total_slice((slice(0, 100), slice(0, 50)), (100, 100)))
assert_false(is_total_slice((slice(0, 50), slice(0, 100)), (100, 100)))
assert_false(is_total_slice((slice(0, 50), slice(0, 50)), (100, 100)))
assert_false(is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100)))
with assert_raises(TypeError):
is_total_slice('foo', (100, ))
def _chunk_setitem_nosync(self, cidx, item, value):
# obtain key for chunk storage
ckey = self._chunk_key(cidx)
if is_total_slice(item, self._chunks):
# totally replace chunk
# optimization: we are completely replacing the chunk, so no need
# to access the existing chunk data
if np.isscalar(value):
# setup array filled with value
chunk = np.empty(self._chunks,
dtype=self._dtype,
order=self._order)
chunk.fill(value)
else:
if not self._compressor and not self._filters:
# https://github.com/alimanfoo/zarr/issues/79
# Ensure a copy is taken so we don't end up storing
# a view into someone else's array.
# N.B., this assumes that filters or compressor always
# take a copy and never attempt to apply encoding in-place.
chunk = np.array(value,
dtype=self._dtype,
order=self._order)
else:
# ensure array is contiguous
if self._order == 'F':
chunk = np.asfortranarray(value, dtype=self._dtype)
else:
chunk = np.ascontiguousarray(value, dtype=self._dtype)
else:
# partially replace the contents of this chunk
try:
# obtain compressed data for chunk
cdata = self._chunk_store[ckey]
except KeyError:
# chunk not initialized
chunk = np.empty(self._chunks,
dtype=self._dtype,
order=self._order)
if self._fill_value is not None:
chunk.fill(self._fill_value)
else:
# decode chunk
chunk = self._decode_chunk(cdata)
if not chunk.flags.writeable:
chunk = chunk.copy(order='K')
# modify
chunk[item] = value
# encode chunk
cdata = self._encode_chunk(chunk)
# store
self._chunk_store[ckey] = cdata
def test_is_total_slice():
# 1D
assert is_total_slice(Ellipsis, (100, ))
assert is_total_slice(slice(None), (100, ))
assert is_total_slice(slice(0, 100), (100, ))
assert not is_total_slice(slice(0, 50), (100, ))
assert not is_total_slice(slice(0, 100, 2), (100, ))
# 2D
assert is_total_slice(Ellipsis, (100, 100))
assert is_total_slice(slice(None), (100, 100))
assert is_total_slice((slice(None), slice(None)), (100, 100))
assert is_total_slice((slice(0, 100), slice(0, 100)), (100, 100))
assert not is_total_slice((slice(0, 100), slice(0, 50)), (100, 100))
assert not is_total_slice((slice(0, 50), slice(0, 100)), (100, 100))
assert not is_total_slice((slice(0, 50), slice(0, 50)), (100, 100))
assert not is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100))
with pytest.raises(TypeError):
is_total_slice('foo', (100, ))
def test_is_total_slice():
# 1D
assert_true(is_total_slice(Ellipsis, (100,)))
assert_true(is_total_slice(slice(None), (100,)))
assert_true(is_total_slice(slice(0, 100), (100,)))
assert_false(is_total_slice(slice(0, 50), (100,)))
assert_false(is_total_slice(slice(0, 100, 2), (100,)))
# 2D
assert_true(is_total_slice(Ellipsis, (100, 100)))
assert_true(is_total_slice(slice(None), (100, 100)))
assert_true(is_total_slice((slice(None), slice(None)), (100, 100)))
assert_true(is_total_slice((slice(0, 100), slice(0, 100)), (100, 100)))
assert_false(is_total_slice((slice(0, 100), slice(0, 50)), (100, 100)))
assert_false(is_total_slice((slice(0, 50), slice(0, 100)), (100, 100)))
assert_false(is_total_slice((slice(0, 50), slice(0, 50)), (100, 100)))
assert_false(is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100)))
with assert_raises(TypeError):
is_total_slice('foo', (100,))
def _chunk_setitem_nosync(self, cidx, item, value):
# obtain key for chunk storage
ckey = self._chunk_key(cidx)
if is_total_slice(item, self._chunks):
# totally replace chunk
# optimization: we are completely replacing the chunk, so no need
# to access the existing chunk data
if np.isscalar(value):
# setup array filled with value
chunk = np.empty(self._chunks, dtype=self._dtype,
order=self._order)
chunk.fill(value)
else:
if not self._compressor and not self._filters:
# https://github.com/alimanfoo/zarr/issues/79
# Ensure a copy is taken so we don't end up storing
# a view into someone else's array.
# N.B., this assumes that filters or compressor always
# take a copy and never attempt to apply encoding in-place.
chunk = np.array(value, dtype=self._dtype,
order=self._order)
else:
# ensure array is contiguous
if self._order == 'F':
chunk = np.asfortranarray(value, dtype=self._dtype)
else:
chunk = np.ascontiguousarray(value, dtype=self._dtype)
else:
# partially replace the contents of this chunk
try:
# obtain compressed data for chunk
cdata = self._chunk_store[ckey]
except KeyError:
# chunk not initialized
chunk = np.empty(self._chunks, dtype=self._dtype,
order=self._order)
if self._fill_value is not None:
chunk.fill(self._fill_value)
else:
# decode chunk
chunk = self._decode_chunk(cdata)
if not chunk.flags.writeable:
chunk = chunk.copy(order='K')
# modify
chunk[item] = value
# encode chunk
cdata = self._encode_chunk(chunk)
# store
self._chunk_store[ckey] = cdata
