def _chunk_getitem(self, cidx, item, dest): """Obtain part or whole of a chunk. Parameters ---------- cidx : tuple of ints Indices of the chunk. item : tuple of slices Location of region within the chunk. dest : ndarray Numpy array to store result in. """ try: # obtain compressed data for chunk ckey = self._chunk_key(cidx) cdata = self._chunk_store[ckey] except KeyError: # chunk not initialized if self._fill_value is not None: dest.fill(self._fill_value) else: if is_total_slice(item, self._chunks) and \ not self._filters and \ ((self._order == 'C' and dest.flags.c_contiguous) or (self._order == 'F' and dest.flags.f_contiguous)): # optimization: we want the whole chunk, and the destination is # contiguous, so we can decompress directly from the chunk # into the destination array if self._compressor: self._compressor.decode(cdata, dest) else: arr = np.frombuffer(cdata, dtype=self._dtype) arr = arr.reshape(self._chunks, order=self._order) np.copyto(dest, arr) else: # decode chunk chunk = self._decode_chunk(cdata) # set data in output array # (split into two lines for profiling) tmp = chunk[item] if dest.shape: dest[:] = tmp else: dest[()] = tmp
def test_is_total_slice(): # 1D assert_true(is_total_slice(Ellipsis, (100, ))) assert_true(is_total_slice(slice(None), (100, ))) assert_true(is_total_slice(slice(0, 100), (100, ))) assert_false(is_total_slice(slice(0, 50), (100, ))) assert_false(is_total_slice(slice(0, 100, 2), (100, ))) # 2D assert_true(is_total_slice(Ellipsis, (100, 100))) assert_true(is_total_slice(slice(None), (100, 100))) assert_true(is_total_slice((slice(None), slice(None)), (100, 100))) assert_true(is_total_slice((slice(0, 100), slice(0, 100)), (100, 100))) assert_false(is_total_slice((slice(0, 100), slice(0, 50)), (100, 100))) assert_false(is_total_slice((slice(0, 50), slice(0, 100)), (100, 100))) assert_false(is_total_slice((slice(0, 50), slice(0, 50)), (100, 100))) assert_false(is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100))) with assert_raises(TypeError): is_total_slice('foo', (100, ))
def _chunk_setitem_nosync(self, cidx, item, value): # obtain key for chunk storage ckey = self._chunk_key(cidx) if is_total_slice(item, self._chunks): # totally replace chunk # optimization: we are completely replacing the chunk, so no need # to access the existing chunk data if np.isscalar(value): # setup array filled with value chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) chunk.fill(value) else: if not self._compressor and not self._filters: # https://github.com/alimanfoo/zarr/issues/79 # Ensure a copy is taken so we don't end up storing # a view into someone else's array. # N.B., this assumes that filters or compressor always # take a copy and never attempt to apply encoding in-place. chunk = np.array(value, dtype=self._dtype, order=self._order) else: # ensure array is contiguous if self._order == 'F': chunk = np.asfortranarray(value, dtype=self._dtype) else: chunk = np.ascontiguousarray(value, dtype=self._dtype) else: # partially replace the contents of this chunk try: # obtain compressed data for chunk cdata = self._chunk_store[ckey] except KeyError: # chunk not initialized chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) if self._fill_value is not None: chunk.fill(self._fill_value) else: # decode chunk chunk = self._decode_chunk(cdata) if not chunk.flags.writeable: chunk = chunk.copy(order='K') # modify chunk[item] = value # encode chunk cdata = self._encode_chunk(chunk) # store self._chunk_store[ckey] = cdata
def test_is_total_slice(): # 1D assert is_total_slice(Ellipsis, (100, )) assert is_total_slice(slice(None), (100, )) assert is_total_slice(slice(0, 100), (100, )) assert not is_total_slice(slice(0, 50), (100, )) assert not is_total_slice(slice(0, 100, 2), (100, )) # 2D assert is_total_slice(Ellipsis, (100, 100)) assert is_total_slice(slice(None), (100, 100)) assert is_total_slice((slice(None), slice(None)), (100, 100)) assert is_total_slice((slice(0, 100), slice(0, 100)), (100, 100)) assert not is_total_slice((slice(0, 100), slice(0, 50)), (100, 100)) assert not is_total_slice((slice(0, 50), slice(0, 100)), (100, 100)) assert not is_total_slice((slice(0, 50), slice(0, 50)), (100, 100)) assert not is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100)) with pytest.raises(TypeError): is_total_slice('foo', (100, ))
def test_is_total_slice(): # 1D assert_true(is_total_slice(Ellipsis, (100,))) assert_true(is_total_slice(slice(None), (100,))) assert_true(is_total_slice(slice(0, 100), (100,))) assert_false(is_total_slice(slice(0, 50), (100,))) assert_false(is_total_slice(slice(0, 100, 2), (100,))) # 2D assert_true(is_total_slice(Ellipsis, (100, 100))) assert_true(is_total_slice(slice(None), (100, 100))) assert_true(is_total_slice((slice(None), slice(None)), (100, 100))) assert_true(is_total_slice((slice(0, 100), slice(0, 100)), (100, 100))) assert_false(is_total_slice((slice(0, 100), slice(0, 50)), (100, 100))) assert_false(is_total_slice((slice(0, 50), slice(0, 100)), (100, 100))) assert_false(is_total_slice((slice(0, 50), slice(0, 50)), (100, 100))) assert_false(is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100))) with assert_raises(TypeError): is_total_slice('foo', (100,))