def array(self, data, expectedlen=None, **kwargs):
# setup
data = _util.ensure_array_like(data)
kwargs = self._set_defaults(kwargs)
# determine chunks
kwargs.setdefault('chunks', default_chunks(data, expectedlen))
# determine object codec
if data.dtype == object:
# peek at first value
peek = data[0]
if isinstance(peek, bytes):
object_codec = numcodecs.VLenBytes()
elif isinstance(peek, str):
object_codec = numcodecs.VLenUTF8()
else:
object_codec = numcodecs.MsgPack()
kwargs.setdefault('object_codec', object_codec)
# create
z = zarr.array(data, **kwargs)
return z
def create_dataset(self, h5g, data=None, **kwargs):
# set defaults
kwargs.setdefault('name', 'data')
for k, v in self.defaults.items():
kwargs.setdefault(k, v)
# handle data
if data is not None:
data = _util.ensure_array_like(data)
# by default, simple chunking across rows
rowsize = data.dtype.itemsize * reduce(operator.mul,
data.shape[1:], 1)
# 1Mb chunks
chunklen = max(1, (2**20) // rowsize)
chunks = (chunklen,) + data.shape[1:]
kwargs.setdefault('chunks', chunks)
# by default, can resize dim 0
maxshape = (None,) + data.shape[1:]
kwargs.setdefault('maxshape', maxshape)
# set data
kwargs['data'] = data
# create dataset
h5d = h5g.create_dataset(**kwargs)
return h5d
def array(self, data, expectedlen=None, **kwargs):
# setup
data = _util.ensure_array_like(data)
kwargs = self._set_defaults(kwargs)
# determine chunks
kwargs.setdefault('chunks', default_chunks(data, expectedlen))
# create
z = zarr.array(data, **kwargs)
return z
def array(self, data, expectedlen=None, **kwargs):
# ignore expectedlen for now
# setup
data = _util.ensure_array_like(data)
# obtain group
h5g = kwargs.pop('group', None)
if h5g is None:
# open file, use root group
h5g, kwargs = self.open_file(**kwargs)
# create dataset
h5d = self.create_dataset(h5g, data=data, **kwargs)
# patch in append method
h5d.append = MethodType(_dataset_append, h5d)
return h5d
def __init__(self, data):
data = _util.ensure_array_like(data)
super(ChunkedArrayWrapper, self).__init__(data)
def array(self, data, expectedlen=None, **kwargs):
data = _util.ensure_array_like(data)
kwargs = self._set_defaults(kwargs)
return bcolz.carray(data, expectedlen=expectedlen, **kwargs)
def __init__(self, data):
data = _util.ensure_array_like(data)
self.data = data