def __init__(self, obj, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# No explicit dshape provided. Infer it from the source.
# if the source is another blaze array just use its shape,
# otherwile let the provider infer its data_shape
if isinstance(obj, Array):
dshape = obj.datashape
else:
dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, ByteProvider):
self.data = obj
elif isinstance(obj, Array):
self.data = CArraySource(obj.data,
dshape=dshape,
params=params)
else:
self.data = CArraySource(obj, dshape=dshape, params=params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def __init__(self,
obj,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, CArraySource):
self.data = obj
else:
self.data = CArraySource(obj, params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = ChunkedL(self.data, cdimension=0)
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def __init__(self, obj, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, CArraySource):
self.data = obj
else:
self.data = CArraySource(obj, params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = ChunkedL(self.data, cdimension=0)
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def test_open_carray():
with temp_dir() as temp:
# Create an array on disk
array_filename = os.path.join(temp, 'carray')
p = params(storage=array_filename)
ds = dshape('1,int32')
a = CArraySource([2], dshape=ds, params=p)
del a
# Open array with open function
uri = 'carray://' + array_filename
c = toplevel.open(uri)
assert c.datashape == ds
# Test delayed mode
c = toplevel.open(uri, eclass=eclass.delayed)
assert c.datashape == ds
class Array(Indexable):
"""
Manifest array, does not create a graph. Forces evaluation on every
call.
Parameters
----------
obj : A list of byte providers, other NDTables or a Python object.
Optional
--------
datashape : dshape
Manual datashape specification for the table, if None then
shape will be inferred if possible.
metadata :
Manual datashape specification for the table, if None then
shape will be inferred if possible.
Usage
-----
>>> Array([1,2,3])
>>> Array([1,2,3], dshape='3, int32')
>>> Array([1,2,3], dshape('3, int32'))
>>> Array([1,2,3], params=params(clevel=3, storage='file'))
"""
eclass = eclass.manifest
_metaheader = [
md.manifest,
md.arraylike,
]
def __init__(self, obj, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, ByteProvider):
self.data = obj
else:
self.data = CArraySource(obj, params=params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def _asdeferred(self):
""" Convert a manifest array into a deferred array """
return NDArray(
self.data,
dshape = self._datashape,
metadata = self._metadata,
layout = self._layout,
params = self.params
)
#------------------------------------------------------------------------
# Properties
#------------------------------------------------------------------------
@property
def datashape(self):
"""
Type deconstructor
"""
return self._datashape
@property
def size(self):
"""
Size of the Array.
"""
# TODO: need to generalize, not every Array will look
# like Numpy
return sum(i.val for i in self._datashape.parameters[:-1])
@property
def backends(self):
"""
The storage backends that make up the space behind the
Array.
"""
return iter(self.space)
#------------------------------------------------------------------------
# Basic Slicing
#------------------------------------------------------------------------
# Immediete slicing
def __getitem__(self, indexer):
cc = self._layout.change_coordinates
return retrieve(cc, indexer)
# Immediete slicing ( Side-effectful )
def __setitem__(self, indexer, value):
cc = self._layout.change_coordinates
write(cc, indexer, value)
def __str__(self):
return generic_str(self, deferred=False)
def __repr__(self):
return generic_repr('Array', self, deferred=False)
def test_simple():
a = CArraySource([1, 2, 3])
layout = ContiguousL(a)
indexer = (0, )
retrieve(layout.change_coordinates, indexer)
def __init__(self,
obj,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# No explicit dshape provided. Infer it from the source.
# if the source is another blaze array just use its shape,
# otherwile let the provider infer its data_shape
if isinstance(obj, Array):
dshape = obj.datashape
else:
dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, ByteProvider):
self.data = obj
elif isinstance(obj, Array):
self.data = CArraySource(obj.data, dshape=dshape, params=params)
else:
self.data = CArraySource(obj, dshape=dshape, params=params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
class Array(Indexable):
"""
Manifest array, does not create a graph. Forces evaluation on every
call.
Parameters
----------
obj : A list of byte providers, other NDTables or a Python object.
Optional
--------
datashape : dshape
Manual datashape specification for the table, if None then
shape will be inferred if possible.
metadata :
Manual datashape specification for the table, if None then
shape will be inferred if possible.
Usage
-----
>>> Array([1,2,3])
>>> Array([1,2,3], dshape='3, int32')
>>> Array([1,2,3], dshape('3, int32'))
>>> Array([1,2,3], params=params(clevel=3, storage='file'))
"""
eclass = eclass.manifest
_metaheader = [
md.manifest,
md.arraylike,
]
def __init__(self,
obj,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# No explicit dshape provided. Infer it from the source.
# if the source is another blaze array just use its shape,
# otherwile let the provider infer its data_shape
if isinstance(obj, Array):
dshape = obj.datashape
else:
dshape = CArraySource.infer_datashape(obj)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CArraySource.check_datashape(obj, given_dshape=dshape)
self._datashape = dshape
# Values
# ------
# Mimic NumPy behavior in that we have a variety of
# possible arguments to the first argument which result
# in different behavior for the values.
if isinstance(obj, ByteProvider):
self.data = obj
elif isinstance(obj, Array):
self.data = CArraySource(obj.data, dshape=dshape, params=params)
else:
self.data = CArraySource(obj, dshape=dshape, params=params)
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDArray._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def _asdeferred(self):
""" Convert a manifest array into a deferred array """
return NDArray(self.data,
dshape=self._datashape,
metadata=self._metadata,
layout=self._layout,
params=self.params)
#------------------------------------------------------------------------
# Properties
#------------------------------------------------------------------------
@property
def datashape(self):
"""
Type deconstructor
"""
return self._datashape
@property
def size(self):
"""
Size of the Array.
"""
# TODO: need to generalize, not every Array will look
# like Numpy
return sum(i.val for i in self._datashape.parameters[:-1])
@property
def backends(self):
"""
The storage backends that make up the space behind the
Array.
"""
return iter(self.space)
#------------------------------------------------------------------------
# Basic Slicing
#------------------------------------------------------------------------
# Immediete slicing
def __getitem__(self, indexer):
cc = self._layout.change_coordinates
return retrieve(cc, indexer)
# Immediete slicing ( Side-effectful )
def __setitem__(self, indexer, value):
cc = self._layout.change_coordinates
write(cc, indexer, value)
#------------------------------------------------------------------------
# Specific functions for carray backend
#------------------------------------------------------------------------
# TODO: don't hardcode against carray, breaks down if we use
# something else
def append(self, data):
self.data.ca.append(data)
# Update the shape
shape, dtype = self.data.ca.shape, self.data.ca.dtype
self._datashape = from_numpy(shape, dtype)
def commit(self):
self.data.ca.flush()
def __str__(self):
return generic_str(self, deferred=False)
def __repr__(self):
return generic_repr('Array', self, deferred=False)