def __init__(self, data, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
if isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
def test_open_ctable():
with temp_dir() as temp:
# Create an table on disk
table_filename = os.path.join(temp, 'ctable')
p = params(storage=table_filename)
ds = dshape('1,{ x: int32; y: int32 }')
t = CTableSource(data=[(1, 1), (2, 2)], dshape=ds, params=p)
del t
# Open table with open function
uri = 'ctable://' + table_filename
c = toplevel.open(uri)
assert c.datashape == ds
# Test delayed mode
c = toplevel.open(uri, eclass=eclass.delayed)
assert c.datashape == ds
class NDTable(Indexable, ArrayNode):
"""
The base NDTable. Indexable contains the indexing logic for
how to access elements, while ArrayNode contains the graph
related logic for building expression trees with this table
as an element.
"""
eclass = eclass.delayed
_metaheader = [
md.deferred,
md.tablelike,
]
def __init__(self, data, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
if isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
@classmethod
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
@property
def datashape(self):
"""
Type deconstructor
"""
return self._datashape
@property
def size(self):
"""
Size of the NDTable.
"""
# TODO: need to generalize, not every Array will look
# like Numpy
return sum(i.val for i in self._datashape.parameters[:-1])
@property
def backends(self):
"""
The storage backends that make up the space behind the Array.
"""
return iter(self.space)
def __repr__(self):
return generic_repr('NDTable', self, deferred=True)
class Table(Indexable):
eclass = eclass.manifest
_metaheader = [
md.manifest,
md.tablelike,
]
def __init__(self, data, dshape=None, metadata=None, layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
elif isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
# TODO: don't hardcode against carray, breaks down if we use
# something else
def append(self, data):
self.data.ca.append(data)
def commit(self):
self.data.ca.flush()
# TODO: don't hardcode against carray
def __len__(self):
return len(self.data.ca)
# TODO: don't hardcode against carray
def __getitem__(self, mask):
ct = (self.data.ca[mask])
dshape = from_numpy(ct.shape, ct.dtype)
source = CTableSource(ct, dshape=dshape)
return Table(source, dshape=dshape)
@classmethod
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
def __repr__(self):
return generic_repr('Table', self, deferred=False)
def __init__(self,
data,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
if isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
class NDTable(Indexable, ArrayNode):
"""
The base NDTable. Indexable contains the indexing logic for
how to access elements, while ArrayNode contains the graph
related logic for building expression trees with this table
as an element.
"""
eclass = eclass.delayed
_metaheader = [
md.deferred,
md.tablelike,
]
def __init__(self,
data,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
if isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
@classmethod
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
@property
def datashape(self):
"""
Type deconstructor
"""
return self._datashape
@property
def size(self):
"""
Size of the NDTable.
"""
# TODO: need to generalize, not every Array will look
# like Numpy
return sum(i.val for i in self._datashape.parameters[:-1])
@property
def backends(self):
"""
The storage backends that make up the space behind the Array.
"""
return iter(self.space)
def __repr__(self):
return generic_repr('NDTable', self, deferred=True)
def __getitem__(self, mask):
ct = (self.data.ca[mask])
dshape = from_numpy(ct.shape, ct.dtype)
source = CTableSource(ct, dshape=dshape)
return Table(source, dshape=dshape)
class Table(Indexable):
eclass = eclass.manifest
_metaheader = [
md.manifest,
md.tablelike,
]
def __init__(self,
data,
dshape=None,
metadata=None,
layout=None,
params=None):
# Datashape
# ---------
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
if not dshape:
# The user just passed in a raw data source, try
# and infer how it should be layed out or fail
# back on dynamic types.
self._datashape = dshape = CTableSource.infer_datashape(data)
else:
# The user overlayed their custom dshape on this
# data, check if it makes sense
CTableSource.check_datashape(data, given_dshape=dshape)
self._datashape = dshape
# Source
# ------
if isinstance(data, ByteProvider):
self.data = data
elif isinstance(data, dict):
ct = self.from_dict(data)
self._axes = data.keys()
dshape = from_numpy(ct.shape, ct.dtype)
self.data = CTableSource(ct, dshape=dshape, params=params)
self._datashape = dshape
elif isinstance(data, (list, tuple)):
self.data = CTableSource(data, dshape=dshape, params=params)
# Pull the labels from the datashape
self._axes = self._datashape[-1].names
else:
raise ValueError
# children graph nodes
self.children = []
self.space = Space(self.data)
# Layout
# ------
if layout:
self._layout = layout
elif not layout:
self._layout = self.data.default_layout()
# Metadata
# --------
self._metadata = NDTable._metaheader + (metadata or [])
# Parameters
# ----------
self.params = params
# TODO: don't hardcode against carray, breaks down if we use
# something else
def append(self, data):
self.data.ca.append(data)
def commit(self):
self.data.ca.flush()
# TODO: don't hardcode against carray
def __len__(self):
return len(self.data.ca)
# TODO: don't hardcode against carray
def __getitem__(self, mask):
ct = (self.data.ca[mask])
dshape = from_numpy(ct.shape, ct.dtype)
source = CTableSource(ct, dshape=dshape)
return Table(source, dshape=dshape)
@classmethod
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
@property
def datashape(self):
"""
Type deconstructor
"""
return self._datashape
def __repr__(self):
return generic_repr('Table', self, deferred=False)
