def __init__(self, data, dshape=None, metadata=None, layout=None, params=None): # Datashape # --------- if isinstance(dshape, basestring): dshape = _dshape(dshape) if not dshape: # The user just passed in a raw data source, try # and infer how it should be layed out or fail # back on dynamic types. self._datashape = dshape = CTableSource.infer_datashape(data) else: # The user overlayed their custom dshape on this # data, check if it makes sense CTableSource.check_datashape(data, given_dshape=dshape) self._datashape = dshape # Source # ------ if isinstance(data, ByteProvider): self.data = data if isinstance(data, dict): ct = self.from_dict(data) self._axes = data.keys() dshape = from_numpy(ct.shape, ct.dtype) self.data = CTableSource(ct, dshape=dshape, params=params) self._datashape = dshape elif isinstance(data, (list, tuple)): self.data = CTableSource(data, dshape=dshape, params=params) # Pull the labels from the datashape self._axes = self._datashape[-1].names else: raise ValueError # children graph nodes self.children = [] self.space = Space(self.data) # Layout # ------ if layout: self._layout = layout elif not layout: self._layout = self.data.default_layout() # Metadata # -------- self._metadata = NDTable._metaheader + (metadata or []) # Parameters # ---------- self.params = params
def test_open_ctable(): with temp_dir() as temp: # Create an table on disk table_filename = os.path.join(temp, 'ctable') p = params(storage=table_filename) ds = dshape('1,{ x: int32; y: int32 }') t = CTableSource(data=[(1, 1), (2, 2)], dshape=ds, params=p) del t # Open table with open function uri = 'ctable://' + table_filename c = toplevel.open(uri) assert c.datashape == ds # Test delayed mode c = toplevel.open(uri, eclass=eclass.delayed) assert c.datashape == ds
class NDTable(Indexable, ArrayNode): """ The base NDTable. Indexable contains the indexing logic for how to access elements, while ArrayNode contains the graph related logic for building expression trees with this table as an element. """ eclass = eclass.delayed _metaheader = [ md.deferred, md.tablelike, ] def __init__(self, data, dshape=None, metadata=None, layout=None, params=None): # Datashape # --------- if isinstance(dshape, basestring): dshape = _dshape(dshape) if not dshape: # The user just passed in a raw data source, try # and infer how it should be layed out or fail # back on dynamic types. self._datashape = dshape = CTableSource.infer_datashape(data) else: # The user overlayed their custom dshape on this # data, check if it makes sense CTableSource.check_datashape(data, given_dshape=dshape) self._datashape = dshape # Source # ------ if isinstance(data, ByteProvider): self.data = data if isinstance(data, dict): ct = self.from_dict(data) self._axes = data.keys() dshape = from_numpy(ct.shape, ct.dtype) self.data = CTableSource(ct, dshape=dshape, params=params) self._datashape = dshape elif isinstance(data, (list, tuple)): self.data = CTableSource(data, dshape=dshape, params=params) # Pull the labels from the datashape self._axes = self._datashape[-1].names else: raise ValueError # children graph nodes self.children = [] self.space = Space(self.data) # Layout # ------ if layout: self._layout = layout elif not layout: self._layout = self.data.default_layout() # Metadata # -------- self._metadata = NDTable._metaheader + (metadata or []) # Parameters # ---------- self.params = params @classmethod def from_dict(self, data): dtype = dtype_from_dict(data) return fromiter(izip(*data.itervalues()), dtype, -1) @property def datashape(self): """ Type deconstructor """ return self._datashape @property def size(self): """ Size of the NDTable. """ # TODO: need to generalize, not every Array will look # like Numpy return sum(i.val for i in self._datashape.parameters[:-1]) @property def backends(self): """ The storage backends that make up the space behind the Array. """ return iter(self.space) def __repr__(self): return generic_repr('NDTable', self, deferred=True)
class Table(Indexable): eclass = eclass.manifest _metaheader = [ md.manifest, md.tablelike, ] def __init__(self, data, dshape=None, metadata=None, layout=None, params=None): # Datashape # --------- if isinstance(dshape, basestring): dshape = _dshape(dshape) if not dshape: # The user just passed in a raw data source, try # and infer how it should be layed out or fail # back on dynamic types. self._datashape = dshape = CTableSource.infer_datashape(data) else: # The user overlayed their custom dshape on this # data, check if it makes sense CTableSource.check_datashape(data, given_dshape=dshape) self._datashape = dshape # Source # ------ if isinstance(data, ByteProvider): self.data = data elif isinstance(data, dict): ct = self.from_dict(data) self._axes = data.keys() dshape = from_numpy(ct.shape, ct.dtype) self.data = CTableSource(ct, dshape=dshape, params=params) self._datashape = dshape elif isinstance(data, (list, tuple)): self.data = CTableSource(data, dshape=dshape, params=params) # Pull the labels from the datashape self._axes = self._datashape[-1].names else: raise ValueError # children graph nodes self.children = [] self.space = Space(self.data) # Layout # ------ if layout: self._layout = layout elif not layout: self._layout = self.data.default_layout() # Metadata # -------- self._metadata = NDTable._metaheader + (metadata or []) # Parameters # ---------- self.params = params # TODO: don't hardcode against carray, breaks down if we use # something else def append(self, data): self.data.ca.append(data) def commit(self): self.data.ca.flush() # TODO: don't hardcode against carray def __len__(self): return len(self.data.ca) # TODO: don't hardcode against carray def __getitem__(self, mask): ct = (self.data.ca[mask]) dshape = from_numpy(ct.shape, ct.dtype) source = CTableSource(ct, dshape=dshape) return Table(source, dshape=dshape) @classmethod def from_dict(self, data): dtype = dtype_from_dict(data) return fromiter(izip(*data.itervalues()), dtype, -1) def __repr__(self): return generic_repr('Table', self, deferred=False)
def __getitem__(self, mask): ct = (self.data.ca[mask]) dshape = from_numpy(ct.shape, ct.dtype) source = CTableSource(ct, dshape=dshape) return Table(source, dshape=dshape)
class Table(Indexable): eclass = eclass.manifest _metaheader = [ md.manifest, md.tablelike, ] def __init__(self, data, dshape=None, metadata=None, layout=None, params=None): # Datashape # --------- if isinstance(dshape, basestring): dshape = _dshape(dshape) if not dshape: # The user just passed in a raw data source, try # and infer how it should be layed out or fail # back on dynamic types. self._datashape = dshape = CTableSource.infer_datashape(data) else: # The user overlayed their custom dshape on this # data, check if it makes sense CTableSource.check_datashape(data, given_dshape=dshape) self._datashape = dshape # Source # ------ if isinstance(data, ByteProvider): self.data = data elif isinstance(data, dict): ct = self.from_dict(data) self._axes = data.keys() dshape = from_numpy(ct.shape, ct.dtype) self.data = CTableSource(ct, dshape=dshape, params=params) self._datashape = dshape elif isinstance(data, (list, tuple)): self.data = CTableSource(data, dshape=dshape, params=params) # Pull the labels from the datashape self._axes = self._datashape[-1].names else: raise ValueError # children graph nodes self.children = [] self.space = Space(self.data) # Layout # ------ if layout: self._layout = layout elif not layout: self._layout = self.data.default_layout() # Metadata # -------- self._metadata = NDTable._metaheader + (metadata or []) # Parameters # ---------- self.params = params # TODO: don't hardcode against carray, breaks down if we use # something else def append(self, data): self.data.ca.append(data) def commit(self): self.data.ca.flush() # TODO: don't hardcode against carray def __len__(self): return len(self.data.ca) # TODO: don't hardcode against carray def __getitem__(self, mask): ct = (self.data.ca[mask]) dshape = from_numpy(ct.shape, ct.dtype) source = CTableSource(ct, dshape=dshape) return Table(source, dshape=dshape) @classmethod def from_dict(self, data): dtype = dtype_from_dict(data) return fromiter(izip(*data.itervalues()), dtype, -1) @property def datashape(self): """ Type deconstructor """ return self._datashape def __repr__(self): return generic_repr('Table', self, deferred=False)