Esempio n. 1
0
    def __init__(self, data, dshape=None, metadata=None, layout=None,
            params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        if isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()

            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata  = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
Esempio n. 2
0
def test_open_ctable():
    with temp_dir() as temp:
        # Create an table on disk
        table_filename = os.path.join(temp, 'ctable')
        p = params(storage=table_filename)
        ds = dshape('1,{ x: int32; y: int32 }')
        t = CTableSource(data=[(1, 1), (2, 2)], dshape=ds, params=p)
        del t

        # Open table with open function
        uri = 'ctable://' + table_filename
        c = toplevel.open(uri)
        assert c.datashape == ds

        # Test delayed mode
        c = toplevel.open(uri, eclass=eclass.delayed)
        assert c.datashape == ds
Esempio n. 3
0
class NDTable(Indexable, ArrayNode):
    """
    The base NDTable. Indexable contains the indexing logic for
    how to access elements, while ArrayNode contains the graph
    related logic for building expression trees with this table
    as an element.
    """

    eclass = eclass.delayed
    _metaheader = [
        md.deferred,
        md.tablelike,
    ]

    def __init__(self, data, dshape=None, metadata=None, layout=None,
            params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        if isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()

            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata  = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    @classmethod
    def from_dict(self, data):
        dtype = dtype_from_dict(data)
        return fromiter(izip(*data.itervalues()), dtype, -1)

    @property
    def datashape(self):
        """
        Type deconstructor
        """
        return self._datashape

    @property
    def size(self):
        """
        Size of the NDTable.
        """
        # TODO: need to generalize, not every Array will look
        # like Numpy
        return sum(i.val for i in self._datashape.parameters[:-1])

    @property
    def backends(self):
        """
        The storage backends that make up the space behind the Array.
        """
        return iter(self.space)

    def __repr__(self):
        return generic_repr('NDTable', self, deferred=True)
Esempio n. 4
0
class Table(Indexable):

    eclass = eclass.manifest
    _metaheader = [
        md.manifest,
        md.tablelike,
    ]

    def __init__(self, data, dshape=None, metadata=None, layout=None,
            params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        elif isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()
            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata  = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    # TODO: don't hardcode against carray,  breaks down if we use
    # something else
    def append(self, data):
        self.data.ca.append(data)

    def commit(self):
        self.data.ca.flush()

    # TODO: don't hardcode against carray
    def __len__(self):
        return len(self.data.ca)

    # TODO: don't hardcode against carray
    def __getitem__(self, mask):
        ct = (self.data.ca[mask])
        dshape = from_numpy(ct.shape, ct.dtype)
        source = CTableSource(ct, dshape=dshape)
        return Table(source, dshape=dshape)

    @classmethod
    def from_dict(self, data):
        dtype = dtype_from_dict(data)
        return fromiter(izip(*data.itervalues()), dtype, -1)

    def __repr__(self):
        return generic_repr('Table', self, deferred=False)
Esempio n. 5
0
    def __init__(self,
                 data,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        if isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()

            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
Esempio n. 6
0
class NDTable(Indexable, ArrayNode):
    """
    The base NDTable. Indexable contains the indexing logic for
    how to access elements, while ArrayNode contains the graph
    related logic for building expression trees with this table
    as an element.
    """

    eclass = eclass.delayed
    _metaheader = [
        md.deferred,
        md.tablelike,
    ]

    def __init__(self,
                 data,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        if isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()

            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    @classmethod
    def from_dict(self, data):
        dtype = dtype_from_dict(data)
        return fromiter(izip(*data.itervalues()), dtype, -1)

    @property
    def datashape(self):
        """
        Type deconstructor
        """
        return self._datashape

    @property
    def size(self):
        """
        Size of the NDTable.
        """
        # TODO: need to generalize, not every Array will look
        # like Numpy
        return sum(i.val for i in self._datashape.parameters[:-1])

    @property
    def backends(self):
        """
        The storage backends that make up the space behind the Array.
        """
        return iter(self.space)

    def __repr__(self):
        return generic_repr('NDTable', self, deferred=True)
Esempio n. 7
0
 def __getitem__(self, mask):
     ct = (self.data.ca[mask])
     dshape = from_numpy(ct.shape, ct.dtype)
     source = CTableSource(ct, dshape=dshape)
     return Table(source, dshape=dshape)
Esempio n. 8
0
class Table(Indexable):

    eclass = eclass.manifest
    _metaheader = [
        md.manifest,
        md.tablelike,
    ]

    def __init__(self,
                 data,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CTableSource.infer_datashape(data)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CTableSource.check_datashape(data, given_dshape=dshape)
            self._datashape = dshape

        # Source
        # ------

        if isinstance(data, ByteProvider):
            self.data = data
        elif isinstance(data, dict):
            ct = self.from_dict(data)
            self._axes = data.keys()
            dshape = from_numpy(ct.shape, ct.dtype)
            self.data = CTableSource(ct, dshape=dshape, params=params)
            self._datashape = dshape
        elif isinstance(data, (list, tuple)):
            self.data = CTableSource(data, dshape=dshape, params=params)
            # Pull the labels from the datashape
            self._axes = self._datashape[-1].names
        else:
            raise ValueError

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata = NDTable._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    # TODO: don't hardcode against carray,  breaks down if we use
    # something else
    def append(self, data):
        self.data.ca.append(data)

    def commit(self):
        self.data.ca.flush()

    # TODO: don't hardcode against carray
    def __len__(self):
        return len(self.data.ca)

    # TODO: don't hardcode against carray
    def __getitem__(self, mask):
        ct = (self.data.ca[mask])
        dshape = from_numpy(ct.shape, ct.dtype)
        source = CTableSource(ct, dshape=dshape)
        return Table(source, dshape=dshape)

    @classmethod
    def from_dict(self, data):
        dtype = dtype_from_dict(data)
        return fromiter(izip(*data.itervalues()), dtype, -1)

    @property
    def datashape(self):
        """
        Type deconstructor
        """
        return self._datashape

    def __repr__(self):
        return generic_repr('Table', self, deferred=False)