コード例 #1
0
ファイル: table.py プロジェクト: imclab/blaze
    def __init__(self, obj, dshape=None, metadata=None, layout=None,
            params=None):


        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # No explicit dshape provided. Infer it from the source.
            # if the source is another blaze array just use its shape,
            # otherwile let the provider infer its data_shape
            if isinstance(obj, Array):
                dshape = obj.datashape
            else:
                dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)

        self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.
        if isinstance(obj, ByteProvider):
            self.data = obj
        elif isinstance(obj, Array):
            self.data = CArraySource(obj.data,
                                     dshape=dshape,
                                     params=params)
        else:
            self.data = CArraySource(obj, dshape=dshape, params=params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata  = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
コード例 #2
0
ファイル: table.py プロジェクト: renjiec/blaze-core
    def __init__(self,
                 obj,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)
            self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.

        if isinstance(obj, CArraySource):
            self.data = obj
        else:
            self.data = CArraySource(obj, params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = ChunkedL(self.data, cdimension=0)

        # Metadata
        # --------

        self._metadata = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
コード例 #3
0
ファイル: table.py プロジェクト: kanghaiyang/blaze
    def __init__(self, obj, dshape=None, metadata=None, layout=None,
            params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)
            self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.

        if isinstance(obj, CArraySource):
            self.data = obj
        else:
            self.data = CArraySource(obj, params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = ChunkedL(self.data, cdimension=0)

        # Metadata
        # --------

        self._metadata  = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
コード例 #4
0
def test_open_carray():
    with temp_dir() as temp:
        # Create an array on disk
        array_filename = os.path.join(temp, 'carray')
        p = params(storage=array_filename)
        ds = dshape('1,int32')
        a = CArraySource([2], dshape=ds, params=p)
        del a

        # Open array with open function
        uri = 'carray://' + array_filename
        c = toplevel.open(uri)
        assert c.datashape == ds

        # Test delayed mode
        c = toplevel.open(uri, eclass=eclass.delayed)
        assert c.datashape == ds
コード例 #5
0
ファイル: table.py プロジェクト: kanghaiyang/blaze
class Array(Indexable):
    """
    Manifest array, does not create a graph. Forces evaluation on every
    call.

    Parameters
    ----------

        obj : A list of byte providers, other NDTables or a Python object.

    Optional
    --------

        datashape : dshape
            Manual datashape specification for the table, if None then
            shape will be inferred if possible.
        metadata :
            Manual datashape specification for the table, if None then
            shape will be inferred if possible.

    Usage
    -----

        >>> Array([1,2,3])
        >>> Array([1,2,3], dshape='3, int32')
        >>> Array([1,2,3], dshape('3, int32'))
        >>> Array([1,2,3], params=params(clevel=3, storage='file'))

    """

    eclass = eclass.manifest
    _metaheader = [
        md.manifest,
        md.arraylike,
    ]

    def __init__(self, obj, dshape=None, metadata=None, layout=None,
            params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # The user just passed in a raw data source, try
            # and infer how it should be layed out or fail
            # back on dynamic types.
            self._datashape = dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)
            self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.

        if isinstance(obj, ByteProvider):
            self.data = obj
        else:
            self.data = CArraySource(obj, params=params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata  = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    def _asdeferred(self):
        """ Convert a manifest array into a deferred array """
        return NDArray(
            self.data,
            dshape   = self._datashape,
            metadata = self._metadata,
            layout   = self._layout,
            params   = self.params
        )

    #------------------------------------------------------------------------
    # Properties
    #------------------------------------------------------------------------

    @property
    def datashape(self):
        """
        Type deconstructor
        """
        return self._datashape

    @property
    def size(self):
        """
        Size of the Array.
        """
        # TODO: need to generalize, not every Array will look
        # like Numpy
        return sum(i.val for i in self._datashape.parameters[:-1])

    @property
    def backends(self):
        """
        The storage backends that make up the space behind the
        Array.
        """
        return iter(self.space)

    #------------------------------------------------------------------------
    # Basic Slicing
    #------------------------------------------------------------------------

    # Immediete slicing
    def __getitem__(self, indexer):
        cc = self._layout.change_coordinates
        return retrieve(cc, indexer)

    # Immediete slicing ( Side-effectful )
    def __setitem__(self, indexer, value):
        cc = self._layout.change_coordinates
        write(cc, indexer, value)

    def __str__(self):
        return generic_str(self, deferred=False)

    def __repr__(self):
        return generic_repr('Array', self, deferred=False)
コード例 #6
0
ファイル: test_query.py プロジェクト: renjiec/blaze-core
def test_simple():
    a = CArraySource([1, 2, 3])
    layout = ContiguousL(a)
    indexer = (0, )

    retrieve(layout.change_coordinates, indexer)
コード例 #7
0
ファイル: table.py プロジェクト: renjiec/blaze-core
    def __init__(self,
                 obj,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # No explicit dshape provided. Infer it from the source.
            # if the source is another blaze array just use its shape,
            # otherwile let the provider infer its data_shape
            if isinstance(obj, Array):
                dshape = obj.datashape
            else:
                dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)

        self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.
        if isinstance(obj, ByteProvider):
            self.data = obj
        elif isinstance(obj, Array):
            self.data = CArraySource(obj.data, dshape=dshape, params=params)
        else:
            self.data = CArraySource(obj, dshape=dshape, params=params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params
コード例 #8
0
ファイル: table.py プロジェクト: renjiec/blaze-core
class Array(Indexable):
    """
    Manifest array, does not create a graph. Forces evaluation on every
    call.

    Parameters
    ----------

        obj : A list of byte providers, other NDTables or a Python object.

    Optional
    --------

        datashape : dshape
            Manual datashape specification for the table, if None then
            shape will be inferred if possible.
        metadata :
            Manual datashape specification for the table, if None then
            shape will be inferred if possible.

    Usage
    -----

        >>> Array([1,2,3])
        >>> Array([1,2,3], dshape='3, int32')
        >>> Array([1,2,3], dshape('3, int32'))
        >>> Array([1,2,3], params=params(clevel=3, storage='file'))

    """

    eclass = eclass.manifest
    _metaheader = [
        md.manifest,
        md.arraylike,
    ]

    def __init__(self,
                 obj,
                 dshape=None,
                 metadata=None,
                 layout=None,
                 params=None):

        # Datashape
        # ---------

        if isinstance(dshape, basestring):
            dshape = _dshape(dshape)

        if not dshape:
            # No explicit dshape provided. Infer it from the source.
            # if the source is another blaze array just use its shape,
            # otherwile let the provider infer its data_shape
            if isinstance(obj, Array):
                dshape = obj.datashape
            else:
                dshape = CArraySource.infer_datashape(obj)
        else:
            # The user overlayed their custom dshape on this
            # data, check if it makes sense
            CArraySource.check_datashape(obj, given_dshape=dshape)

        self._datashape = dshape

        # Values
        # ------
        # Mimic NumPy behavior in that we have a variety of
        # possible arguments to the first argument which result
        # in different behavior for the values.
        if isinstance(obj, ByteProvider):
            self.data = obj
        elif isinstance(obj, Array):
            self.data = CArraySource(obj.data, dshape=dshape, params=params)
        else:
            self.data = CArraySource(obj, dshape=dshape, params=params)

        # children graph nodes
        self.children = []

        self.space = Space(self.data)

        # Layout
        # ------

        if layout:
            self._layout = layout
        elif not layout:
            self._layout = self.data.default_layout()

        # Metadata
        # --------

        self._metadata = NDArray._metaheader + (metadata or [])

        # Parameters
        # ----------
        self.params = params

    def _asdeferred(self):
        """ Convert a manifest array into a deferred array """
        return NDArray(self.data,
                       dshape=self._datashape,
                       metadata=self._metadata,
                       layout=self._layout,
                       params=self.params)

    #------------------------------------------------------------------------
    # Properties
    #------------------------------------------------------------------------

    @property
    def datashape(self):
        """
        Type deconstructor
        """
        return self._datashape

    @property
    def size(self):
        """
        Size of the Array.
        """
        # TODO: need to generalize, not every Array will look
        # like Numpy
        return sum(i.val for i in self._datashape.parameters[:-1])

    @property
    def backends(self):
        """
        The storage backends that make up the space behind the
        Array.
        """
        return iter(self.space)

    #------------------------------------------------------------------------
    # Basic Slicing
    #------------------------------------------------------------------------

    # Immediete slicing
    def __getitem__(self, indexer):
        cc = self._layout.change_coordinates
        return retrieve(cc, indexer)

    # Immediete slicing ( Side-effectful )
    def __setitem__(self, indexer, value):
        cc = self._layout.change_coordinates
        write(cc, indexer, value)

    #------------------------------------------------------------------------
    # Specific functions for carray backend
    #------------------------------------------------------------------------

    # TODO: don't hardcode against carray,  breaks down if we use
    # something else
    def append(self, data):
        self.data.ca.append(data)
        # Update the shape
        shape, dtype = self.data.ca.shape, self.data.ca.dtype
        self._datashape = from_numpy(shape, dtype)

    def commit(self):
        self.data.ca.flush()

    def __str__(self):
        return generic_str(self, deferred=False)

    def __repr__(self):
        return generic_repr('Array', self, deferred=False)