Пример #1
0
def is_npy_npz(filename):
    """
    The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz)
    """
    # See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
    from numpy.lib.format import MAGIC_PREFIX
    MAGIC_PREFIX_NPZ = b'PK\x03\x04'  # first 4 bytes for a zipfile
    tester = has_extension('npz .npz')
    with open(filename, 'rb') as infile:
        prefix = infile.read(6)
    return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
Пример #2
0
def is_npy_npz(filename):
    """
    The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz)
    """
    # See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
    from numpy.lib.format import MAGIC_PREFIX
    MAGIC_PREFIX_NPZ = b'PK\x03\x04'  # first 4 bytes for a zipfile
    tester = has_extension('npz .npz')
    with open(filename, 'rb') as infile:
        prefix = infile.read(6)
    return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
Пример #3
0
def is_npz(filename):
    """
    The first bytes are: x93NUMPY
    see: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
    """
    tester = has_extension('npz .npz')

    MAGIC_PREFIX = b'PK\x03\x04' # first 4 bytes for a zipfile
    with open(filename, 'rb') as infile:
        check = infile.read(4) == MAGIC_PREFIX
    return check and tester(filename)
Пример #4
0
    try:
        from skimage import img_as_ubyte
        from skimage.io import imread
        return np.asarray(img_as_ubyte(imread(file_name)))
    except ImportError:
        pass

    try:
        from PIL import Image
        return np.asarray(Image.open(file_name))
    except ImportError:
        raise ImportError("Reading %s requires PIL or scikit-image" %
                          file_name)


@data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT)))
def img_data(file_name):
    """Load common image files into a Glue data object"""
    result = Data()

    data = img_loader(file_name)
    data = np.flipud(data)
    shp = data.shape

    comps = []
    labels = []

    # split 3 color images into each color plane
    if len(shp) == 3 and shp[2] in [3, 4]:
        comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
        labels.extend(['red', 'green', 'blue'])
Пример #5
0
        if table.masked:
            # fill array for now
            try:
                c = c.filled(fill_value=np.nan)
            except (ValueError, TypeError):  # assigning nan to integer dtype
                c = c.filled(fill_value=-1)

        nc = Component.autotyped(c, units=u)
        result.add_component(nc, column_name)

    return result


@data_factory(label="VO table",
              identifier=has_extension('xml vot xml.gz vot.gz'),
              priority=1)
def astropy_tabular_data_votable(*args, **kwargs):
    kwargs['format'] = 'votable'
    return astropy_tabular_data(*args, **kwargs)


@data_factory(label="FITS table",
              identifier=has_extension('fits fits.gz fit fit.gz'),
              priority=1)
def astropy_tabular_data_fits(*args, **kwargs):
    kwargs['format'] = 'fits'
    return astropy_tabular_data(*args, **kwargs)


# Add explicit factories for the formats which astropy.table
Пример #6
0
        if table.masked:
            # fill array for now
            try:
                c = c.filled(fill_value=np.nan)
            except (ValueError, TypeError):  # assigning nan to integer dtype
                c = c.filled(fill_value=-1)

        nc = Component.autotyped(c, units=u)
        result.add_component(nc, column_name)

    return result


@data_factory(label="VO table",
              identifier=has_extension('xml vot xml.gz vot.gz'),
              priority=1)
def astropy_tabular_data_votable(*args, **kwargs):
    kwargs['format'] = 'votable'
    return astropy_tabular_data(*args, **kwargs)


@data_factory(label="FITS table",
              identifier=has_extension('fits fits.gz fit fit.gz'),
              priority=1)
def astropy_tabular_data_fits(*args, **kwargs):
    kwargs['format'] = 'fits'
    return astropy_tabular_data(*args, **kwargs)


# Add explicit factories for the formats which astropy.table
Пример #7
0
        # here.
        if not isinstance(name, str):
            name = str(name)

        # strip off leading #
        name = name.strip()
        if name.startswith('#'):
            name = name[1:].strip()

        result.add_component(c, name)

    return result


@data_factory(label="Pandas Table",
              identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
    """ A factory for reading tabular data using pandas
    :param path: path/to/file
    :param kwargs: All kwargs are passed to pandas.read_csv
    :returns: :class:`glue.core.data.Data` object
    """
    try:
        from pandas.errors import ParserError
    except ImportError:
        try:
            from pandas.io.common import CParserError as ParserError
        except ImportError:  # pragma: no cover
            try:
                from pandas.parser import CParserError as ParserError
            except ImportError:  # pragma: no cover
Пример #8
0
import numpy as np
from astropy.table import Table

from glue.core import Data, Component
from glue.core.data import CategoricalComponent
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs
from glue.external.astro import fits
from glue.utils import coerce_numeric

from .core.data_objects import CubeData, SpectrumData, ImageData
from .core.custom_registry import CubeDataIOError


@data_factory("STcube", has_extension("fits fit"))
def read_cube(filename, **kwargs):
    cube_data = None
    exclude_exts = []
    data_collection = []
    hdulist = fits.open(filename)
    try:
        cube_data = CubeData.read(hdulist)
    except CubeDataIOError as e:
        warnings.warn('No CubeData found in "{}": {}'.format(
            filename,
            e.message
        ))

    if cube_data is not None:
        data = Data()
Пример #9
0
from glue.core.data_factories.helpers import has_extension
from glue.config import data_factory

__all__ = ['tabular_data']


@data_factory(label="ASCII Table",
              identifier=has_extension('csv txt tsv tbl dat '
                                       'csv.gz txt.gz tbl.bz '
                                       'dat.gz'),
              priority=1)
def tabular_data(path, **kwargs):
    from glue.core.data_factories.astropy_table import astropy_tabular_data
    from glue.core.data_factories.pandas import pandas_read_table
    for fac in [astropy_tabular_data, pandas_read_table]:
        try:
            return fac(path, **kwargs)
        except Exception:
            pass
    else:
        raise IOError("Could not parse file: %s" % path)
Пример #10
0
from os.path import basename
import warnings

from astropy.table import Table

from glue.core import Data, Component
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs
from glue.external.astro import fits

from .core.data_objects import CubeData
from .core.custom_registry import CubeDataIOError


@data_factory("STcube", has_extension("fits fit"))
def read_cube(filename, **kwargs):
    cube_data = None
    exclude_exts = []
    data_collection = []
    hdulist = fits.open(filename)
    try:
        cube_data = CubeData.read(hdulist)
    except CubeDataIOError as e:
        warnings.warn('No CubeData found in "{}": {}'.format(
            filename,
            e.message
        ))

    if cube_data is not None:
        data = Data()
Пример #11
0
import os

from glue.core.data_factories.helpers import has_extension
from glue.core.data_factories.pandas import panda_process
from glue.config import data_factory

__all__ = []


@data_factory(label="Excel", identifier=has_extension('xls xlsx'))
def panda_read_excel(path, sheet=None, **kwargs):
    """
    A factory for reading excel data using pandas.

    Parameters
    ----------
    path : str
        Path to the file.

    sheet : str, optional
        The sheet to read. If `None`, all sheets are read.

    **kwargs
        All other kwargs are passed to :func:`pandas.read_excel`.
    """

    try:
        import pandas as pd
    except ImportError:
        raise ImportError('Pandas is required for Excel input.')
Пример #12
0
from __future__ import absolute_import, division, print_function

from glue.core.data_factories.helpers import has_extension
from glue.config import data_factory


__all__ = ['tabular_data']


@data_factory(label="ASCII Table",
              identifier=has_extension('csv txt tsv tbl dat '
                                       'csv.gz txt.gz tbl.bz '
                                       'dat.gz'),
              priority=1)
def tabular_data(path, **kwargs):
    from glue.core.data_factories.astropy_table import astropy_tabular_data
    from glue.core.data_factories.pandas import pandas_read_table
    for fac in [astropy_tabular_data, pandas_read_table]:
        try:
            return fac(path, **kwargs)
        except:
            pass
    else:
        raise IOError("Could not parse file: %s" % path)
Пример #13
0
from os.path import basename
from collections import defaultdict

from glue.core import Data, Component
from glue.core.coordinates import coordinates_from_header
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.external.astro import fits

import numpy as np
from astropy.table import Table

from cube_tools.core.data_objects import CubeData


@data_factory('Generic FITS', has_extension('fits fit'))
def _load_fits_generic(filename, **kwargs):
    hdulist = fits.open(filename)
    groups = dict()
    label_base = basename(filename).rpartition('.')[0]

    if not label_base:
        label_base = basename(filename)

    for extnum, hdu in enumerate(hdulist):
        if hdu.data is not None:
            hdu_name = hdu.name if hdu.name else str(extnum)
            if is_image_hdu(hdu):
                shape = hdu.data.shape
                try:
                    data = groups[shape]
Пример #14
0
        # numbers, these are cast to numerical types, so we want to change that
        # here.
        if not isinstance(name, six.string_types):
            name = str(name)

        # strip off leading #
        name = name.strip()
        if name.startswith('#'):
            name = name[1:].strip()

        result.add_component(c, name)

    return result


@data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
    """ A factory for reading tabular data using pandas
    :param path: path/to/file
    :param kwargs: All kwargs are passed to pandas.read_csv
    :returns: :class:`glue.core.data.Data` object
    """
    import pandas as pd
    try:
        from pandas.parser import CParserError
    except ImportError:  # pragma: no cover
        from pandas._parser import CParserError

    # iterate over common delimiters to search for best option
    delimiters = kwargs.pop('delimiter', [None] + list(',|\t '))
Пример #15
0
from __future__ import absolute_import, division, print_function

import os

from glue.core.data_factories.helpers import has_extension
from glue.core.data_factories.pandas import panda_process
from glue.config import data_factory


__all__ = []


@data_factory(label="Excel", identifier=has_extension("xls xlsx"))
def panda_read_excel(path, sheet=None, **kwargs):
    """ A factory for reading excel data using pandas.
    :param path: path/to/file
    :param sheet: The sheet to read. If `None`, all sheets are read.
    :param kwargs: All other kwargs are passed to pandas.read_excel
    :return: core.data.Data object.
    """

    try:
        import pandas as pd
    except ImportError:
        raise ImportError("Pandas is required for Excel input.")

    try:
        import xlrd
    except ImportError:
        raise ImportError("xlrd is required for Excel input.")
Пример #16
0
    try:
        from skimage import img_as_ubyte
        from skimage.io import imread
        return np.asarray(img_as_ubyte(imread(file_name)))
    except ImportError:
        pass

    try:
        from PIL import Image
        return np.asarray(Image.open(file_name))
    except ImportError:
        raise ImportError("Reading %s requires PIL or scikit-image" %
                          file_name)


@data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT)))
def img_data(file_name):
    """Load common image files into a Glue data object"""
    result = Data()

    data = img_loader(file_name)
    data = np.flipud(data)
    shp = data.shape

    comps = []
    labels = []

    # split 3 color images into each color plane
    if len(shp) == 3 and shp[2] in [3, 4]:
        comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
        labels.extend(['red', 'green', 'blue'])
Пример #17
0
        # numbers, these are cast to numerical types, so we want to change that
        # here.
        if not isinstance(name, str):
            name = str(name)

        # strip off leading #
        name = name.strip()
        if name.startswith('#'):
            name = name[1:].strip()

        result.add_component(c, name)

    return result


@data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
    """
    A factory for reading tabular data using pandas.

    Parameters
    ----------
    path : str
        Path to the file.

    **kwargs
        All other kwargs are passed to :func:`pandas.read_csv`.

    Returns
    -------
    :class:`glue.core.data.Data` object