def is_npy_npz(filename): """ The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz) """ # See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst from numpy.lib.format import MAGIC_PREFIX MAGIC_PREFIX_NPZ = b'PK\x03\x04' # first 4 bytes for a zipfile tester = has_extension('npz .npz') with open(filename, 'rb') as infile: prefix = infile.read(6) return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
def is_npy_npz(filename): """ The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz) """ # See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst from numpy.lib.format import MAGIC_PREFIX MAGIC_PREFIX_NPZ = b'PK\x03\x04' # first 4 bytes for a zipfile tester = has_extension('npz .npz') with open(filename, 'rb') as infile: prefix = infile.read(6) return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
def is_npz(filename): """ The first bytes are: x93NUMPY see: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst """ tester = has_extension('npz .npz') MAGIC_PREFIX = b'PK\x03\x04' # first 4 bytes for a zipfile with open(filename, 'rb') as infile: check = infile.read(4) == MAGIC_PREFIX return check and tester(filename)
try: from skimage import img_as_ubyte from skimage.io import imread return np.asarray(img_as_ubyte(imread(file_name))) except ImportError: pass try: from PIL import Image return np.asarray(Image.open(file_name)) except ImportError: raise ImportError("Reading %s requires PIL or scikit-image" % file_name) @data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT))) def img_data(file_name): """Load common image files into a Glue data object""" result = Data() data = img_loader(file_name) data = np.flipud(data) shp = data.shape comps = [] labels = [] # split 3 color images into each color plane if len(shp) == 3 and shp[2] in [3, 4]: comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]]) labels.extend(['red', 'green', 'blue'])
if table.masked: # fill array for now try: c = c.filled(fill_value=np.nan) except (ValueError, TypeError): # assigning nan to integer dtype c = c.filled(fill_value=-1) nc = Component.autotyped(c, units=u) result.add_component(nc, column_name) return result @data_factory(label="VO table", identifier=has_extension('xml vot xml.gz vot.gz'), priority=1) def astropy_tabular_data_votable(*args, **kwargs): kwargs['format'] = 'votable' return astropy_tabular_data(*args, **kwargs) @data_factory(label="FITS table", identifier=has_extension('fits fits.gz fit fit.gz'), priority=1) def astropy_tabular_data_fits(*args, **kwargs): kwargs['format'] = 'fits' return astropy_tabular_data(*args, **kwargs) # Add explicit factories for the formats which astropy.table
if table.masked: # fill array for now try: c = c.filled(fill_value=np.nan) except (ValueError, TypeError): # assigning nan to integer dtype c = c.filled(fill_value=-1) nc = Component.autotyped(c, units=u) result.add_component(nc, column_name) return result @data_factory(label="VO table", identifier=has_extension('xml vot xml.gz vot.gz'), priority=1) def astropy_tabular_data_votable(*args, **kwargs): kwargs['format'] = 'votable' return astropy_tabular_data(*args, **kwargs) @data_factory(label="FITS table", identifier=has_extension('fits fits.gz fit fit.gz'), priority=1) def astropy_tabular_data_fits(*args, **kwargs): kwargs['format'] = 'fits' return astropy_tabular_data(*args, **kwargs) # Add explicit factories for the formats which astropy.table
# here. if not isinstance(name, str): name = str(name) # strip off leading # name = name.strip() if name.startswith('#'): name = name[1:].strip() result.add_component(c, name) return result @data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat')) def pandas_read_table(path, **kwargs): """ A factory for reading tabular data using pandas :param path: path/to/file :param kwargs: All kwargs are passed to pandas.read_csv :returns: :class:`glue.core.data.Data` object """ try: from pandas.errors import ParserError except ImportError: try: from pandas.io.common import CParserError as ParserError except ImportError: # pragma: no cover try: from pandas.parser import CParserError as ParserError except ImportError: # pragma: no cover
import numpy as np from astropy.table import Table from glue.core import Data, Component from glue.core.data import CategoricalComponent from glue.config import data_factory from glue.core.data_factories.helpers import has_extension from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs from glue.external.astro import fits from glue.utils import coerce_numeric from .core.data_objects import CubeData, SpectrumData, ImageData from .core.custom_registry import CubeDataIOError @data_factory("STcube", has_extension("fits fit")) def read_cube(filename, **kwargs): cube_data = None exclude_exts = [] data_collection = [] hdulist = fits.open(filename) try: cube_data = CubeData.read(hdulist) except CubeDataIOError as e: warnings.warn('No CubeData found in "{}": {}'.format( filename, e.message )) if cube_data is not None: data = Data()
from glue.core.data_factories.helpers import has_extension from glue.config import data_factory __all__ = ['tabular_data'] @data_factory(label="ASCII Table", identifier=has_extension('csv txt tsv tbl dat ' 'csv.gz txt.gz tbl.bz ' 'dat.gz'), priority=1) def tabular_data(path, **kwargs): from glue.core.data_factories.astropy_table import astropy_tabular_data from glue.core.data_factories.pandas import pandas_read_table for fac in [astropy_tabular_data, pandas_read_table]: try: return fac(path, **kwargs) except Exception: pass else: raise IOError("Could not parse file: %s" % path)
from os.path import basename import warnings from astropy.table import Table from glue.core import Data, Component from glue.config import data_factory from glue.core.data_factories.helpers import has_extension from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs from glue.external.astro import fits from .core.data_objects import CubeData from .core.custom_registry import CubeDataIOError @data_factory("STcube", has_extension("fits fit")) def read_cube(filename, **kwargs): cube_data = None exclude_exts = [] data_collection = [] hdulist = fits.open(filename) try: cube_data = CubeData.read(hdulist) except CubeDataIOError as e: warnings.warn('No CubeData found in "{}": {}'.format( filename, e.message )) if cube_data is not None: data = Data()
import os from glue.core.data_factories.helpers import has_extension from glue.core.data_factories.pandas import panda_process from glue.config import data_factory __all__ = [] @data_factory(label="Excel", identifier=has_extension('xls xlsx')) def panda_read_excel(path, sheet=None, **kwargs): """ A factory for reading excel data using pandas. Parameters ---------- path : str Path to the file. sheet : str, optional The sheet to read. If `None`, all sheets are read. **kwargs All other kwargs are passed to :func:`pandas.read_excel`. """ try: import pandas as pd except ImportError: raise ImportError('Pandas is required for Excel input.')
from __future__ import absolute_import, division, print_function from glue.core.data_factories.helpers import has_extension from glue.config import data_factory __all__ = ['tabular_data'] @data_factory(label="ASCII Table", identifier=has_extension('csv txt tsv tbl dat ' 'csv.gz txt.gz tbl.bz ' 'dat.gz'), priority=1) def tabular_data(path, **kwargs): from glue.core.data_factories.astropy_table import astropy_tabular_data from glue.core.data_factories.pandas import pandas_read_table for fac in [astropy_tabular_data, pandas_read_table]: try: return fac(path, **kwargs) except: pass else: raise IOError("Could not parse file: %s" % path)
from os.path import basename from collections import defaultdict from glue.core import Data, Component from glue.core.coordinates import coordinates_from_header from glue.config import data_factory from glue.core.data_factories.helpers import has_extension from glue.external.astro import fits import numpy as np from astropy.table import Table from cube_tools.core.data_objects import CubeData @data_factory('Generic FITS', has_extension('fits fit')) def _load_fits_generic(filename, **kwargs): hdulist = fits.open(filename) groups = dict() label_base = basename(filename).rpartition('.')[0] if not label_base: label_base = basename(filename) for extnum, hdu in enumerate(hdulist): if hdu.data is not None: hdu_name = hdu.name if hdu.name else str(extnum) if is_image_hdu(hdu): shape = hdu.data.shape try: data = groups[shape]
# numbers, these are cast to numerical types, so we want to change that # here. if not isinstance(name, six.string_types): name = str(name) # strip off leading # name = name.strip() if name.startswith('#'): name = name[1:].strip() result.add_component(c, name) return result @data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat')) def pandas_read_table(path, **kwargs): """ A factory for reading tabular data using pandas :param path: path/to/file :param kwargs: All kwargs are passed to pandas.read_csv :returns: :class:`glue.core.data.Data` object """ import pandas as pd try: from pandas.parser import CParserError except ImportError: # pragma: no cover from pandas._parser import CParserError # iterate over common delimiters to search for best option delimiters = kwargs.pop('delimiter', [None] + list(',|\t '))
from __future__ import absolute_import, division, print_function import os from glue.core.data_factories.helpers import has_extension from glue.core.data_factories.pandas import panda_process from glue.config import data_factory __all__ = [] @data_factory(label="Excel", identifier=has_extension("xls xlsx")) def panda_read_excel(path, sheet=None, **kwargs): """ A factory for reading excel data using pandas. :param path: path/to/file :param sheet: The sheet to read. If `None`, all sheets are read. :param kwargs: All other kwargs are passed to pandas.read_excel :return: core.data.Data object. """ try: import pandas as pd except ImportError: raise ImportError("Pandas is required for Excel input.") try: import xlrd except ImportError: raise ImportError("xlrd is required for Excel input.")
try: from skimage import img_as_ubyte from skimage.io import imread return np.asarray(img_as_ubyte(imread(file_name))) except ImportError: pass try: from PIL import Image return np.asarray(Image.open(file_name)) except ImportError: raise ImportError("Reading %s requires PIL or scikit-image" % file_name) @data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT))) def img_data(file_name): """Load common image files into a Glue data object""" result = Data() data = img_loader(file_name) data = np.flipud(data) shp = data.shape comps = [] labels = [] # split 3 color images into each color plane if len(shp) == 3 and shp[2] in [3, 4]: comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]]) labels.extend(['red', 'green', 'blue'])
# numbers, these are cast to numerical types, so we want to change that # here. if not isinstance(name, str): name = str(name) # strip off leading # name = name.strip() if name.startswith('#'): name = name[1:].strip() result.add_component(c, name) return result @data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat')) def pandas_read_table(path, **kwargs): """ A factory for reading tabular data using pandas. Parameters ---------- path : str Path to the file. **kwargs All other kwargs are passed to :func:`pandas.read_csv`. Returns ------- :class:`glue.core.data.Data` object