def is_npy_npz(filename):
"""
The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz)
"""
# See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
from numpy.lib.format import MAGIC_PREFIX
MAGIC_PREFIX_NPZ = b'PK\x03\x04' # first 4 bytes for a zipfile
tester = has_extension('npz .npz')
with open(filename, 'rb') as infile:
prefix = infile.read(6)
return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
def is_npy_npz(filename):
"""
The first bytes are x93NUMPY (for npy) or PKx03x04 (for npz)
"""
# See: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
from numpy.lib.format import MAGIC_PREFIX
MAGIC_PREFIX_NPZ = b'PK\x03\x04' # first 4 bytes for a zipfile
tester = has_extension('npz .npz')
with open(filename, 'rb') as infile:
prefix = infile.read(6)
return prefix == MAGIC_PREFIX or (tester(filename) and prefix[:4] == MAGIC_PREFIX_NPZ)
def is_npz(filename):
"""
The first bytes are: x93NUMPY
see: https://github.com/numpy/numpy/blob/master/doc/neps/npy-format.rst
"""
tester = has_extension('npz .npz')
MAGIC_PREFIX = b'PK\x03\x04' # first 4 bytes for a zipfile
with open(filename, 'rb') as infile:
check = infile.read(4) == MAGIC_PREFIX
return check and tester(filename)
try:
from skimage import img_as_ubyte
from skimage.io import imread
return np.asarray(img_as_ubyte(imread(file_name)))
except ImportError:
pass
try:
from PIL import Image
return np.asarray(Image.open(file_name))
except ImportError:
raise ImportError("Reading %s requires PIL or scikit-image" %
file_name)
@data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT)))
def img_data(file_name):
"""Load common image files into a Glue data object"""
result = Data()
data = img_loader(file_name)
data = np.flipud(data)
shp = data.shape
comps = []
labels = []
# split 3 color images into each color plane
if len(shp) == 3 and shp[2] in [3, 4]:
comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
labels.extend(['red', 'green', 'blue'])
if table.masked:
# fill array for now
try:
c = c.filled(fill_value=np.nan)
except (ValueError, TypeError): # assigning nan to integer dtype
c = c.filled(fill_value=-1)
nc = Component.autotyped(c, units=u)
result.add_component(nc, column_name)
return result
@data_factory(label="VO table",
identifier=has_extension('xml vot xml.gz vot.gz'),
priority=1)
def astropy_tabular_data_votable(*args, **kwargs):
kwargs['format'] = 'votable'
return astropy_tabular_data(*args, **kwargs)
@data_factory(label="FITS table",
identifier=has_extension('fits fits.gz fit fit.gz'),
priority=1)
def astropy_tabular_data_fits(*args, **kwargs):
kwargs['format'] = 'fits'
return astropy_tabular_data(*args, **kwargs)
# Add explicit factories for the formats which astropy.table
if table.masked:
# fill array for now
try:
c = c.filled(fill_value=np.nan)
except (ValueError, TypeError): # assigning nan to integer dtype
c = c.filled(fill_value=-1)
nc = Component.autotyped(c, units=u)
result.add_component(nc, column_name)
return result
@data_factory(label="VO table",
identifier=has_extension('xml vot xml.gz vot.gz'),
priority=1)
def astropy_tabular_data_votable(*args, **kwargs):
kwargs['format'] = 'votable'
return astropy_tabular_data(*args, **kwargs)
@data_factory(label="FITS table",
identifier=has_extension('fits fits.gz fit fit.gz'),
priority=1)
def astropy_tabular_data_fits(*args, **kwargs):
kwargs['format'] = 'fits'
return astropy_tabular_data(*args, **kwargs)
# Add explicit factories for the formats which astropy.table
# here.
if not isinstance(name, str):
name = str(name)
# strip off leading #
name = name.strip()
if name.startswith('#'):
name = name[1:].strip()
result.add_component(c, name)
return result
@data_factory(label="Pandas Table",
identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
""" A factory for reading tabular data using pandas
:param path: path/to/file
:param kwargs: All kwargs are passed to pandas.read_csv
:returns: :class:`glue.core.data.Data` object
"""
try:
from pandas.errors import ParserError
except ImportError:
try:
from pandas.io.common import CParserError as ParserError
except ImportError: # pragma: no cover
try:
from pandas.parser import CParserError as ParserError
except ImportError: # pragma: no cover
import numpy as np
from astropy.table import Table
from glue.core import Data, Component
from glue.core.data import CategoricalComponent
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs
from glue.external.astro import fits
from glue.utils import coerce_numeric
from .core.data_objects import CubeData, SpectrumData, ImageData
from .core.custom_registry import CubeDataIOError
@data_factory("STcube", has_extension("fits fit"))
def read_cube(filename, **kwargs):
cube_data = None
exclude_exts = []
data_collection = []
hdulist = fits.open(filename)
try:
cube_data = CubeData.read(hdulist)
except CubeDataIOError as e:
warnings.warn('No CubeData found in "{}": {}'.format(
filename,
e.message
))
if cube_data is not None:
data = Data()
from glue.core.data_factories.helpers import has_extension
from glue.config import data_factory
__all__ = ['tabular_data']
@data_factory(label="ASCII Table",
identifier=has_extension('csv txt tsv tbl dat '
'csv.gz txt.gz tbl.bz '
'dat.gz'),
priority=1)
def tabular_data(path, **kwargs):
from glue.core.data_factories.astropy_table import astropy_tabular_data
from glue.core.data_factories.pandas import pandas_read_table
for fac in [astropy_tabular_data, pandas_read_table]:
try:
return fac(path, **kwargs)
except Exception:
pass
else:
raise IOError("Could not parse file: %s" % path)
from os.path import basename
import warnings
from astropy.table import Table
from glue.core import Data, Component
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.core.coordinates import coordinates_from_header, coordinates_from_wcs
from glue.external.astro import fits
from .core.data_objects import CubeData
from .core.custom_registry import CubeDataIOError
@data_factory("STcube", has_extension("fits fit"))
def read_cube(filename, **kwargs):
cube_data = None
exclude_exts = []
data_collection = []
hdulist = fits.open(filename)
try:
cube_data = CubeData.read(hdulist)
except CubeDataIOError as e:
warnings.warn('No CubeData found in "{}": {}'.format(
filename,
e.message
))
if cube_data is not None:
data = Data()
import os
from glue.core.data_factories.helpers import has_extension
from glue.core.data_factories.pandas import panda_process
from glue.config import data_factory
__all__ = []
@data_factory(label="Excel", identifier=has_extension('xls xlsx'))
def panda_read_excel(path, sheet=None, **kwargs):
"""
A factory for reading excel data using pandas.
Parameters
----------
path : str
Path to the file.
sheet : str, optional
The sheet to read. If `None`, all sheets are read.
**kwargs
All other kwargs are passed to :func:`pandas.read_excel`.
"""
try:
import pandas as pd
except ImportError:
raise ImportError('Pandas is required for Excel input.')
from __future__ import absolute_import, division, print_function
from glue.core.data_factories.helpers import has_extension
from glue.config import data_factory
__all__ = ['tabular_data']
@data_factory(label="ASCII Table",
identifier=has_extension('csv txt tsv tbl dat '
'csv.gz txt.gz tbl.bz '
'dat.gz'),
priority=1)
def tabular_data(path, **kwargs):
from glue.core.data_factories.astropy_table import astropy_tabular_data
from glue.core.data_factories.pandas import pandas_read_table
for fac in [astropy_tabular_data, pandas_read_table]:
try:
return fac(path, **kwargs)
except:
pass
else:
raise IOError("Could not parse file: %s" % path)
from os.path import basename
from collections import defaultdict
from glue.core import Data, Component
from glue.core.coordinates import coordinates_from_header
from glue.config import data_factory
from glue.core.data_factories.helpers import has_extension
from glue.external.astro import fits
import numpy as np
from astropy.table import Table
from cube_tools.core.data_objects import CubeData
@data_factory('Generic FITS', has_extension('fits fit'))
def _load_fits_generic(filename, **kwargs):
hdulist = fits.open(filename)
groups = dict()
label_base = basename(filename).rpartition('.')[0]
if not label_base:
label_base = basename(filename)
for extnum, hdu in enumerate(hdulist):
if hdu.data is not None:
hdu_name = hdu.name if hdu.name else str(extnum)
if is_image_hdu(hdu):
shape = hdu.data.shape
try:
data = groups[shape]
# numbers, these are cast to numerical types, so we want to change that
# here.
if not isinstance(name, six.string_types):
name = str(name)
# strip off leading #
name = name.strip()
if name.startswith('#'):
name = name[1:].strip()
result.add_component(c, name)
return result
@data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
""" A factory for reading tabular data using pandas
:param path: path/to/file
:param kwargs: All kwargs are passed to pandas.read_csv
:returns: :class:`glue.core.data.Data` object
"""
import pandas as pd
try:
from pandas.parser import CParserError
except ImportError: # pragma: no cover
from pandas._parser import CParserError
# iterate over common delimiters to search for best option
delimiters = kwargs.pop('delimiter', [None] + list(',|\t '))
from __future__ import absolute_import, division, print_function
import os
from glue.core.data_factories.helpers import has_extension
from glue.core.data_factories.pandas import panda_process
from glue.config import data_factory
__all__ = []
@data_factory(label="Excel", identifier=has_extension("xls xlsx"))
def panda_read_excel(path, sheet=None, **kwargs):
""" A factory for reading excel data using pandas.
:param path: path/to/file
:param sheet: The sheet to read. If `None`, all sheets are read.
:param kwargs: All other kwargs are passed to pandas.read_excel
:return: core.data.Data object.
"""
try:
import pandas as pd
except ImportError:
raise ImportError("Pandas is required for Excel input.")
try:
import xlrd
except ImportError:
raise ImportError("xlrd is required for Excel input.")
try:
from skimage import img_as_ubyte
from skimage.io import imread
return np.asarray(img_as_ubyte(imread(file_name)))
except ImportError:
pass
try:
from PIL import Image
return np.asarray(Image.open(file_name))
except ImportError:
raise ImportError("Reading %s requires PIL or scikit-image" %
file_name)
@data_factory(label='Image', identifier=has_extension(' '.join(IMG_FMT)))
def img_data(file_name):
"""Load common image files into a Glue data object"""
result = Data()
data = img_loader(file_name)
data = np.flipud(data)
shp = data.shape
comps = []
labels = []
# split 3 color images into each color plane
if len(shp) == 3 and shp[2] in [3, 4]:
comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
labels.extend(['red', 'green', 'blue'])
# numbers, these are cast to numerical types, so we want to change that
# here.
if not isinstance(name, str):
name = str(name)
# strip off leading #
name = name.strip()
if name.startswith('#'):
name = name[1:].strip()
result.add_component(c, name)
return result
@data_factory(label="Pandas Table", identifier=has_extension('csv csv txt tsv tbl dat'))
def pandas_read_table(path, **kwargs):
"""
A factory for reading tabular data using pandas.
Parameters
----------
path : str
Path to the file.
**kwargs
All other kwargs are passed to :func:`pandas.read_csv`.
Returns
-------
:class:`glue.core.data.Data` object