def merge(self, *data, **kwargs):
"""
Merge two or more datasets into a single dataset.
This has the following effects:
All components from all datasets are added to the first argument
All datasets except the first argument are removed from the collection
Any component name conflicts are disambiguated
The pixel and world components apart from the first argument are discarded
:note: All arguments must have the same shape
:param data: One or more :class:`~glue.core.data.Data` instances.
:returns: self
"""
if len(data) < 2:
raise ValueError("merge requires 2 or more arguments")
shp = data[0].shape
for d in data:
if d.shape != shp:
raise ValueError("All arguments must have the same shape")
label = kwargs.get('label', data[0].label)
master = Data(label=label)
self.append(master)
master.coords = data[0].coords
for i, d in enumerate(data):
if isinstance(d.coords, WCSCoordinates):
master.coords = d.coords
break
# Find ambiguous components (ones which have labels in more than one
# dataset
from collections import Counter
clabel_count = Counter(
[c.label for d in data for c in d.visible_components])
for d in data:
for c in d.components:
if c in master.components: # already present (via a link)
continue
lbl = c.label
if clabel_count[lbl] > 1:
lbl = lbl + " [{0}]".format(d.label)
c._label = lbl
c.parent = master
master.add_component(d.get_component(c), c)
self.remove(d)
return self
def merge(self, *data, **kwargs):
"""
Merge two or more datasets into a single dataset.
This has the following effects:
All components from all datasets are added to the first argument
All datasets except the first argument are removed from the collection
Any component name conflicts are disambiguated
The pixel and world components apart from the first argument are discarded
:note: All arguments must have the same shape
:param data: One or more :class:`~glue.core.data.Data` instances.
:returns: self
"""
if len(data) < 2:
raise ValueError("merge requires 2 or more arguments")
shp = data[0].shape
for d in data:
if d.shape != shp:
raise ValueError("All arguments must have the same shape")
label = kwargs.get('label', data[0].label)
master = Data(label=label)
self.append(master)
master.coords = data[0].coords
for i, d in enumerate(data):
if isinstance(d.coords, WCSCoordinates):
master.coords = d.coords
break
# Find ambiguous components (ones which have labels in more than one
# dataset
from collections import Counter
clabel_count = Counter([c.label for d in data for c in d.visible_components])
for d in data:
for c in d.components:
if c in master.components: # already present (via a link)
continue
lbl = c.label
if clabel_count[lbl] > 1:
lbl = lbl + " [{0}]".format(d.label)
c._label = lbl
c.parent = master
master.add_component(d.get_component(c), c)
self.remove(d)
return self
def merge(self, *data, **kwargs):
"""
Merge two or more datasets into a single dataset.
This has the following effects:
All components from all datasets are added to the first argument
All datasets except the first argument are removed from the collection
Any component name conflicts are disambiguated
The pixel and world components apart from the first argument are discarded
:note: All arguments must have the same shape
:param data: One or more :class:`~glue.core.data.Data` instances.
:returns: self
"""
if len(data) < 2:
raise ValueError("merge requires 2 or more arguments")
shp = data[0].shape
for d in data:
if d.shape != shp:
raise ValueError("All arguments must have the same shape")
label = kwargs.get('label', data[0].label)
master = Data(label=label)
self.append(master)
master.coords = data[0].coords
for d in data:
skip = d.pixel_component_ids + d.world_component_ids
for c in d.components:
if c in skip:
continue
if c in master.components: # already present (via a link)
continue
taken = [_.label for _ in master.components]
lbl = c.label
# Special-case 'PRIMARY', rename to data label
if lbl == 'PRIMARY':
lbl = d.label
# First-pass disambiguation, try component_data
if lbl in taken:
lbl = '%s_%s' % (lbl, d.label)
lbl = disambiguate(lbl, taken)
c._label = lbl
master.add_component(d.get_component(c), c)
self.remove(d)
return self
def new_data(suffix=True):
if suffix:
label = '{0}[{1}]'.format(label_base, hdu_name)
else:
label = label_base
data = Data(label=label)
data.coords = coords
# We need to be careful here because some header values are special
# objects that we should convert to strings
for key, value in hdu.header.items():
if (key == 'COMMENT' or key == 'HISTORY'):
if key not in data.meta:
data.meta[key] = [str(value)]
else:
data.meta[key].append(str(value))
elif isinstance(value, str) or isinstance(value,
(int, float, bool)):
data.meta[key] = value
else:
data.meta[key] = str(value)
groups[hdu_name] = data
extension_by_shape[shape] = hdu_name
return data
def _nddata_to_glue_data(ndd, data_label):
if ndd.data.ndim != 2:
raise ValueError(f'Imviz cannot load this NDData with ndim={ndd.data.ndim}')
for attrib in ['data', 'mask', 'uncertainty']:
arr = getattr(ndd, attrib)
if arr is None:
continue
comp_label = attrib.upper()
cur_label = f'{data_label}[{comp_label}]'
cur_data = Data(label=cur_label)
cur_data.meta.update(ndd.meta)
if ndd.wcs is not None:
cur_data.coords = ndd.wcs
raw_arr = arr
if attrib == 'data':
bunit = ndd.unit or ''
elif attrib == 'uncertainty':
raw_arr = arr.array
bunit = arr.unit or ''
else:
bunit = ''
component = Component.autotyped(raw_arr, units=bunit)
cur_data.add_component(component=component, label=comp_label)
yield cur_data, cur_label
def _load_data(rec, context):
label = rec['label']
result = Data(label=label)
result.coords = context.object(rec['coords'])
# we manually rebuild pixel/world components, so
# we override this function. This is pretty ugly
result._create_pixel_and_world_components = lambda: None
comps = [list(map(context.object, [cid, comp]))
for cid, comp in rec['components']]
comps = sorted(comps,
key=lambda x: isinstance(x[1], (DerivedComponent,
CoordinateComponent)))
for cid, comp in comps:
if isinstance(comp, CoordinateComponent):
comp._data = result
result.add_component(comp, cid)
assert result._world_component_ids == []
coord = [c for c in comps if isinstance(c[1], CoordinateComponent)]
coord = [x[0] for x in sorted(coord, key=lambda x: x[1])]
assert len(coord) == result.ndim * 2
result._world_component_ids = coord[:len(coord) // 2]
result._pixel_component_ids = coord[len(coord) // 2:]
for s in rec['subsets']:
result.add_subset(context.object(s))
return result
def new_data():
label = '{0}[{1}]'.format(label_base, hdu_name)
data = Data(label=label)
data.coords = coords
groups[hdu_name] = data
extension_by_shape[shape] = hdu_name
return data
def load_stacked_sequence(self, raster_data):
for window, window_data in raster_data.items():
w_data = Data(label=f"{window.replace(' ', '_')}")
w_data.coords = window_data.wcs
w_data.add_component(Component(window_data.data),
f"{window.replace(' ', '_')}")
w_data.style = VisualAttributes(color='#7A617C')
self.datasets.append(w_data)
def new_data():
label = '{0}[{1}]'.format(
label_base,
hdu_name
)
data = Data(label=label)
data.coords = coords
groups[hdu_name] = data
extension_by_shape[shape] = hdu_name
return data
def _jwst2data(file_obj, ext, data_label):
comp_label = ext.upper()
new_data_label = f'{data_label}[{comp_label}]'
data = Data(label=new_data_label)
unit_attr = f'bunit_{ext}'
try:
# This is very specific to JWST pipeline image output.
with AsdfInFits.open(file_obj) as af:
dm = af.tree
dm_meta = af.tree["meta"]
if (unit_attr in dm_meta and
_validate_bunit(dm_meta[unit_attr], raise_error=False)):
bunit = dm_meta[unit_attr]
else:
bunit = ''
# This is instance of gwcs.WCS, not astropy.wcs.WCS
if 'wcs' in dm_meta:
data.coords = dm_meta['wcs']
imdata = dm[ext]
component = Component.autotyped(imdata, units=bunit)
# Might have bad GWCS. If so, we exclude it.
try:
data.add_component(component=component, label=comp_label)
except Exception: # pragma: no cover
data.coords = None
data.add_component(component=component, label=comp_label)
# TODO: Do not need this when jwst.datamodels finally its own package.
# This might happen for grism image; fall back to FITS loader without WCS.
except Exception:
if ext == 'data':
ext = 'sci'
hdu = file_obj[ext]
return _hdu2data(hdu, data_label, file_obj, include_wcs=False)
return data, new_data_label
def load_sunpy_map(self, sunpy_map):
sunpy_map_loaded = sunpy.map.Map(sunpy_map)
label = 'sunpy-map-' + sunpy_map_loaded.name
data = Data(label=label)
data.coords = sunpy_map_loaded.wcs # preferred way, preserves more info in some cases
data.meta = sunpy_map_loaded.meta
data.add_component(Component(sunpy_map_loaded.data),
sunpy_map_loaded.name)
data.style = VisualAttributes(color='#FDB813',
preferred_cmap=sunpy_map.cmap)
self.datasets.append(data)
def load_sequence(self, raster_data):
for window, window_data in raster_data.items():
for i, scan_data in enumerate(window_data):
w_data = Data(
label=
f"{window.replace(' ', '_')}-{scan_data.meta['OBSID']}-scan-{i}"
)
w_data.coords = scan_data.wcs
w_data.add_component(Component(scan_data.data),
f"{window.replace(' ', '_')}-scan-{i}")
w_data.meta = scan_data.meta
w_data.style = VisualAttributes(color='#5A4FCF')
self.datasets.append(w_data)
def load_sji(self, sji):
with fits.open(sji) as hdul:
hdul.verify("fix")
label = hdul[0].header['TDESC1'] + hdul[0].header['OBSID']
data = Data(label=label)
data.coords = WCSCoordinates(hdul[0].header)
data.meta = hdul[0].header
preferred_cmap_name = 'IRIS ' + hdul[0].header['TDESC1'].replace(
'_', ' ')
data.style = VisualAttributes(preferred_cmap=preferred_cmap_name)
data.add_component(Component(hdul[0].data), label)
self.datasets.append(data)
def _load_data(rec, context):
label = rec['label']
result = Data(label=label)
if 'coords' in rec:
result.coords = context.object(rec['coords'])
# we manually rebuild pixel/world components, so
# we override this function. This is pretty ugly
result._create_pixel_and_world_components = lambda ndim: None
comps = [
list(map(context.object, [cid, comp]))
for cid, comp in rec['components']
]
for icomp, (cid, comp) in enumerate(comps):
if isinstance(comp, CoordinateComponent):
comp._data = result
# For backward compatibility, we need to check for cases where
# the component ID for the pixel components was not a PixelComponentID
# and upgrade it to one. This can be removed once we no longer
# support pre-v0.8 session files.
if not comp.world and not isinstance(cid, PixelComponentID):
cid = PixelComponentID(comp.axis, cid.label, parent=cid.parent)
comps[icomp] = (cid, comp)
result.add_component(comp, cid)
assert result._world_component_ids == []
coord = [c for c in comps if isinstance(c[1], CoordinateComponent)]
coord = [x[0] for x in sorted(coord, key=lambda x: x[1])]
if getattr(result, 'coords') is not None:
assert len(coord) == result.ndim * 2
result._world_component_ids = coord[:len(coord) // 2]
result._pixel_component_ids = coord[len(coord) // 2:]
else:
assert len(coord) == result.ndim
result._pixel_component_ids = coord
# We can now re-generate the coordinate links
result._set_up_coordinate_component_links(result.ndim)
for s in rec['subsets']:
result.add_subset(context.object(s))
return result
def _parse_iris_raster(data, label):
"""
Parse IRIS Level 2 raster files so that it can be loaded by glue.
"""
w_dataset = []
for window, window_data in data.items():
for i, scan_data in enumerate(window_data):
w_data = Data(label=f"{window.replace(' ', '_')}-{scan_data.meta['OBSID']}-scan-{i}")
w_data.coords = WCSCoordinates(scan_data.wcs.to_header())
w_data.add_component(Component(scan_data.data), f"{window.replace(' ', '_')}-{scan_data.meta['OBSID']}-scan-{i}")
w_data.meta = scan_data.meta
w_data.style = VisualAttributes(color='#5A4FCF')
w_dataset.append(w_data)
return w_dataset
def _hdu2data(hdu, data_label, hdulist, include_wcs=True):
if 'BUNIT' in hdu.header and _validate_bunit(hdu.header['BUNIT'], raise_error=False):
bunit = hdu.header['BUNIT']
else:
bunit = ''
comp_label = f'{hdu.name.upper()},{hdu.ver}'
new_data_label = f'{data_label}[{comp_label}]'
data = Data(label=new_data_label)
if include_wcs:
data.coords = WCS(hdu.header, hdulist)
component = Component.autotyped(hdu.data, units=bunit)
data.add_component(component=component, label=comp_label)
return data, new_data_label
def casalike_cube(filename, **kwargs):
"""
This provides special support for 4D CASA FITS - like cubes,
which have 2 spatial axes, a spectral axis, and a stokes axis
in that order.
Each stokes cube is split out as a separate component
"""
from astropy.io import fits
result = Data()
with fits.open(filename, **kwargs) as hdulist:
array = hdulist[0].data
header = hdulist[0].header
result.coords = coordinates_from_header(header)
for i in range(array.shape[0]):
result.add_component(array[[i]], label='STOKES %i' % i)
return result
def casalike_cube(filename, **kwargs):
"""
This provides special support for 4D CASA FITS - like cubes,
which have 2 spatial axes, a spectral axis, and a stokes axis
in that order.
Each stokes cube is split out as a separate component
"""
from astropy.io import fits
result = Data()
with fits.open(filename, **kwargs) as hdulist:
array = hdulist[0].data
header = hdulist[0].header
result.coords = coordinates_from_header(header)
for i in range(array.shape[0]):
result.add_component(array[[i]], label='STOKES %i' % i)
return result
def _hdu2data(hdu, data_label, hdulist, include_wcs=True):
if 'BUNIT' in hdu.header:
bunit = _validate_bunit(hdu.header['BUNIT'], raise_error=False)
else:
bunit = ''
comp_label = f'{hdu.name.upper()},{hdu.ver}'
new_data_label = f'{data_label}[{comp_label}]'
data = Data(label=new_data_label)
if hdulist is not None and hdu.name != 'PRIMARY' and 'PRIMARY' in hdulist:
data.meta.update(dict(hdulist['PRIMARY'].header))
data.meta.update(dict(hdu.header))
if include_wcs:
data.coords = WCS(hdu.header, hdulist)
component = Component.autotyped(hdu.data, units=bunit)
data.add_component(component=component, label=comp_label)
return data, new_data_label
def _load_data(rec, context):
label = rec['label']
result = Data(label=label)
result.coords = context.object(rec['coords'])
# we manually rebuild pixel/world components, so
# we override this function. This is pretty ugly
result._create_pixel_and_world_components = lambda ndim: None
comps = [list(map(context.object, [cid, comp]))
for cid, comp in rec['components']]
for icomp, (cid, comp) in enumerate(comps):
if isinstance(comp, CoordinateComponent):
comp._data = result
# For backward compatibility, we need to check for cases where
# the component ID for the pixel components was not a PixelComponentID
# and upgrade it to one. This can be removed once we no longer
# support pre-v0.8 session files.
if not comp.world and not isinstance(cid, PixelComponentID):
cid = PixelComponentID(comp.axis, cid.label, parent=cid.parent)
comps[icomp] = (cid, comp)
result.add_component(comp, cid)
assert result._world_component_ids == []
coord = [c for c in comps if isinstance(c[1], CoordinateComponent)]
coord = [x[0] for x in sorted(coord, key=lambda x: x[1])]
assert len(coord) == result.ndim * 2
result._world_component_ids = coord[:len(coord) // 2]
result._pixel_component_ids = coord[len(coord) // 2:]
# We can now re-generate the coordinate links
result._set_up_coordinate_component_links(result.ndim)
for s in rec['subsets']:
result.add_subset(context.object(s))
return result
def img_data(file_name):
"""Load common image files into a Glue data object"""
result = Data()
data = img_loader(file_name)
data = np.flipud(data)
shp = data.shape
comps = []
labels = []
# split 3 color images into each color plane
if len(shp) == 3 and shp[2] in [3, 4]:
comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
labels.extend(['red', 'green', 'blue'])
if shp[2] == 4:
comps.append(data[:, :, 3])
labels.append('alpha')
else:
comps = [data]
labels = ['PRIMARY']
# look for AVM coordinate metadata
try:
from pyavm import AVM
avm = AVM(str(file_name)) # avoid unicode
wcs = avm.to_wcs()
except:
pass
else:
result.coords = coordinates_from_wcs(wcs)
for c, l in zip(comps, labels):
result.add_component(c, l)
return result
def img_data(file_name):
"""Load common image files into a Glue data object"""
result = Data()
data = img_loader(file_name)
data = np.flipud(data)
shp = data.shape
comps = []
labels = []
# split 3 color images into each color plane
if len(shp) == 3 and shp[2] in [3, 4]:
comps.extend([data[:, :, 0], data[:, :, 1], data[:, :, 2]])
labels.extend(['red', 'green', 'blue'])
if shp[2] == 4:
comps.append(data[:, :, 3])
labels.append('alpha')
else:
comps = [data]
labels = ['PRIMARY']
# look for AVM coordinate metadata
try:
from pyavm import AVM
avm = AVM.from_image(str(file_name)) # avoid unicode
wcs = avm.to_wcs()
except Exception:
pass
else:
result.coords = coordinates_from_wcs(wcs)
for c, l in zip(comps, labels):
result.add_component(c, l)
return result
def new_data(suffix=True):
if suffix:
label = '{0}[{1}]'.format(label_base, hdu_name)
else:
label = label_base
data = Data(label=label)
data.coords = coords
# We need to be careful here because some header values are special
# objects that we should convert to strings
for key, value in hdu.header.items():
if (key == 'COMMENT' or key == 'HISTORY'):
if key not in data.meta:
data.meta[key] = [str(value)]
else:
data.meta[key].append(str(value))
elif isinstance(value, string_types) or isinstance(value, (int, float, bool)):
data.meta[key] = value
else:
data.meta[key] = str(value)
groups[hdu_name] = data
extension_by_shape[shape] = hdu_name
return data
def casalike_cube(filename, **kwargs):
"""
This provides special support for 4D CASA FITS - like cubes,
which have 2 spatial axes, a spectral axis, and a stokes axis
in that order.
Each stokes cube is split out as a separate component
"""
from astropy.io import fits
result = Data()
if 'ignore_missing_end' not in kwargs:
kwargs['ignore_missing_end'] = True
with fits.open(filename, **kwargs) as hdulist:
array = hdulist[0].data
header = hdulist[0].header
result.coords = coordinates_from_header(header)
for i in range(array.shape[0]):
units = header.get('BUNIT')
component = Component.autotyped(array[[i]], units=units)
result.add_component(component, label='STOKES %i' % i)
return result
def gridded_data(filename, format='auto', **kwargs):
result = Data()
# Try and automatically find the format if not specified
if format == 'auto':
format = file_format(filename)
# Read in the data
if is_fits(filename):
from astropy.io import fits
arrays = extract_data_fits(filename, **kwargs)
header = fits.getheader(filename)
result.coords = coordinates_from_header(header)
elif is_hdf5(filename):
arrays = extract_data_hdf5(filename, **kwargs)
else:
raise Exception("Unkonwn format: %s" % format)
for component_name in arrays:
comp = Component.autotyped(arrays[component_name])
result.add_component(comp, component_name)
return result
def casalike_cube(filename, **kwargs):
"""
This provides special support for 4D CASA FITS - like cubes,
which have 2 spatial axes, a spectral axis, and a stokes axis
in that order.
Each stokes cube is split out as a separate component
"""
from astropy.io import fits
result = Data()
if 'ignore_missing_end' not in kwargs:
kwargs['ignore_missing_end'] = True
with fits.open(filename, mode='denywrite', **kwargs) as hdulist:
array = hdulist[0].data
header = hdulist[0].header
result.coords = coordinates_from_header(header)
for i in range(array.shape[0]):
units = header.get('BUNIT')
component = Component.autotyped(array[[i]], units=units)
result.add_component(component, label='STOKES %i' % i)
return result
def gridded_data(filename, format='auto', **kwargs):
result = Data()
# Try and automatically find the format if not specified
if format == 'auto':
format = file_format(filename)
# Read in the data
if is_fits(filename):
from astropy.io import fits
arrays = extract_data_fits(filename, **kwargs)
header = fits.getheader(filename)
result.coords = coordinates_from_header(header)
elif is_hdf5(filename):
arrays = extract_data_hdf5(filename, **kwargs)
else:
raise Exception("Unkonwn format: %s" % format)
for component_name in arrays:
comp = Component.autotyped(arrays[component_name])
result.add_component(comp, component_name)
return result
def merge(self, *data, **kwargs):
"""
Merge two or more datasets into a single dataset.
This has the following effects:
All components from all datasets are added to the first argument.
All datasets except the first argument are removed from the collection.
Any component name conflicts are disambiguated.
The pixel and world components apart from the first argument are discarded.
Parameters
----------
data : `iterable` of :class:`~glue.core.data.Data`
Two or more datasets to be added to this collection.
Notes
-----
All arguments must have the same shape.
Returns
-------
self
"""
if len(data) < 2:
raise ValueError("merge requires 2 or more arguments")
shp = data[0].shape
for d in data:
if d.shape != shp:
raise ValueError("All arguments must have the same shape")
label = kwargs.get('label', data[0].label)
master = Data(label=label)
self.append(master)
master.coords = data[0].coords
for i, d in enumerate(data):
if isinstance(d.coords, WCSCoordinates):
master.coords = d.coords
break
# Find ambiguous components (ones which have labels in more than one
# dataset
from collections import Counter
clabel_count = Counter([
c.label for d in data
for c in d.main_components + d.derived_components
])
for d in data:
for c in d.components:
if c in master.components: # already present (via a link)
continue
# Don't include coordinate components here as they will be
# recomputed separately once the first non-coordinate component
# is added.
if c in d.coordinate_components:
continue
lbl = c.label
if clabel_count[lbl] > 1:
lbl = lbl + " [{0}]".format(d.label)
c._label = lbl
c.parent = master
master.add_component(d.get_component(c), c)
self.remove(d)
return master
def merge(self, *data, **kwargs):
"""
Merge two or more datasets into a single dataset.
This has the following effects:
All components from all datasets are added to the first argument
All datasets except the first argument are removed from the collection
Any component name conflicts are disambiguated
The pixel and world components apart from the first argument are discarded
:note: All arguments must have the same shape
:param data: One or more :class:`~glue.core.data.Data` instances.
:returns: self
"""
if len(data) < 2:
raise ValueError("merge requires 2 or more arguments")
shp = data[0].shape
for d in data:
if d.shape != shp:
raise ValueError("All arguments must have the same shape")
label = kwargs.get('label', data[0].label)
master = Data(label=label)
self.append(master)
master.coords = data[0].coords
for i, d in enumerate(data):
if isinstance(d.coords, WCSCoordinates):
master.coords = d.coords
break
for d in data:
skip = d.pixel_component_ids + d.world_component_ids
for c in d.components:
if c in skip:
continue
if c in master.components: # already present (via a link)
continue
taken = [_.label for _ in master.components]
lbl = c.label
# Special-case 'PRIMARY', rename to data label
if lbl == 'PRIMARY':
lbl = d.label
# First-pass disambiguation, try component_data
if lbl in taken:
lbl = '%s_%s' % (lbl, d.label)
lbl = disambiguate(lbl, taken)
c._label = lbl
c.parent = master
master.add_component(d.get_component(c), c)
self.remove(d)
return self
