def test_template_match_inverse_variance():
"""
Test template_match when both observed and template spectra have the same wavelength axis.
"""
# Seed np.random so that results are consistent
np.random.seed(42)
# Create test spectra
spec_axis = np.linspace(0, 50, 50) * u.AA
spec = Spectrum1D(spectral_axis=spec_axis,
flux=np.random.randn(50) * u.Jy,
uncertainty=InverseVariance(1 / np.random.sample(50)**2,
unit='1 / Jy2'))
spec1 = Spectrum1D(spectral_axis=spec_axis,
flux=np.random.randn(50) * u.Jy,
uncertainty=InverseVariance(1 / np.random.sample(50)**2,
unit='1 / Jy2'))
# Get result from template_match
tm_result = template_comparison.template_match(spec, spec1)
# Create new spectrum for comparison
spec_result = Spectrum1D(
spectral_axis=spec_axis,
flux=spec1.flux *
template_comparison._normalize_for_template_matching(spec, spec1))
assert quantity_allclose(tm_result[0].flux,
spec_result.flux,
atol=0.01 * u.Jy)
np.testing.assert_almost_equal(tm_result[3], 40093.28353756253)
def load_sdss_boss(path, hdu=1, **kwargs):
r"""
Read a spectrum from a path that is described by the SDSS BOSS 'spec' data model
https://data.SDSS.org/datamodel/files/BOSS_SPECTRO_REDUX/RUN2D/spectra/PLATE4/spec.html
:param path:
The local path of the spectrum.
:returns:
A `specutils.Spectrum1D` object.
"""
units = u.Unit("1e-17 erg / (Angstrom cm2 s)")
with fits.open(path, **kwargs) as image:
spectral_axis = 10**image[hdu].data["loglam"] * u.Angstrom
flux = np.atleast_2d(image[hdu].data["flux"]) * units
uncertainty = InverseVariance(image[hdu].data["ivar"].reshape(flux.shape))
meta = OrderedDict([
("header", image[0].header),
("hdu_headers", [hdu.header for hdu in image]),
("bitmask", dict(and_mask=image[hdu].data["and_mask"].reshape(flux.shape),
or_mask=image[hdu].data["or_mask"].reshape(flux.shape))),
("wavelength", image[hdu].data["wdisp"]),
("model", image[hdu].data["model"].reshape(flux.shape)),
])
return Spectrum1D(spectral_axis=spectral_axis, flux=flux, uncertainty=uncertainty, meta=meta)
def load_sdss_mastar(path, hdu=1, **kwargs):
r"""
Read a list of spectrum from a path that is described by the SDSS MaNGA MaStar data model,
which actually describes a collection of spectra of different sources:
https://data.sdss.org/datamodel/files/MANGA_SPECTRO_MASTAR/DRPVER/MPROCVER/mastar-goodspec-DRPVER-MPROCVER.html
:param path:
The local path of the spectrum.
:returns:
A `specutils.Spectrum1D` object.
"""
spectra = []
units = u.Unit("1e-17 erg / (Angstrom cm2 s)")
with fits.open(path, **kwargs) as image:
_hdu = image[hdu]
for i in range(_hdu.header["NAXIS2"]):
meta = OrderedDict(zip(_hdu.data.dtype.names, _hdu.data[i]))
spectral_axis = meta.pop("WAVE") * u.Angstrom
flux = np.atleast_2d(meta.pop("FLUX") * units)
uncertainty = InverseVariance(meta.pop("IVAR").reshape(flux.shape))
spectra.append(Spectrum1D(spectral_axis=spectral_axis,
flux=flux, uncertainty=uncertainty, meta=meta))
return SpectrumList(spectra)
def load_sdss_apstar(path, **kwargs):
r"""
Read a spectrum from a path that is described by the SDSS apStar data model
https://data.SDSS.org/datamodel/files/APOGEE_REDUX/APRED_VERS/APSTAR_VERS/TELESCOPE/LOCATION_ID/apStar.html
:param path:
The local path of the spectrum.
:returns:
A `specutils.Spectrum1D` object.
"""
units = u.Unit("1e-17 erg / (Angstrom cm2 s)")
with fits.open(path, **kwargs) as image:
# Build spectral axis ourselves because specutils does not handle
# log-linear transformations yet.
spectral_axis = _wcs_log_linear(image[1].header)
data_slice = kwargs.get("data_slice", None)
if data_slice is None:
slicer = np.atleast_2d
else:
slicer = lambda _: np.atleast_2d(_)[data_slice]
flux = slicer(image[1].data) * units
uncertainty = InverseVariance(slicer(image[2].data)**-2)
verbosity = kwargs.get("verbosity", 1)
snr = [image[0].header["SNR"]]
n_visits = image[0].header["NVISITS"]
if n_visits > 1:
snr.append(snr[0])
snr.extend([image[0].header[f"SNRVIS{i}"] for i in range(1, 1 + n_visits)])
if data_slice is not None:
snr = snr[data_slice]
meta = OrderedDict([
("header", image[0].header),
("bitmask", slicer(image[3].data)),
("snr", snr)
])
if verbosity >= 1:
meta["hdu_headers"] = [hdu.header for hdu in image]
if verbosity >= 2:
meta.update(OrderedDict([
("sky_flux", slicer(image[4].data) * units),
("sky_error", slicer(image[5].data) * units),
("telluric_flux", slicer(image[6].data) * units),
("telluric_error", slicer(image[7].data) * units),
("lsf_coefficients", slicer(image[8].data)),
("rv_ccf_structure", slicer(image[9].data)),
]))
return Spectrum1D(spectral_axis=spectral_axis, flux=flux, uncertainty=uncertainty, meta=meta)
def _load_manga_spectra(hdulist, per_unit=None, transpose=None):
""" Return a MaNGA Spectrum1D object
Returns a Spectrum1D object for a MaNGA data files. Set
`transpose` kwarg to True for MaNGA cubes, as they are flipped relative
to what Spectrum1D expects. Use the `per_unit` kwarg to indicate the
"spaxel" or "fiber" unit for cubes and rss files, respectively.
Parameters
----------
hdulist : fits.HDUList
A MaNGA read astropy fits HDUList
per_unit : astropy.units.Unit
An astropy unit to divide the default flux unit by
transpose : bool
If True, transpose the data arrays
Returns
-------
Spectrum1D
The spectrum contained in the file.
"""
unit = u.Unit('1e-17 erg / (Angstrom cm2 s)')
if per_unit:
unit = unit / per_unit
hdr = hdulist['PRIMARY'].header
wave = hdulist['WAVE'].data * u.angstrom
if transpose:
flux = hdulist['FLUX'].data.T * unit
ivar = InverseVariance(hdulist["IVAR"].data.T)
# SDSS masks are arrays of bit values storing multiple boolean conditions.
# Setting non-zero bit values to True to map to specutils standard
mask = hdulist['MASK'].data.T != 0
else:
flux = hdulist['FLUX'].data * unit
ivar = InverseVariance(hdulist["IVAR"].data)
mask = hdulist['MASK'].data != 0
return Spectrum1D(flux=flux,
meta={'header': hdr},
spectral_axis=wave,
uncertainty=ivar,
mask=mask)
def spPlate_loader(file_obj, limit=None, **kwargs):
"""
Loader for SDSS spPlate files, reading flux spectra from all fibres into single array.
Parameters
----------
file_obj: str, file-like, or HDUList
FITS file name, object (provided from name by Astropy I/O Registry),
or HDUList (as resulting from astropy.io.fits.open()).
limit : :class:`int`, optional
If set, only return the first `limit` spectra in `flux` array.
Returns
-------
Spectrum1D
The spectra represented by the wavelength solution from the header WCS
and the data array of the primary HDU (typically 640 along dimension 1).
"""
if isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist = file_obj
elif fits.util.fileobj_closed(file_obj):
hdulist = fits.open(file_obj.name, **kwargs)
else:
hdulist = fits.open(file_obj, **kwargs)
header = hdulist[0].header
meta = {'header': header}
wcs = WCS(header).dropaxis(1)
if limit is None:
limit = header['NAXIS2']
bunit = header.get('BUNIT', '1e-17 erg / (Angstrom cm2 s)')
if 'Ang' in bunit and 'strom' not in bunit:
bunit = bunit.replace('Ang', 'Angstrom')
flux_unit = Unit(bunit)
flux = hdulist[0].data[0:limit, :] * flux_unit
uncertainty = InverseVariance(hdulist[1].data[0:limit, :] / flux_unit**2)
# dispersion along NAXIS1 from the WCS
wcs = WCS(header).dropaxis(1)
dispersion = wcs.pixel_to_world(np.arange(flux.shape[-1]))
# convert out of logspace (default for spSpec/spPlate spectra)?
if header.get('DC-Flag', 1) == 1:
dispersion = 10**dispersion
dispersion_unit = Unit('Angstrom')
mask = hdulist[2].data[0:limit, :] != 0
meta['plugmap'] = Table.read(hdulist[5])[0:limit]
if not isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist.close()
return Spectrum1D(flux=flux, spectral_axis=dispersion*dispersion_unit,
uncertainty=uncertainty, meta=meta, mask=mask)
def load_sdss_apvisit(path, **kwargs):
r"""
Read a spectrum from a path that is described by the SDSS apVisit data model
https://data.SDSS.org/datamodel/files/APOGEE_REDUX/APRED_VERS/TELESCOPE/PLATE_ID/MJD5/apVisit.html
:param path:
The local path of the spectrum.
:returns:
A `specutils.Spectrum1D` object.
"""
units = u.Unit("1e-17 erg / (Angstrom cm2 s)")
with fits.open(path, **kwargs) as image:
image = fits.open(path, **kwargs)
order_and_shape = lambda A: A.flatten()[::-1]
spectral_axis = order_and_shape(image[4].data) * u.Angstrom
flux = order_and_shape(image[1].data) * units
uncertainty = InverseVariance(order_and_shape(image[2].data)**-2)
# Multiple spectra, so need to generate these and send back a SpectrumList
meta = OrderedDict([
("header", image[0].header),
("snr", image[0].header["snr"]),
])
verbosity = kwargs.get("verbosity", 1)
if verbosity >= 1:
meta["hdu_headers"] = [hdu.header for hdu in image]
meta.update(OrderedDict([
("bitmask", order_and_shape(image[3].data)),
]))
if verbosity >= 2:
meta.update(OrderedDict([
("sky_flux", order_and_shape(image[5].data) * units),
("sky_error", order_and_shape(image[6].data) * units),
("telluric_flux", order_and_shape(image[7].data) * units),
("telluric_error", order_and_shape(image[8].data) * units),
("wavelength_coefficients", image[9].data),
("lsf_coefficients", image[10].data)
]))
spectrum = Spectrum1D(
spectral_axis=spectral_axis,
flux=flux,
uncertainty=uncertainty,
meta=meta
)
return spectrum
def test_inverse_variance_uncert(self):
ccd = self.ccd
uncert = InverseVariance(0.01 * np.ones(self.shape),
unit=self.unit + '-2')
ccd.uncertainty = uncert
f = _extract_ccddata(ccd)
assert_equal(f['data'], 100 * np.ones(self.shape))
assert_equal(f['mask'], self.mask)
assert_equal(f['unit'], self.unit)
assert_equal(f['meta'], self.meta)
assert_equal(f['uncertainty'], 10 * np.ones(self.shape))
def spPlate_loader(file_obj, limit=None, **kwargs):
"""
Loader for SDSS spPlate files, reading flux spectra from all fibres into single array.
Parameters
----------
file_obj: str, file-like, or HDUList
FITS file name, object (provided from name by Astropy I/O Registry),
or HDUList (as resulting from astropy.io.fits.open()).
limit : :class:`int`, optional
If set, only return the first `limit` spectra in `flux` array.
Returns
-------
Spectrum1D
The spectra represented by the wavelength solution from the header WCS
and the data array of the primary HDU (typically 640 along dimension 1).
"""
with read_fileobj_or_hdulist(file_obj, **kwargs) as hdulist:
header = hdulist[0].header
meta = {'header': header}
wcs = WCS(header).dropaxis(1)
if limit is None:
limit = header['NAXIS2']
bunit = header.get('BUNIT', '1e-17 erg / (Angstrom cm2 s)')
if 'Ang' in bunit and 'strom' not in bunit:
bunit = bunit.replace('Ang', 'Angstrom')
flux_unit = Unit(bunit)
flux = hdulist[0].data[0:limit, :] * flux_unit
uncertainty = InverseVariance(hdulist[1].data[0:limit, :] /
flux_unit**2)
# Fix the WCS if it is claimed to be linear
if header.get('DC-Flag', 1) == 1:
fixed_wcs = _sdss_wcs_to_log_wcs(wcs)
else:
fixed_wcs = wcs
mask = hdulist[2].data[0:limit, :] != 0
meta['plugmap'] = Table.read(hdulist[5])[0:limit]
return Spectrum1D(flux=flux,
wcs=fixed_wcs,
uncertainty=uncertainty,
meta=meta,
mask=mask)
def test_same_grid_fluxconserving(simulated_spectra):
"""
Test that feeding in the original dispersion axis returns the
same flux after resampling.
"""
input_spectra = simulated_spectra.s1_um_mJy_e1
input_spectra.uncertainty = InverseVariance(
[0.5] * len(simulated_spectra.s1_um_mJy_e1.flux))
inst = FluxConservingResampler()
results = inst(input_spectra, simulated_spectra.s1_um_mJy_e1.spectral_axis)
assert np.allclose(np.array(simulated_spectra.s1_um_mJy_e1.flux),
np.array(results.flux))
assert np.allclose(input_spectra.uncertainty.array,
results.uncertainty.array)
def spec_loader(file_obj, **kwargs):
"""
Loader for SDSS-III/IV optical spectrum "spec" files.
Parameters
----------
file_obj: str, file-like, or HDUList
FITS file name, object (provided from name by Astropy I/O Registry),
or HDUList (as resulting from astropy.io.fits.open()).
Returns
-------
data: Spectrum1D
The spectrum that is represented by the 'loglam' (wavelength) and 'flux'
data columns in the BINTABLE extension of the FITS `file_obj`.
"""
if isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist = file_obj
elif fits.util.fileobj_closed(file_obj):
hdulist = fits.open(file_obj.name, **kwargs)
else:
hdulist = fits.open(file_obj, **kwargs)
header = hdulist[0].header
name = header.get('NAME')
meta = {'header': header}
bunit = header.get('BUNIT', '1e-17 erg / (Angstrom cm2 s)')
if 'Ang' in bunit and 'strom' not in bunit:
bunit = bunit.replace('Ang', 'Angstrom')
flux_unit = Unit(bunit)
# spectrum is in HDU 1
flux = hdulist[1].data['flux'] * flux_unit
uncertainty = InverseVariance(hdulist[1].data['ivar'] / flux_unit**2)
dispersion = 10**hdulist[1].data['loglam']
dispersion_unit = Unit('Angstrom')
mask = hdulist[1].data['and_mask'] != 0
if not isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist.close()
return Spectrum1D(flux=flux, spectral_axis=dispersion * dispersion_unit,
uncertainty=uncertainty, meta=meta, mask=mask)
def test_smooth_custom_kernel_uncertainty(simulated_spectra):
"""
Test CustomKernel smoothing with correct parmaeters.
"""
np.random.seed(42)
# Create a custom kernel (some weird asymmetric-ness)
numpy_kernel = np.array([0.5, 1, 2, 0.5, 0.2])
numpy_kernel = numpy_kernel / np.sum(numpy_kernel)
custom_kernel = convolution.CustomKernel(numpy_kernel)
spec1 = simulated_spectra.s1_um_mJy_e1
uncertainty = np.abs(np.random.random(spec1.flux.shape))
# Test StdDevUncertainty
spec1.uncertainty = StdDevUncertainty(uncertainty)
spec1_smoothed = convolution_smooth(spec1, custom_kernel)
tt = convolution.convolve(1 / (spec1.uncertainty.array**2), custom_kernel)
uncertainty_smoothed_astropy = 1 / np.sqrt(tt)
assert np.allclose(spec1_smoothed.uncertainty.array,
uncertainty_smoothed_astropy)
# Test VarianceUncertainty
spec1.uncertainty = VarianceUncertainty(uncertainty)
spec1_smoothed = convolution_smooth(spec1, custom_kernel)
uncertainty_smoothed_astropy = 1 / convolution.convolve(
1 / spec1.uncertainty.array, custom_kernel)
assert np.allclose(spec1_smoothed.uncertainty.array,
uncertainty_smoothed_astropy)
# Test InverseVariance
spec1.uncertainty = InverseVariance(uncertainty)
spec1_smoothed = convolution_smooth(spec1, custom_kernel)
uncertainty_smoothed_astropy = convolution.convolve(
spec1.uncertainty.array, custom_kernel)
assert np.allclose(spec1_smoothed.uncertainty.array,
uncertainty_smoothed_astropy)
def _load_manga_spectra(hdulist, per_unit=None):
""" Return a MaNGA Spectrum1D object
Returns a Spectrum1D object for a MaNGA data files. Use the `per_unit`
kwarg to indicate the "spaxel" or "fiber" unit for cubes and rss files,
respectively. Note that the spectral axis will automatically be moved to
be last during Spectrum1D initialization.
Parameters
----------
hdulist : fits.HDUList
A MaNGA read astropy fits HDUList
per_unit : astropy.units.Unit
An astropy unit to divide the default flux unit by
Returns
-------
Spectrum1D
The spectrum contained in the file.
"""
unit = u.Unit('1e-17 erg / (Angstrom cm2 s)')
if per_unit:
unit = unit / per_unit
hdr = hdulist['PRIMARY'].header
wcs = WCS(hdulist['FLUX'].header)
flux = hdulist['FLUX'].data * unit
ivar = InverseVariance(hdulist["IVAR"].data)
# SDSS masks are arrays of bit values storing multiple boolean conditions.
mask = hdulist['MASK'].data != 0
return Spectrum1D(flux=flux,
meta={'header': hdr},
wcs=wcs,
uncertainty=ivar,
mask=mask)
def read_spPlate(filename, limit=None):
"""Read a SDSS spPlate file.
Parameters
----------
filename : :class:`str`
Name of the spPlate file.
limit : :class:`int`, optional
If set, only return the first `limit` spectra.
Returns
-------
Spectrum1D
The spectra.
"""
with fits.open(filename) as hdulist:
header = hdulist[0].header
meta = {'header': header}
try:
flux_unit = u.Unit(hdulist[0].header['BUNIT'])
except ValueError:
flux_unit = u.Unit('1e-17 erg / (Angstrom cm2 s)')
if limit is None:
limit = header['NAXIS2']
flux = hdulist[0].data[0:limit, :] * flux_unit
wcs = WCS(header)
dispersion_unit = u.Unit('Angstrom')
dispersion = 10**wcs.all_pix2world(np.vstack((np.arange(flux.shape[1]),
np.zeros((flux.shape[1],)))).T,
0)[:, 0]
uncertainty = InverseVariance(hdulist[1].data[0:limit, :])
mask = hdulist[2].data[0:limit, :] != 0
meta['plugmap'] = Table.read(hdulist[5])[0:limit]
return Spectrum1D(flux=flux, spectral_axis=dispersion*dispersion_unit,
uncertainty=uncertainty, meta=meta, mask=mask)
def resample1d(self, orig_spectrum, fin_spec_axis):
"""
Create a re-sampling matrix to be used in re-sampling spectra in a way
that conserves flux. If an uncertainty is present in the input spectra
it will be propagated through to the final resampled output spectra
as an InverseVariance uncertainty.
Parameters
----------
orig_spectrum : `~specutils.Spectrum1D`
The original 1D spectrum.
fin_spec_axis : Quantity
The desired spectral axis array.
Returns
-------
resample_spectrum : `~specutils.Spectrum1D`
An output spectrum containing the resampled `~specutils.Spectrum1D`
"""
# Check if units on original spectrum and new wavelength (if defined)
# match
if isinstance(fin_spec_axis, Quantity):
if orig_spectrum.spectral_axis.unit != fin_spec_axis.unit:
raise ValueError(
"Original spectrum spectral axis grid and new"
"spectral axis grid must have the same units.")
if not isinstance(fin_spec_axis, SpectralAxis):
fin_spec_axis = SpectralAxis(fin_spec_axis)
# todo: Would be good to return uncertainty in type it was provided?
# todo: add in weighting options
# Get provided uncertainty into variance
if orig_spectrum.uncertainty is not None:
if isinstance(orig_spectrum.uncertainty, StdDevUncertainty):
pixel_uncer = np.square(orig_spectrum.uncertainty.array)
elif isinstance(orig_spectrum.uncertainty, VarianceUncertainty):
pixel_uncer = orig_spectrum.uncertainty.array
elif isinstance(orig_spectrum.uncertainty, InverseVariance):
pixel_uncer = np.reciprocal(orig_spectrum.uncertainty.array)
else:
pixel_uncer = None
orig_axis_in_fin = orig_spectrum.spectral_axis.to(fin_spec_axis.unit)
resample_grid = self._resample_matrix(orig_axis_in_fin, fin_spec_axis)
# Now for some broadcasting magic to handle multi dimensional flux inputs
# Essentially this part is inserting length one dimensions as fillers
# For example, if we have a (5,6,10) input flux, and an output grid
# of 3, flux will be broadcast to (5,6,1,10) and resample_grid will
# Be broadcast to (1,1,3,10). The sum then reduces down the 10, the
# original dispersion grid, leaving 3, the new dispersion grid, as
# the last index.
new_flux_shape = list(orig_spectrum.flux.shape)
new_flux_shape.insert(-1, 1)
in_flux = orig_spectrum.flux.reshape(new_flux_shape)
ones = [1] * len(orig_spectrum.flux.shape[:-1])
new_shape_resample_grid = ones + list(resample_grid.shape)
resample_grid = resample_grid.reshape(new_shape_resample_grid)
# Calculate final flux
out_flux = np.sum(in_flux * resample_grid, axis=-1) / np.sum(
resample_grid, axis=-1)
# Calculate output uncertainty
if pixel_uncer is not None:
pixel_uncer = pixel_uncer.reshape(new_flux_shape)
out_variance = np.sum(pixel_uncer * resample_grid**2,
axis=-1) / np.sum(resample_grid**2, axis=-1)
out_uncertainty = InverseVariance(np.reciprocal(out_variance))
else:
out_uncertainty = None
# nan-filling happens by default - replace with zeros if requested:
if self.extrapolation_treatment == 'zero_fill':
origedges = orig_spectrum.spectral_axis.bin_edges
off_edges = (fin_spec_axis < origedges[0]) | (origedges[-1] <
fin_spec_axis)
out_flux[off_edges] = 0
if out_uncertainty is not None:
out_uncertainty.array[off_edges] = 0
# todo: for now, use the units from the pre-resampled
# spectra, although if a unit is defined for fin_spec_axis and it doesn't
# match the input spectrum it won't work right, will have to think
# more about how to handle that... could convert before and after
# calculation, which is probably easiest. Matrix math algorithm is
# geometry based, so won't work to just let quantity math handle it.
resampled_spectrum = Spectrum1D(flux=out_flux,
spectral_axis=np.array(fin_spec_axis) *
orig_spectrum.spectral_axis.unit,
uncertainty=out_uncertainty)
return resampled_spectrum
def __init__(self, bands=[], wave={}, flux={}, ivar={}, mask=None, resolution_data=None,
fibermap=None, meta=None, extra=None, single=False, scores=None):
self._bands = tuple(bands)
self._single = single
self._ftype = np.float64
if single:
self._ftype = np.float32
self._reset_properties()
self.scores = scores
if meta is None:
self.meta = dict()
elif isinstance(meta, fits.Header):
self.meta = {'header': meta}
else:
self.meta = meta.copy()
nspec = 0
# check consistency of input dimensions
for b in self._bands:
if wave[b].ndim != 1:
raise ValueError("Wavelength array for band {} should have shape (Nwave, ).".format(b))
if flux[b].ndim != 2:
raise ValueError("Flux array for band {} should have shape (Nspec, Nwave).".format(b))
if flux[b].shape[1] != wave[b].shape[0]:
raise ValueError("Flux array wavelength dimension for band {} does not match wavelength grid.".format(b))
if nspec is None:
nspec = flux[b].shape[0]
if fibermap is not None:
if len(fibermap) != flux[b].shape[0]:
raise ValueError("Flux array number of spectra for band {} does not match fibermap.".format(b))
if ivar[b].shape != flux[b].shape:
raise ValueError("Inverse variance array dimensions do not match flux for band {}.".format(b))
if mask is not None:
if mask[b].shape != flux[b].shape:
raise ValueError("Mask array dimensions do not match flux for band {}.".format(b))
if mask[b].dtype not in (int, np.int64, np.int32, np.uint64, np.uint32):
raise ValueError("Bad mask type {}.".format(mask.dtype))
if resolution_data is not None:
if resolution_data[b].ndim != 3:
raise ValueError("Resolution array for band {} should have shape (Nspec, Ndiag, Nwave).".format(b))
if resolution_data[b].shape[0] != flux[b].shape[0]:
raise ValueError("Resolution array spectrum dimension for band {} does not match flux.".format(b))
if resolution_data[b].shape[2] != wave[b].shape[0]:
raise ValueError("Resolution array wavelength dimension for band {} does not match wavelength grid.".format(b))
if extra is not None:
for ex in extra[b].items():
if ex[1].shape != flux[b].shape:
raise ValueError("Extra arrays must have the same shape as the flux array.")
if fibermap is not None:
self.fibermap = fibermap.copy()
else:
self.fibermap = None
# copy band-based data
for b in self._bands:
band_meta = dict()
if mask is None:
band_meta['mask'] = None
bool_mask = None
else:
band_meta['mask'] = np.copy(mask[b])
bool_mask = band_meta['mask'] != 0
if resolution_data is None:
band_meta['resolution_data'] = None
band_meta['R'] = None
else:
band_meta['resolution_data'] = np.copy(resolution_data[b].astype(self._ftype))
band_meta['R'] = np.array([Resolution(r) for r in resolution_data[b]])
if extra is None:
band_meta['extra'] = None
else:
band_meta['extra'] = dict()
for k, v in extra[b].items():
band_meta['extra'][k] = np.copy(v.astype(self._ftype))
self.append(Spectrum1D(spectral_axis=np.copy(wave[b].astype(self._ftype))*u.Angstrom,
flux=np.copy(flux[b].astype(self._ftype))*u.Unit('10**-17 erg/(s cm2 Angstrom)'),
uncertainty=InverseVariance(np.copy(ivar[b].astype(self._ftype))),
mask=bool_mask,
meta=band_meta))
# NUV image
filename = data_ext / 'HST' / 'filterImages' / f'hlsp_phangs-hst_hst_wfc3-uvis_{gal_name.lower()}_f275w_v1_exp-drc-sci.fits'
error_file = data_ext / 'HST' / 'filterImages' / f'hlsp_phangs-hst_hst_wfc3-uvis_{gal_name.lower()}_f275w_v1_err-drc-wht.fits'
if not filename.is_file():
print(f'no NUV data for {gal_name}')
continue
else:
with fits.open(filename) as hdul:
F275 = NDData(hdul[0].data,
mask=hdul[0].data == 0,
meta=hdul[0].header,
wcs=WCS(hdul[0].header))
with fits.open(error_file) as hdul:
F275.uncertainty = InverseVariance(hdul[0].data)
print(f'read in HST data')
muse_regions = Regions(mask=nebulae_mask.data,
projection=nebulae_mask.meta,
bkg=-1)
hst_regions = muse_regions.reproject(F275.meta)
print('regions reprojected')
muse_reproj, footprint = reproject_interp(
(nebulae_mask.mask, nebulae_mask.wcs), F275.meta)
mean, median, std = sigma_clipped_stats(F275.data[footprint.astype(bool)])
print('measuring sigma_clipped_stats')
tmp = nebulae[nebulae['gal_name'] == gal_name]
def _jwst_s3d_loader(filename, **kwargs):
"""
Loader for JWST s3d 3D rectified spectral data in FITS format.
Parameters
----------
filename : str
The path to the FITS file
Returns
-------
SpectrumList
The spectra contained in the file.
"""
spectra = []
# Get a list of GWCS objects from the slits
with asdf.open(filename) as af:
wcslist = [af.tree["meta"]["wcs"]]
with fits.open(filename, memmap=False) as hdulist:
primary_header = hdulist["PRIMARY"].header
hdulist_sci = [hdu for hdu in hdulist if hdu.name == "SCI"]
for hdu, wcs in zip(hdulist_sci, wcslist):
# Get flux
try:
flux_unit = u.Unit(hdu.header["BUNIT"])
except (ValueError, KeyError):
flux_unit = None
# The spectral axis is first. We need it last
flux_array = hdu.data.T
flux = Quantity(flux_array, unit=flux_unit)
# Get the wavelength array from the GWCS object which returns a
# tuple of (RA, Dec, lambda).
# Since the spatial and spectral axes are orthogonal in s3d data,
# it is much faster to compute a slice down the spectral axis.
grid = grid_from_bounding_box(wcs.bounding_box)[:, :, 0, 0]
_, _, wavelength_array = wcs(*grid)
_, _, wavelength_unit = wcs.output_frame.unit
wavelength = Quantity(wavelength_array, unit=wavelength_unit)
# Merge primary and slit headers and dump into meta
slit_header = hdu.header
header = primary_header.copy()
header.extend(slit_header, strip=True, update=True)
meta = {'header': header}
# get uncertainty information
ext_name = primary_header.get("ERREXT", "ERR")
err_type = hdulist[ext_name].header.get("ERRTYPE", 'ERR')
err_unit = hdulist[ext_name].header.get("BUNIT", None)
err_array = hdulist[ext_name].data.T
# ERRTYPE can be one of "ERR", "IERR", "VAR", "IVAR"
# but mostly ERR for JWST cubes
# see https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html#s3d
if err_type == "ERR":
err = StdDevUncertainty(err_array, unit=err_unit)
elif err_type == 'VAR':
err = VarianceUncertainty(err_array, unit=err_unit)
elif err_type == 'IVAR':
err = InverseVariance(err_array, unit=err_unit)
elif err_type == 'IERR':
warnings.warn(
"Inverse error is not yet a supported astropy.nddata "
"uncertainty. Setting err to None.")
err = None
# get mask information
mask_name = primary_header.get("MASKEXT", "DQ")
mask = hdulist[mask_name].data.T
spec = Spectrum1D(flux=flux,
spectral_axis=wavelength,
meta=meta,
uncertainty=err,
mask=mask)
spectra.append(spec)
return SpectrumList(spectra)
def update(self, other):
"""
Overwrite or append new data.
Given another Spectra object, compare the fibermap information with
the existing one. For spectra that already exist, overwrite existing
data with the new values. For spectra that do not exist, append that
data to the end of the spectral data.
Args:
other (Spectra): the new data to add.
Returns:
nothing (object updated in place).
"""
if not isinstance(other, Spectra):
raise ValueError("New data has incorrect type!")
# Does the other Spectra object have any data?
if other.num_spectra() == 0:
return
# Do we have new bands to add?
newbands = []
for b in other.bands:
if b not in self.bands:
newbands.append(b)
else:
if not np.allclose(self.wave[b], other.wave[b]):
raise ValueError("Band {} has an incompatible wavelength grid.".format(b))
bands = list(self.bands)
bands.extend(newbands)
# Are we adding mask data in this update?
add_mask = False
if other.mask is None:
if self.mask is not None:
raise ValueError("Existing spectra has a mask, cannot "
"update it to a spectra with no mask.")
else:
if self.mask is None:
add_mask = True
# Are we adding resolution data in this update?
ndiag = {}
add_res = False
if other.resolution_data is None:
if self.resolution_data is not None:
raise ValueError("Existing spectra has resolution data, cannot "
"update it to a spectra with none.")
else:
if self.resolution_data is not None:
for b in self.bands:
ndiag[b] = self.resolution_data[b].shape[1]
for b in other.bands:
odiag = other.resolution_data[b].shape[1]
if b not in self.bands:
ndiag[b] = odiag
else:
if odiag != ndiag[b]:
raise ValueError("Resolution matrices for a"
" given band must have the same dimensions.")
else:
add_res = True
for b in other.bands:
ndiag[b] = other.resolution_data[b].shape[1]
# Are we adding extra data in this update?
add_extra = False
if other.extra is None:
if self.extra is not None:
raise ValueError("Existing spectra has extra data, cannot "
"update it to a spectra with none.")
else:
if self.extra is None:
add_extra = True
# Compute which targets / exposures are new
nother = len(other.fibermap)
exists = np.zeros(nother, dtype=np.int)
indx_original = []
if self.fibermap is not None:
for r in range(nother):
expid = other.fibermap[r]["EXPID"]
fiber = other.fibermap[r]["FIBER"]
for i, row in enumerate(self.fibermap):
if (expid == row["EXPID"]) and (fiber == row["FIBER"]):
indx_original.append(i)
exists[r] += 1
if len(np.where(exists > 1)[0]) > 0:
raise ValueError("Found duplicate spectra (same EXPID and FIBER) in the fibermap.")
indx_exists = np.where(exists == 1)[0]
indx_new = np.where(exists == 0)[0]
# Make new data arrays of the correct size to hold both the old and
# new data
nupdate = len(indx_exists)
nnew = len(indx_new)
if self.fibermap is None:
nold = 0
newfmap = other.fibermap.copy()
else:
nold = len(self.fibermap)
newfmap = encode_table(np.zeros( (nold + nnew, ),
dtype=self.fibermap.dtype))
if self.scores is None:
if other.scores is None:
newscores = None
else:
newscores = other.scores.copy()
else:
newscores = encode_table(np.zeros( (nold + nnew, ),
dtype=self.scores.dtype))
newwave = {}
newflux = {}
newivar = {}
newmask = None
if add_mask or self.mask is not None:
newmask = {}
newres = None
newR = None
if add_res or self.resolution_data is not None:
newres = {}
newR = {}
newextra = None
if add_extra or self.extra is not None:
newextra = {}
for b in bands:
nwave = None
if b in self.bands:
nwave = self.wave[b].shape[0]
newwave[b] = self.wave[b]
else:
nwave = other.wave[b].shape[0]
newwave[b] = other.wave[b].astype(self._ftype)
newflux[b] = np.zeros( (nold + nnew, nwave), dtype=self._ftype)
newivar[b] = np.zeros( (nold + nnew, nwave), dtype=self._ftype)
if newmask is not None:
newmask[b] = np.zeros( (nold + nnew, nwave), dtype=np.uint32)
newmask[b][:,:] = specmask["NODATA"]
if newres is not None:
newres[b] = np.zeros( (nold + nnew, ndiag[b], nwave), dtype=self._ftype)
if newextra is not None:
newextra[b] = {}
# Copy the old data
if nold > 0:
# We have some data (i.e. we are not starting with an empty Spectra)
newfmap[:nold] = self.fibermap
if newscores is not None:
newscores[:nold] = self.scores
for b in self.bands:
newflux[b][:nold,:] = self.flux[b]
newivar[b][:nold,:] = self.ivar[b]
if self.mask is not None:
newmask[b][:nold,:] = self.mask[b]
elif add_mask:
newmask[b][:nold,:] = 0
if self.resolution_data is not None:
newres[b][:nold,:,:] = self.resolution_data[b]
if self.extra is not None:
for ex in self.extra[b].items():
newextra[b][ex[0]] = np.zeros( newflux[b].shape,
dtype=self._ftype)
newextra[b][ex[0]][:nold,:] = ex[1]
# Update existing spectra
for i, s in enumerate(indx_exists):
row = indx_original[i]
for b in other.bands:
newflux[b][row,:] = other.flux[b][s,:].astype(self._ftype)
newivar[b][row,:] = other.ivar[b][s,:].astype(self._ftype)
if other.mask is not None:
newmask[b][row,:] = other.mask[b][s,:]
else:
newmask[b][row,:] = 0
if other.resolution_data is not None:
newres[b][row,:,:] = other.resolution_data[b][s,:,:].astype(self._ftype)
if other.extra is not None:
for ex in other.extra[b].items():
if ex[0] not in newextra[b]:
newextra[b][ex[0]] = np.zeros(newflux[b].shape,
dtype=self._ftype)
newextra[b][ex[0]][row,:] = ex[1][s,:].astype(self._ftype)
# Append new spectra
if nnew > 0:
newfmap[nold:] = other.fibermap[indx_new]
if newscores is not None:
newscores[nold:] = other.scores[indx_new]
for b in other.bands:
newflux[b][nold:,:] = other.flux[b][indx_new].astype(self._ftype)
newivar[b][nold:,:] = other.ivar[b][indx_new].astype(self._ftype)
if other.mask is not None:
newmask[b][nold:,:] = other.mask[b][indx_new]
else:
newmask[b][nold:,:] = 0
if other.resolution_data is not None:
newres[b][nold:,:,:] = other.resolution_data[b][indx_new].astype(self._ftype)
if other.extra is not None:
for ex in other.extra[b].items():
if ex[0] not in newextra[b]:
newextra[b][ex[0]] = np.zeros(newflux[b].shape,
dtype=self._ftype)
newextra[b][ex[0]][nold:,:] = ex[1][indx_new].astype(self._ftype)
# Swap data into place
self._bands = bands
self.fibermap = newfmap
self.scores = newscores
self._reset_properties()
for i, b in enumerate(self._bands):
band_meta = dict()
if newmask is None:
band_meta['mask'] = None
bool_mask = None
else:
band_meta['mask'] = newmask[b]
bool_mask = band_meta['mask'] != 0
if newres is None:
band_meta['resolution_data'] = None
band_meta['R'] = None
else:
band_meta['resolution_data'] = newres[b]
band_meta['R'] = np.array([Resolution(r) for r in newres[b]])
if newextra is None:
band_meta['extra'] = None
else:
band_meta['extra'] = dict()
for k, v in newextra[b].items():
band_meta['extra'][k] = v
s = Spectrum1D(spectral_axis=newwave[b]*u.Angstrom,
flux=newflux[b]*u.Unit('10**-17 erg/(s cm2 Angstrom)'),
uncertainty=InverseVariance(newivar),
mask=bool_mask,
meta=band_meta)
try:
self[i] = s
except IndexError:
self.append(s)
return
def chunk_redshift(data_wave, data_flux, data_noise, template_path, z_lit, targ_delta, overhang, z_test, z_bound, position):
"""Returns the bestfit redshift of each chunk.
Parameters
----------
data_wave : tuple
Data wavelength array
data_flux : tuple
Data flux array
data_noise : tuple
Data noise array
template_path : str
Path to template spectrum file
z_lit : float
Literature redshift of target object
targ_delta : float
Wavelength chunk size in Angstroms
overhang : float
Amount of wavelength overhang template chunks should have in Angstroms
z_test : float
Starting redshift for chunks (measured by eye in 1 chunk)
z_bound : float
Amount to add and subtract from z_lit for redshifts to test
position : str
'before' or 'after' to indicate if pre- or post-flexure correction
Results
-------
bestfit_redshift : tuple
Best fitting redshift for each chunk
best_chi2 : tuple
Minimum chi squared for each chunk
redshifted_spectra : tuple
Redshifted chunks
chi2 : tuple
All chi2
"""
#Get data chunks
data_wave_chunks, data_flux_chunks, data_noise_chunks = data_chunks(data_wave, data_flux, data_noise, targ_delta)
#Get template_chunks
temp_wave_chunks, temp_flux_chunks, temp_noise_chunks, temp_central_wavelengths, central_waves = template_chunks(data_wave,
data_flux,
data_noise, template_path,
z_lit, targ_delta,
overhang, position)
#Find redshifts of each chunk
observed_chunks = []
temp_chunks = []
for i in range(len(data_wave_chunks)):
observed_chunks.append(Spectrum1D(spectral_axis=data_wave_chunks[i]*u.Angstrom,
flux=data_flux_chunks[i]*(u.erg/u.s/u.cm**2/u.Angstrom),
uncertainty=InverseVariance(data_noise_chunks[i])))
temp_chunks.append(Spectrum1D(spectral_axis=temp_wave_chunks[i]*u.Angstrom,
flux=temp_flux_chunks[i]*(u.Lsun/u.micron),
uncertainty=StdDevUncertainty(temp_noise_chunks[i])))
redshifts_chunks = np.linspace(z_test-z_bound, z_test+z_bound, 1000)
fitted_redshift_results = []
bestfit_redshift = np.zeros(len(data_wave_chunks))
best_chi2 = np.zeros(len(data_wave_chunks))
redshifted_spectra = []
chi2 = []
for i in range(len(data_wave_chunks)):
fitted_redshift_results.append(template_redshift(observed_spectrum=observed_chunks[i],
template_spectrum=temp_chunks[i],
redshift=redshifts_chunks))
bestfit_redshift[i] = fitted_redshift_results[i][0]
best_chi2[i] = fitted_redshift_results[i][1]
redshifted_spectra.append(fitted_redshift_results[i][2])
chi2.append(fitted_redshift_results[i][3])
return bestfit_redshift, best_chi2, redshifted_spectra, chi2
