def fromMarzJSON(json_spectrum):
from astropy.nddata import (
VarianceUncertainty,
StdDevUncertainty,
InverseVariance,
)
wavelength = Quantity(json_spectrum["wavelength"], u.Angstrom)
spectrum = SpectrumList()
uncertainty = None
if "variance" in json_spectrum.keys():
uncertainty = StdDevUncertainty(
Quantity(np.array(json_spectrum["variance"], dtype=np.float)))
for key in json_spectrum.keys():
if key != 'wavelength' and key != 'variance':
flux = Quantity(np.array(json_spectrum[key], dtype=np.float))
if key == 'intensity' and uncertainty:
aspectrum = Spectrum1D(flux=flux,
spectral_axis=wavelength,
uncertainty=uncertainty,
mask=np.isnan(flux),
meta={'purpose': 'reduced'})
else:
aspectrum = Spectrum1D(flux=flux,
spectral_axis=wavelength,
mask=np.isnan(flux),
meta={'purpose': key})
spectrum.append(aspectrum)
return spectrum
def load_sdss_mastar(path, hdu=1, **kwargs):
r"""
Read a list of spectrum from a path that is described by the SDSS MaNGA MaStar data model,
which actually describes a collection of spectra of different sources:
https://data.sdss.org/datamodel/files/MANGA_SPECTRO_MASTAR/DRPVER/MPROCVER/mastar-goodspec-DRPVER-MPROCVER.html
:param path:
The local path of the spectrum.
:returns:
A `specutils.Spectrum1D` object.
"""
spectra = []
units = u.Unit("1e-17 erg / (Angstrom cm2 s)")
with fits.open(path, **kwargs) as image:
_hdu = image[hdu]
for i in range(_hdu.header["NAXIS2"]):
meta = OrderedDict(zip(_hdu.data.dtype.names, _hdu.data[i]))
spectral_axis = meta.pop("WAVE") * u.Angstrom
flux = np.atleast_2d(meta.pop("FLUX") * units)
uncertainty = InverseVariance(meta.pop("IVAR").reshape(flux.shape))
spectra.append(Spectrum1D(spectral_axis=spectral_axis,
flux=flux, uncertainty=uncertainty, meta=meta))
return SpectrumList(spectra)
def smooth_spectra(spectra, *, filter="box", **kwargs):
"""
Smooth spectra via a given filter.
"""
if filter == "box" and not kwargs:
kwargs["width"] = DEFAULT_BOX_FILTER_WIDTH
if filter in NAMED_FILTERS:
filter = NAMED_FILTERS[filter]
if isinstance(spectra, Spectrum1D):
return convolution_smooth(spectra, filter(**kwargs))
if isinstance(spectra, SpectrumList):
return SpectrumList([
convolution_smooth(spectrum, filter(**kwargs))
for spectrum in spectra
])
if isinstance(spectra, SpectrumCollection):
return SpectrumCollection([
convolution_smooth(spectrum, filter(**kwargs))
for spectrum in spectra
])
try:
spectra_iter = iter(spectra)
except TypeError:
raise TypeError(
"Must be a specutils spectrum object or an iterable object")
return [
convolution_smooth(spectrum, filter(**kwargs))
for spectrum in spectra_iter
]
def setup_class(self, specviz_helper, spectrum1d):
self.spec_app = specviz_helper
self.spec = spectrum1d
self.spec_list = SpectrumList([spectrum1d] * 3)
self.label = "Test 1D Spectrum"
self.spec_app.load_spectrum(spectrum1d, data_label=self.label)
def sixdfgs_combined_fits_loader(file_obj, **kwargs):
"""
Load the combined variant of a 6dF Galaxy Survey (6dFGS) file.
6dFGS used the Six-degree Field instrument on the UK Schmidt Telescope
(UKST) at the Siding Spring Observatory (SSO) near Coonabarabran,
Australia. Further details can be found at http://www.6dfgs.net/, or
https://docs.datacentral.org.au/6dfgs/. Catalogues and spectra were
produced, with the spectra being provided as both fits tables and as fits
images. This loads the combined variant of the spectra.
Parameters
----------
file_obj: str, file-like or HDUList
FITS file name, object (provided from name by Astropy I/O Registry),
or HDUList (as resulting from astropy.io.fits.open()).
Returns
-------
data: SpectrumList
The 6dF spectra that are represented by the data in this file.
"""
if isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist = file_obj
else:
hdulist = fits.open(file_obj, **kwargs)
specs = SpectrumList([_load_single_6dfgs_hdu(hdu) for hdu in hdulist[5:]])
if not isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist.close()
return specs
def test_asdf_spectrumlist(tmpdir):
spectra = SpectrumList([
create_spectrum1d(5100, 5300),
create_spectrum1d(5000, 5500),
create_spectrum1d(0, 100),
create_spectrum1d(1, 5)
])
tree = dict(spectra=spectra)
assert_roundtrip_tree(tree, tmpdir)
def twoslaq_lrg_fits_loader(file_obj, **kwargs):
"""
Load a file from the LRG subset of the 2dF-SDSS LRG/QSO survey (2SLAQ-LRG)
file.
2SLAQ was one of a number of surveys that used the 2dF instrument on the
Anglo-Australian Telescope (AAT) at Siding Spring Observatory (SSO) near
Coonabarabran, Australia, and followed up the 2QZ survey. Further details
can be seen at http://www.physics.usyd.edu.au/2slaq/ or at
https://docs.datacentral.org.au/.
The LRG and QSO data appear to be in different formats, this only loads the
LRG version. As there is a science and sky spectrum, the `SpectrumList`
loader provides both, whereas the `Spectrum1D` loader only provides the
science.
Parameters
----------
file_name: str
The path to the FITS file
Returns
-------
data: SpectrumList
The 2SLAQ-LRG spectrum that is represented by the data in this file.
"""
if isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist = file_obj
else:
hdulist = fits.open(file_obj, **kwargs)
header = hdulist[0].header
spectrum = hdulist[0].data[0, 0] * Unit("count/s")
sky = hdulist[0].data[1, 0] * Unit("count/s")
# Due to the odd structure of the file, the WCS needs to be read in
# manually
wcs = WCS(naxis=1)
wcs.wcs.cdelt[0] = header["CD1_1"]
wcs.wcs.crval[0] = header["CRVAL1"]
wcs.wcs.crpix[0] = header["CRPIX1"]
wcs.wcs.cunit[0] = Unit("Angstrom")
meta = {"header": header}
if not isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist.close()
return SpectrumList([
Spectrum1D(flux=spectrum, wcs=wcs, meta=meta),
Spectrum1D(flux=sky, wcs=wcs, meta=meta),
])
def twodfgrs_fits_loader(file_obj, **kwargs):
"""
Load a file from the 2dF Galaxy Redshift Survey.
The 2dF Galaxy Redshift Survey (2dFGRS) was a major spectroscopic survey
taking full advantage of the unique capabilities of the 2dF facility built
by the Anglo-Australian Observatory. The 2dFGRS is integrated with the 2dF
QSO survey. Further details can be seen at http://www.2dfgrs.net/ or at
https://docs.datacentral.org.au/.
Parameters
----------
file_obj: str or HDUList
The path to the FITS file, or the loaded file as an HDUList
Returns
-------
data: SpectrumList
The 2dFGRS spectra represented by the data in this file.
"""
if isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist = file_obj
else:
hdulist = fits.open(file_obj, **kwargs)
spectra = []
primary_header = hdulist[0].header
for hdu in hdulist:
if hdu.header.get("IMAGE", "").strip() == "SKYCHART":
continue
spectra.append(load_spectrum_from_extension(hdu, primary_header))
if not isinstance(file_obj, fits.hdu.hdulist.HDUList):
hdulist.close()
return SpectrumList(spectra)
def load_multiline_single_file(
filename,
*,
hdu,
wcs,
units,
all_standard_units,
all_keywords,
valid_wcs,
label=True,
):
spectra_map = {
"sky": [],
"combined": [],
"unreduced": [],
"reduced": [],
}
with fits.open(filename) as fits_file:
fits_header = fits_file[0].header
fits_data = fits_file[0].data
hdu = deepcopy(hdu)
if hdu is not None:
# extract hdu information and validate it
if hdu.pop("require_transpose", False):
fits_data = fits_data.T
purpose_prefix = hdu.pop("purpose_prefix", None)
num_rows = fits_data.shape[0]
if len(hdu) != 0:
if len(hdu) < num_rows:
print("len hdu "+str(len(hdu)))
print("len nrows "+str(num_rows))
print(hdu)
print(fits_header)
raise ValueError("Not all rows have been specified")
if len(hdu) > num_rows:
raise ValueError("Too many rows have been specified")
else:
purpose_prefix = None
for i, row in enumerate(fits_data, start=1):
if hdu is not None:
row_info = hdu.get(str(i))
else:
row_info = None
add_single_spectra_to_map(
spectra_map,
header=fits_header,
data=row,
index=i,
spec_info=row_info,
wcs_info=wcs,
units_info=units,
purpose_prefix=purpose_prefix,
all_standard_units=all_standard_units,
all_keywords=all_keywords,
valid_wcs=valid_wcs,
)
if label:
add_labels(spectra_map["combined"])
add_labels(spectra_map["reduced"])
add_labels(spectra_map["unreduced"], use_purpose=True)
add_labels(spectra_map["sky"], use_purpose=True)
return SpectrumList(
spectra_map["combined"] +
spectra_map["reduced"] +
spectra_map["unreduced"] +
spectra_map["sky"]
)
def load_single_split_file(
filename,
*,
hdus,
wcs,
units,
all_standard_units,
all_keywords,
valid_wcs,
label=True,
):
spectra_map = {
"sky": [],
"combined": [],
"unreduced": [],
"reduced": [],
}
with fits.open(filename) as fits_file:
hdus = deepcopy(hdus)
if hdus is not None:
# extract hdu information and validate it
cycle = hdus.pop("cycle", None)
cycle_start = hdus.pop("cycle_start", None)
purpose_prefix = hdus.pop("purpose_prefix", None)
if len(hdus) != 0 and cycle is None:
if len(hdus) < len(fits_file):
raise ValueError("Not all HDUs have been specified")
if len(hdus) > len(fits_file):
raise ValueError("Too many HDUs have been specified")
if cycle is not None and cycle_start is None:
if len(hdus) == 0:
raise ValueError(
"If HDUs are not specified, cycle_start must be used"
)
cycle_start = len(hdus)
if cycle is not None:
cycle_purpose_prefix = cycle.pop("purpose_prefix", None)
cycle_length = len(cycle)
# validate cycle
if (len(fits_file) - cycle_start) % cycle_length != 0:
print("len="+str(len(fits_file)))
print("cycle_start="+str(cycle_start))
print("cycle_length="+str(cycle_length))
raise ValueError(
"Full cycle cannot be read from fits file"
)
else:
cycle = None
cycle_start = 0
purpose_prefix = None
cycle_purpose_prefix = None
cycle_length = 0
for i, fits_hdu in enumerate(fits_file):
if cycle is not None and i >= cycle_start:
hdu_info = cycle.get(str((i - cycle_start) % cycle_length))
hdu_purpose_prefix = cycle_purpose_prefix
elif hdus is not None:
hdu_info = hdus.get(str(i))
hdu_purpose_prefix = purpose_prefix
else:
hdu_info = None
hdu_purpose_prefix = None
header_info = fits_hdu.header
# Begin ADACS snippet
# This allows wcs keywords to be used if they only appear in the primary header
# but will overwrite them with the current header if they exist
# It may require testing to make sure there are no undesired effects
if i == 0:
primary_wcs_data = fits.Header()
if not valid_wcs:
desired_wcs_keys = set(list(fits_hdu.header.keys())) & set(list(wcs.values()))
for key in desired_wcs_keys:
primary_wcs_data.set(key, fits_hdu.header.get(key))
header_info = primary_wcs_data + fits_hdu.header
# End ADACS snippet
add_single_spectra_to_map(
spectra_map,
data=fits_hdu.data,
header=header_info,
spec_info=hdu_info,
wcs_info=wcs,
units_info=units,
purpose_prefix=hdu_purpose_prefix,
all_standard_units=all_standard_units,
all_keywords=all_keywords,
valid_wcs=valid_wcs,
)
if label:
add_labels(spectra_map["combined"])
add_labels(spectra_map["reduced"])
add_labels(spectra_map["unreduced"], use_purpose=True)
add_labels(spectra_map["sky"], use_purpose=True)
return SpectrumList(
spectra_map["combined"] +
spectra_map["reduced"] +
spectra_map["unreduced"] +
spectra_map["sky"]
)
def load_aaomega_file(filename, *args, **kwargs):
with read_fileobj_or_hdulist(filename, *args, **kwargs) as fits_file:
fits_header = fits_file[AAOMEGA_SCIENCE_INDEX].header
# fits_file is the hdulist
var_idx = None
rwss_idx = None
for idx, extn in enumerate(fits_file):
if extn.name == "VARIANCE":
var_idx = idx
if extn.name == "RWSS":
rwss_idx = idx
# science data
fits_data = fits_file[AAOMEGA_SCIENCE_INDEX].data
# read in Fibre table data....
ftable = Table(fits_file[AAOMEGA_FIBRE_INDEX].data)
# A SpectrumList to hold all the Spectrum1D objects
sl = SpectrumList()
# the row var contains the pixel data from the science frame
for i, row in enumerate(fits_data):
# Definitely need deepcopy here, otherwise it does *NOT* work!
fib_header = deepcopy(fits_header)
# Adjusting some values from primary header so individual fibre
# spectra have meaningful headers
fib_header["FLDNAME"] = (fits_header["OBJECT"],
"Name of 2dF .fld file")
fib_header["FLDRA"] = (
fits_header["MEANRA"],
"Right Ascension of 2dF field",
)
fib_header["FLDDEC"] = (fits_header["MEANDEC"],
"Declination of 2dF field")
# Now for the fibre specific information from the Fibre Table
# (extension 2)
# Caution: RA and DEC are stored in RADIANS in the FIBRE TABLE!
fib_header["RA"] = (
ftable["RA"][i] * 180.0 / np.pi,
"Right Ascension of fibre from configure .fld file",
)
fib_header["DEC"] = (
ftable["DEC"][i] * 180.0 / np.pi,
"Declination of fibre from configure .fld file",
)
fib_header["OBJECT"] = (
ftable["NAME"][i],
"Name of target observed by fibre",
)
fib_header["OBJCOM"] = (
ftable["COMMENT"][i],
"Comment from configure .fld file for target",
)
fib_header["OBJMAG"] = (
ftable["MAGNITUDE"][i],
"Magnitude of target observed by fibre",
)
fib_header["OBJTYPE"] = (
ftable["TYPE"][i],
"Type of target observed by fibre",
)
fib_header["OBJPIV"] = (
ftable["PIVOT"][i],
"Pivot number used to observe target",
)
fib_header["OBJPID"] = (
ftable["PID"][i],
"Program ID from configure .fld file",
)
fib_header["OBJX"] = (
ftable["X"][i],
"X coord of target observed by fibre (microns)",
)
fib_header["OBJY"] = (
ftable["Y"][i],
"Y coord of target observed by fibre (microns)",
)
fib_header["OBJXERR"] = (
ftable["XERR"][i],
"X coord error of target observed by fibre (microns)",
)
fib_header["OBJYERR"] = (
ftable["YERR"][i],
"Y coord error of target observed by fibre (microns)",
)
fib_header["OBJTHETA"] = (
ftable["THETA"][i],
"Angle of fibre used to observe target",
)
fib_header["OBJRETR"] = (
ftable["RETRACTOR"][i],
"Retractor number used to observe target",
)
# WLEN added around 2005 according to AAOmega obs manual...
# so not always available
if "WLEN" in ftable.colnames:
fib_header["OBJWLEN"] = (
ftable["WLEN"][i],
"Retractor of target observed by fibre",
)
# ftable['TYPE'][i]:
# P == program (science)
# S == sky
# U == unallocated or unused
# F == fiducial (guide) fibre
# N == broken, dead or no fibre
meta = {"header": fib_header}
if ftable["TYPE"][i] == "P":
meta["purpose"] = "reduced"
elif ftable["TYPE"][i] == "S":
meta["purpose"] = "sky"
else:
# Don't include other fibres that are not science or sky
continue
wcs = compute_wcs_from_keys_and_values(fib_header,
**AAOMEGA_2DF_WCS_SETTINGS)
flux = row * AAOMEGA_2DF_FLUX_UNIT
meta["fibre_index"] = i
# Our science spectrum
spectrum = Spectrum1D(wcs=wcs, flux=flux, meta=meta)
# If the VARIANCE spectrum exists, add it as an additional spectrum
# in the meta dict with key 'variance'
if var_idx is not None:
var_data = fits_file[var_idx].data
var_flux = var_data[i] * AAOMEGA_2DF_FLUX_UNIT**2
spectrum.uncertainty = VarianceUncertainty(var_flux)
# If the RWSS spectrum exists, add it as an additional spectrum in
# the meta dict with key 'science_sky'
# This is an optional extension produced by 2dfdr on request: all
# spectra without the average/median sky subtraction
# Useful in case users want to do their own sky subtraction.
if rwss_idx is not None:
rwss_data = fits_file[rwss_idx].data
rwss_flux = rwss_data[i] * AAOMEGA_2DF_FLUX_UNIT
rwss_meta = {"header": fib_header, "purpose": "science_sky"}
spectrum.meta["science_sky"] = Spectrum1D(wcs=wcs,
flux=rwss_flux,
meta=rwss_meta)
# Add our spectrum to the list.
# The additional spectra are accessed using
# spectrum.meta['variance'] and spectrum.meta['science_sky']
sl.append(spectrum)
add_labels(sl)
return sl
source_dir = pathlib.Path.home() / read_setting(spec_set_settings,
"source_dir",
pathlib.Path().cwd())
output_dir = pathlib.Path.home() / read_setting(spec_set_settings,
"output_dir",
pathlib.Path().cwd())
output_dir.mkdir(parents=True, exist_ok=True)
for fits_group in spec_set_settings["subtraction_settings"]:
print(
f"\n\nProcessing fits group: {fits_group}\n--------------------------------------"
)
fits_group_settings = spec_set_settings["subtraction_settings"][
fits_group]
fits_group_ss_spectra = SpectrumList()
# We'll note these as we go through the group as these will form the basis of the output fits.
source_fits_full_name1 = None
spec_ss_header = None
for fits_file_name in spec_set_settings["subtraction_settings"][
fits_group]:
print(
f"* Processing {spec_set} / subtraction_settings / {fits_group} / {fits_file_name}"
)
fits_settings = fits_group_settings[fits_file_name]
source_fits_full_name = source_dir / fits_file_name
non_ss_spectra, hdr = FrodoSpecDS.read_spectra(
source_fits_full_name, "RSS_NONSS", None, True, fits_file_name)