def extract_spectrum(cube,pixels,wvslice=None):
from linetools.spectra.xspectrum1d import XSpectrum1D
if isinstance(cube,str):
cube = DataCube(cube)
if wvslice is not None:
cube = kt.slice_cube(cube,wvslice[0],wvslice[1])
dat = cube.data
# Get rid of NaNs
dc = dat.copy()
nans = np.where(np.isnan(dc))
dc[nans]=0
# Perform the extraction
spaxels = []
for i,px in enumerate(pixels):
thisspec = dc[:,px[0],px[1]]
spaxels.append(thisspec)
spaxels=np.array(spaxels)
medspec = np.median(spaxels, axis=0)
# Get the wavelength array
newwavearr = cube.wavelength
# Create the spectrum
try:
spec = XSpectrum1D(wave = newwavearr,flux=medspec)
except:
import pdb; pdb.set_trace()
return spec
def parse_DESI_brick(hdulist, select=0, **kwargs):
""" Read a spectrum from a DESI brick format HDU list
Parameters
----------
hdulist : FITS HDU list
select : int, optional
Spectrum selected. Default is 0
Returns
-------
xspec1d : XSpectrum1D
Parsed spectrum
"""
fx = hdulist[0].data
# Sig
if hdulist[1].name in ['ERROR', 'SIG']:
sig = hdulist[1].data
else:
ivar = hdulist[1].data
sig = np.zeros_like(ivar)
gdi = ivar > 0.
sig[gdi] = np.sqrt(1. / ivar[gdi])
# Wave
wave = hdulist[2].data
wave = give_wv_units(wave)
if wave.shape != fx.shape:
wave = np.tile(wave, (fx.shape[0], 1))
# Finish
xspec1d = XSpectrum1D(wave, fx, sig, select=select, **kwargs)
return xspec1d
def grab_specmeta(self,
rows,
verbose=None,
masking='edges',
use_XSpec=True,
**kwargs):
""" Grab the spectra and meta data for an input set of rows
Aligned to the rows input
Parameters
----------
rows : int or ndarray
verbose
kwargs
Returns
-------
spec : XSpectrum1D or ndarray
Spectra requested, ordered by the input rows
meta : Table -- THIS MAY BE DEPRECATED
Meta table, ordered by the input rows
"""
if isinstance(rows, (int, np.int64)):
rows = np.array([rows]) # Insures meta and other arrays are proper
if verbose is None:
verbose = self.verbose
# Check spectra even exist! (can be only meta data)
if 'spec' not in list(self.hdf[self.group].keys()):
warnings.warn("No spectra in group: {:s}".format(self.group))
return None, None
# Check memory
if self.stage_data(rows, **kwargs):
if verbose:
print("Loaded spectra")
# Load
msk = np.array([False] * len(self.meta))
msk[rows] = True
tmp_data = self.hdf[self.group]['spec'][msk]
# Replicate and sort according to input rows
idx = match_ids(rows, np.where(msk)[0])
data = tmp_data[idx]
else:
print("Staging failed.. Not returning spectra")
return
# Generate XSpectrum1D
if 'co' in data.dtype.names:
co = data['co']
else:
co = None
if use_XSpec:
spec = XSpectrum1D(data['wave'],
data['flux'],
sig=data['sig'],
co=co,
masking=masking)
else:
spec = data
# Return
return spec, self.meta[rows]
def loop_grab_spec(self, survey, IDs, verbose=None, **kwargs):
""" Grab spectra using staged IDs
All IDs must occur in each of the surveys listed
Order of spectra and meta tables will match the input IDs
Parameters
----------
survey : str or list
IDs : int or intarr
Returns
-------
spec : XSpectrum1D
Spectra requested, ordered by the IDs
meta : Table
Meta table, ordered by the IDs
"""
if verbose is None:
verbose = self.verbose
if isinstance(survey, list):
all_spec = []
all_meta = []
for isurvey in survey:
spec, meta = self.grab_spec(isurvey, IDs, **kwargs)
if spec is not None:
all_spec.append(spec.copy())
all_meta.append(meta.copy())
return all_spec, all_meta
# Grab IDs
if self.stage_data(survey, IDs, **kwargs):
if np.sum(self.survey_bool) == 0:
if verbose:
print("No spectra matching in survey {:s}".format(survey))
return None, None
else:
if verbose:
print("Loaded spectra")
tmp_data = self.hdf[survey]['spec'][self.survey_bool]
# Replicate and sort according to input IDs
data = tmp_data[self.indices]
else:
print("Staging failed.. Not returning spectra")
return
# Generate XSpectrum1D
if 'co' in data.dtype.names:
co = data['co']
else:
co = None
spec = XSpectrum1D(data['wave'],
data['flux'],
sig=data['sig'],
co=co,
masking='edges')
# Return
return spec, self.meta
def parse_hdf5(inp, close=True, **kwargs):
""" Read a spectrum from HDF5 written in XSpectrum1D format
Expects: meta, data, units
Parameters
----------
inp : str or hdf5
Returns
-------
"""
import json
import h5py
# Path
path = kwargs.pop('path', '/')
# Open
if isinstance(inp, basestring):
hdf5 = h5py.File(inp, 'r')
else:
hdf5 = inp
# Data
data = hdf5[path+'data'].value
# Meta
if 'meta' in hdf5[path].keys():
meta = json.loads(hdf5[path+'meta'].value)
# Headers
for jj,heads in enumerate(meta['headers']):
try:
meta['headers'][jj] = fits.Header.fromstring(meta['headers'][jj])
except TypeError: # dict
if not isinstance(meta['headers'][jj], dict):
raise IOError("Bad meta type")
else:
meta = None
# Units
units = json.loads(hdf5[path+'units'].value)
for key,item in units.items():
if item == 'dimensionless_unit':
units[key] = u.dimensionless_unscaled
else:
units[key] = getattr(u, item)
# Other arrays
try:
sig = data['sig']
except (NameError, IndexError):
sig = None
try:
co = data['co']
except (NameError, IndexError):
co = None
# Finish
if close:
hdf5.close()
return XSpectrum1D(data['wave'], data['flux'], sig=sig, co=co,
meta=meta, units=units, **kwargs)
def test_errors():
# from_tuple
try:
spec = XSpectrum1D.from_tuple('this_is_not_a_tuple')
except IOError:
pass
try:
n_tuple = np.array([np.ones(5), np.ones(5)]), np.ones(5)
spec = XSpectrum1D.from_tuple(n_tuple)
except IOError:
pass
# wrong instances
flux = [1, 2, 3]
wave = [1, 2, 3]
try:
spec = XSpectrum1D(wave, flux)
except IOError:
pass
#wrong shapes
flux = np.ones(5)
wave = np.array([1, 2, 3])
try:
spec = XSpectrum1D(wave, flux)
except IOError:
pass
try:
spec = XSpectrum1D(wave, np.ones(len(wave)), sig=np.ones(2))
except IOError:
pass
try:
spec = XSpectrum1D(wave,
np.ones(len(wave)),
co=np.ones(2),
verbose=True) # test verbose here too
except IOError:
pass
# wrong masking
try:
spec = XSpectrum1D(wave, np.ones(len(wave)), masking='wrong_masking')
except IOError:
pass
#wrong units input
try:
spec = XSpectrum1D(wave, np.ones(len(wave)), units='not_a_dict')
except IOError:
pass
try:
spec = XSpectrum1D(wave, np.ones(len(wave)), units=dict(wrong_key=2))
except IOError:
pass
def smash_spectra(spec, method='average', debug=False):
""" Collapse the data in XSpectrum1D.data
One might call this 'stacking'
Note: This works on the unmasked data array and returns
unmasked spectra
Parameters
----------
spec : XSpectrum1D
method : str, optional
Approach to the smash ['smash']
debug : bool, optional
Returns
-------
new_spec : XSpectrum1D
"""
from linetools.spectra.xspectrum1d import XSpectrum1D
# Checks
if spec.nspec <= 1:
raise IOError(
"This method smashes an XSpectrum1D instance with multiple spectra"
)
np.testing.assert_allclose(spec.data['wave'][0], spec.data['wave'][1])
# Generate mask
stack_msk = spec.data['sig'] > 0.
if method == 'average':
tot_flx = np.sum(spec.data['flux'] * stack_msk, 0)
navg = np.sum(stack_msk, 0)
fin_flx = tot_flx / np.maximum(navg, 1.)
elif method == 'median':
fin_flx = np.median(spec.data['flux'], 0)
else:
raise IOError("Not prepared for this smash method")
# Finish
new_spec = XSpectrum1D(spec.data['wave'][0],
fin_flx,
masking='none',
units=spec.units.copy())
new_spec.meta = spec.meta.copy()
# Return
return new_spec
def clean(spec, red):
"""
:param spec: (numpy array) XSpectrum1D objects
:param red: (numpy array) their redshift values (z)
:returns:
rest_spec: (numpy array) truncatted, normalized and restframed XSpectrum1D objects
"""
import numpy as np
import astropy.units as u
from linetools.spectra import utils as ltsu
from linetools.spectra.xspectrum1d import XSpectrum1D
r = range(len(spec))
temp = [np.asarray(spec[i].wavelength / (1 + red[i])) for i in r]
# truncating each spectra to only include wavelength values from 1000 to 1450
wv_coverage = [(1000 < entry) & (entry < 1450) for entry in temp]
wave = np.asarray([spec[i].wavelength[wv_coverage[i]] for i in r])
flux = np.asarray([spec[i].flux[wv_coverage[i]] for i in r])
error = np.asarray([spec[i].sig[wv_coverage[i]] for i in r])
temp = [np.asarray(wave[i] / (1 + red[i])) for i in r]
# normalizing just between the SiII lines
wv_norm = [(1260 < entry) & (entry < 1304)
for entry in temp] # just between the SiII lines
flux_range = np.asarray([flux[i][wv_norm[i]] for i in r])
medians = np.asarray([np.median(flux_range[i]) for i in r])
norm_flux = np.asarray([(flux[i] / medians[i]) for i in r])
# getting a single XSpec object to feed into the stack
spec = [XSpectrum1D(wave[i], norm_flux[i], error[i]) for i in r]
collate = ltsu.collate(spec)
rest_spec = ltsu.rebin_to_rest(collate,
red,
300 * u.km / u.s,
grow_bad_sig=True)
return rest_spec
def convert_wavelength(cube, outfile=None):
from linetools.spectra.xspectrum1d import XSpectrum1D
dims = np.shape(cube.data[0])
lastwaveidx = np.max(dims) - 1
wave1 = cube.wcs.wcs_pix2world([[0, 0, 0]], 1)[0][2]
wave2 = cube.wcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2]
import pdb
pdb.set_trace()
if wave1 > 1000.:
inc = 1.
else:
inc = 1e-10
newwavearr = np.arange(wave1, wave2 + inc, inc)
newspec = XSpectrum1D(newwavearr * inc, np.zeros_like(newwavearr))
newspec.meta['airvac'] = 'air'
newspec.airtovac()
diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1]
# TODO: Come up with a way to keep the header info and WCS synced
cube.wcs.wcs.crval[2] = newspec.wvmin.value
cube.wcs.wcs.crpix[2] = 1.
cube.wcs.wcs.pc[2][2] = np.median(diffs.value)
newhdulist = cube.wcs.to_fits()
newhdulist[0].data = cube.data
newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths')
newhdulist[0].header['CUNIT3'] = 'Angstrom'
newhdulist[0].header['CD3_3'] = np.median(diffs.value)
# cube.header['CRPIX3'] = 1
# cube.header['PC3_3'] = np.median(diffs.value)
# cube.header['CRVAL3'] = newspec.wvmin.value
cube.header = newhdulist[0].header
#import pdb; pdb.set_trace()
if outfile is not None:
newhdulist.writeto(outfile, overwrite=True)
def median_coadd(cubes, stackwcs, outfile=None, air_to_vac=True):
nc_arr = np.array(cubes)
nc_med = np.median(nc_arr, axis=0)
newhdulist = stackwcs.to_fits()
newhdulist[0].data = nc_med
if air_to_vac is True:
dims = np.shape(cubes[0])
lastwaveidx = np.max(dims) - 1
wave1 = stackwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2]
wave2 = stackwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2]
newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10)
newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr))
newspec.meta['airvac'] = 'air'
newspec.airtovac()
diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1]
newhdulist[0].header['CRVAL3'] = newspec.wvmin.value
newhdulist[0].header['CRPIX3'] = 1
newhdulist[0].header['CD3_3'] = np.median(diffs.value)
newhdulist[0].header['PC3_3'] = np.median(diffs.value)
newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths')
newhdulist[0].header['CUNIT3'] = 'Angstrom'
if outfile is not None:
newhdulist.writeto(outfile, overwrite=True)
return newhdulist
def rtModel_conv_rb_spec(input,
outfil=None,
specR=1800.,
zgal=0.6942,
restlam=2796.35,
dlam_rb=None,
fmt=['lam', 'y', 'x'],
return_cube=True):
'''Convolve and rebin a radiative transfer model cube in the spectral direction
Parameters
----------
input : str or DataCube
Input filename or DataCube to convolve/rebin spectrally
outfil : str, optional
Output filename
specR : float, optional
Resolving power of spectrum
zgal : float, optional
Redshift of object
restlam : float, optional
Reference rest-frame wavelength for calculating kernel sigma
dlam_rb : float, optional
New wavelength bin size; if None, do not rebin
fmt : list, optional
Axis order of data cube, must have 'x', 'y', and 'lam' in some order
return_cube : bool, optional
If True, return DataCube object
Returns
-------
newwave : array
Wavelength vector (same as that of input cube if dlam_rb is None)
specconv_cube : 3D array
Datacube with convolved spaxels (spatial, spatial, spectral)
'''
from linetools.spectra.xspectrum1d import XSpectrum1D
dlam_obs = restlam * (1.0 + zgal) / specR
dlam_rest = dlam_obs / (1.0 + zgal
) # FWHM of resolution element in Angstroms
if isinstance(input, str):
### Assume data format is as the output of
### KR's read_spec_output.read_fullfits_conv_rebin()
hdu = fits.open(input)
data_pad = hdu[0].data # Original data, just padded
cubedat = hdu[1].data # Spatially convolved data
wave = hdu[2].data # Wavelength array
nx, ny, nlam = cubedat.shape
cubewcs = None
else:
wave = input.wavelength / (1. + zgal)
cubedat = input.data
cubewcs = input.wcs
### Establish dimensions of input cube
xs = fmt.index('x')
ys = fmt.index('y')
sp = fmt.index('lam')
cdims = cubedat.shape
nlam = cdims[sp]
nx = cdims[xs]
ny = cdims[ys]
### Rearrange the cube (temporarily) : (y,x,lam)
cubedat_tp = np.transpose(cubedat, axes=[ys, xs, sp])
fmt_internal = ['y', 'x', 'lam']
### Set up kernel to convolve spaxels
dwv = np.abs(wave[1] - wave[0])
fwhm_specpix = dlam_rest / dwv # FWHM in number of pixels
stddev_specpix = fwhm_specpix / (2.0 * (2.0 * np.log(2.0))**0.5)
spec_kernel = Gaussian1DKernel(stddev_specpix)
### Rebin, if desired, and convolve
if dlam_rb is None:
specconv_cube = np.zeros((ny, nx, nlam))
newwave = wave
for spaxx in range(nx):
for spaxy in range(ny):
spax_spec = cubedat_tp[spaxy, spaxx, :]
conv_spec = convolve(spax_spec,
spec_kernel,
normalize_kernel=True)
specconv_cube[spaxy, spaxx, :] = conv_spec
else:
waveq = wave * u.Angstrom
newwave = np.arange(int(wave[0]), int(wave[-1]), dlam_rb) * u.Angstrom
specconv_cube = np.zeros((ny, nx, len(newwave)))
for spaxx in range(nx):
for spaxy in range(ny):
spax_spec = cubedat_tp[spaxy, spaxx, :]
conv_spec = convolve(spax_spec,
spec_kernel,
normalize_kernel=True)
# Use the linetools rebinning method
csX1d = XSpectrum1D(wave=waveq, flux=conv_spec)
cs_rb = csX1d.rebin(newwave)
try:
specconv_cube[spaxy, spaxx, :] = cs_rb.flux
except:
import pdb
pdb.set_trace()
if cubewcs is not None:
cubewcs.wcs.pc[2, 2] = dlam_rb ### Probably shouldn't be hard-coded
### Transpose back to input dimensions
axs = ['a'] * 3
for cc in fmt_internal:
axs[fmt.index(cc)] = fmt_internal.index(cc)
newdat = np.transpose(specconv_cube, axes=axs)
if return_cube:
newcube = DataCube(wavelength=newwave,
data=newdat,
include_wcs=cubewcs)
return newwave, newcube
else:
return newwave, specconv_cube
def stacks(spec, red, N, zbin, plot=False):
"""
:param spec: (numpy array) XSpectrum1D objects
:param red: (numpy array) their redshift values (z)
:param N: (int) number of bootstrap iterations
:param zbin: (str) redshift bin "lowz" or "hiz"
"""
import numpy as np
from astropy.io import fits
from numpy import random as ran
from linetools.spectra import utils as ltsu
from linetools.spectra.xspectrum1d import XSpectrum1D
print("Number of spectra provided =", spec.nspec)
# the actual stack
stack = ltsu.smash_spectra(spec)
z_med = np.median(red)
# the bootstrap
R = (spec.nspec) # restraints on the random variable
N_stack = []
N_z_med = []
for i in np.arange(N): # this might take a while
if i % 10 == 0:
print(i)
choice = np.asarray(ran.randint(0, R, R)) # the shuffle
rand_spec = np.asarray([spec[index] for index in choice])
rand_red = np.asarray([red[index] for index in choice])
rand_collate = ltsu.collate(rand_spec) # collate and stack
N_stack.append(ltsu.smash_spectra(rand_collate))
N_z_med.append(np.median(rand_red))
# matrix math to create the error array
N_flux = np.array([entry.flux for entry in N_stack])
N_matrix = np.array([N_flux[i] - stack.flux for i in range(N)])
tranpose = np.transpose(N_matrix) # matrix math
covariance = np.dot(tranpose, N_matrix)
sigma = np.sqrt(np.diagonal(covariance) / (N - 1)) # the final error array
composite = XSpectrum1D(stack.wavelength, stack.flux, sig=sigma)
# plot with bootstrap generated error
if plot == True:
composite.plot()
# writing out the stack
hdr = fits.Header()
hdr["REDSHIFT"] = z_med
hdr["NSPEC"] = spec.nspec
header = fits.PrimaryHDU(header=hdr)
c_wave = fits.Column(name='WAVELENGTH', array=stack.wavelength, unit="angstroms", format="E")
c_flux = fits.Column(name='FLUX', array=stack.flux, unit="relative flux", format="E")
c_noise = fits.Column(name='FLUX_ERR', array=sigma, unit="relative flux", format="E")
table = fits.BinTableHDU.from_columns([c_wave, c_flux, c_noise])
full_hdu = fits.HDUList([header, table])
full_hdu.writeto("../../fits/composites/" + zbin + "/composite.fits")
# writing out the boostrap
hdr = fits.Header()
hdr["NSPEC"] = spec.nspec
hdr["NBOOT"] = N
header = fits.PrimaryHDU(header=hdr)
c_wave = fits.Column(name='WAVELENGTH', array=stack.wavelength, unit="angstroms", format="E")
c_noise = fits.Column(name='FLUX_ERR', array=sigma, unit="relative flux", format="E")
table = fits.BinTableHDU.from_columns([c_wave, c_noise])
flux_data = fits.ImageHDU(N_flux, name="FLUX_ITERATIONS")
full_hdu = fits.HDUList([header, table, flux_data])
full_hdu.writeto("../../fits/bootstrap/" + zbin + "/stacks.fits")
def reading_data(path, resol, zlow, zhigh):
'''
function for requesting the spectra from desi mocks
:param resol float: dleta v for rebin
:param zlow float: lower limit for request redshift
:param zhigh float: higher limit for request redshift
:return: two class object: 1.meta data and unbinned spectra 2. xspectrum object
'''
#define the variable
dataz = []
z = []
zerr = []
fluxt = []
wavetotal = []
ivar = []
ra = []
dec = []
id = []
filename = []
wvmin = []
wvmax = []
npix = []
date = []
#loop through the whole directory for requesting spectra
specf = []
mag = []
item1 = os.listdir(str(path))
for k in item1:
item = os.listdir(str(path) + str(k))
# print (item)
for j in item:
if os.listdir(str(path) + str(k) + '/' + str(j) + '/'):
dataz = fits.open(
str(path) + str(k) + '/' + str(j) + '/zbest-16-' + str(j) +
'.fits')
data = fits.open(
str(path) + str(k) + '/' + str(j) + '/spectra-16-' +
str(j) + '.fits') # k j j
zcut = (zlow < np.array(dataz[1].data['Z'])) & (np.array(
dataz[1].data['Z']) < zhigh)
wavecut = (np.array(data[2].data) < 5730)
wavecut2 = ((np.array(data[7].data) > 5730) &
(np.array(data[7].data) < 7560))
wavecut3 = (np.array(data[12].data) > 7560)
z.append(dataz[1].data['Z'][zcut])
zerr.append(dataz[1].data['ZERR'][zcut])
ra.append(data[1].data['TARGET_RA'][zcut])
dec.append(data[1].data['TARGET_DEC'][zcut])
id.append(data[1].data['TARGETID'][zcut])
filename.append(
np.tile([k, j, j], (len(dataz[1].data['Z'][zcut]), 1)))
mag.append(1.0)
date.append(data[1].data['NIGHT'][zcut])
#combining the spectra from three channels
wavetotal.append(
np.repeat([
np.concatenate([
data[2].data[wavecut], data[7].data[wavecut2],
data[12].data[wavecut3]
])
],
len(dataz[1].data['Z']),
axis=0)[zcut])
fluxt.append(
np.concatenate(
(data[3].data[:, wavecut], data[8].data[:, wavecut2],
data[13].data[:, wavecut3]),
axis=1)[zcut])
ivar.append(
np.concatenate(
(data[4].data[:, wavecut], data[9].data[:, wavecut2],
data[14].data[:, wavecut3]),
axis=1)[zcut])
sp = XSpectrum1D(np.vstack(np.array(wavetotal)),
np.vstack(np.array(fluxt)),
np.sqrt(1 / np.vstack(np.array(ivar))),
verbose=False)
#ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr
binspec = ltsu.rebin_to_rest(sp,
np.zeros(len(np.array(np.concatenate(z)))),
resol * u.km / u.s)
specf = spec(np.concatenate(ra), np.concatenate(dec), np.concatenate(id),
np.concatenate(filename), np.min(binspec.wavelength),
np.max(binspec.wavelength), len(binspec.wavelength),
np.concatenate(date), np.array(mag), np.concatenate(z),
np.concatenate(zerr))
# newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso)
return specf, binspec
def extinction(spec, red, coord):
"""
:param spec: (numpy array) XSpectrum1D objects: use clamato_read.py
:param red: (numpy array) redshift values
:param coord: (numpy array) coordinates
:return:
unred_spec: (numpy array) de-reddened spec
"""
import numpy as np
from numpy.lib.polynomial import poly1d
import astropy.units as u
from astropy.coordinates import SkyCoord
from dustmaps.bayestar import BayestarQuery
from astropy.cosmology import WMAP9 as cosmo
from linetools.spectra.xspectrum1d import XSpectrum1D
r = range(len(spec))
Mpc = cosmo.comoving_distance(red)
bayestar = BayestarQuery()
coords = [
SkyCoord(coord[i][0] * u.deg,
coord[i][1] * u.deg,
distance=Mpc[i],
frame='fk5') for i in r
]
ebv = [bayestar(i) for i in coords] #to get the ebv values for each galaxy
unred_spec = []
for i in r:
x = 10000. / np.array(spec[i].wavelength) # Convert to inverse microns
npts = x.size
a = np.zeros(npts, dtype=np.float)
b = np.zeros(npts, dtype=np.float)
r_v = 3.1
good = np.where((x >= 0.3) & (x < 1.1))
if len(good[0]) > 0:
a[good] = 0.574 * x[good]**(1.61)
b[good] = -0.527 * x[good]**(1.61)
good = np.where((x >= 1.1) & (x < 3.3))
if len(good[0]) > 0: # Use new constants from O'Donnell (1994)
y = x[good] - 1.82
c1 = np.array([
1., 0.104, -0.609, 0.701, 1.137, -1.718, -0.827, 1.647, -0.505
]) # from O'Donnell
c2 = np.array([
0., 1.952, 2.908, -3.989, -7.985, 11.102, 5.491, -10.805, 3.347
])
a[good] = poly1d(c1[::-1])(y)
b[good] = poly1d(c2[::-1])(y)
good = np.where((x >= 3.3) & (x < 8))
if len(good[0]) > 0:
y = x[good]
a[good] = 1.752 - 0.316 * y - (0.104 /
((y - 4.67)**2 + 0.341)) # + f_a
b[good] = -3.090 + 1.825 * y + (1.206 /
((y - 4.62)**2 + 0.263)) # + f_b
good = np.where((x >= 8) & (x <= 11))
if len(good[0]) > 0:
y = x[good] - 8.
c1 = np.array([-1.073, -0.628, 0.137, -0.070])
c2 = np.array([13.670, 4.257, -0.420, 0.374])
a[good] = poly1d(c1[::-1])(y)
b[good] = poly1d(c2[::-1])(y)
# Now apply extinction correction to input flux vector
a_v = r_v * ebv[i]
a_lambda = a_v * (a + b / r_v)
funred = spec[i].flux * 10.**(0.4 * a_lambda) # Derive unreddened flux
funred = np.asarray(funred)
unred_spec.append(XSpectrum1D(spec[i].wavelength, funred, spec[i].sig))
return np.asarray(unred_spec)
def rebin_to_rest(spec, zarr, dv, debug=False, **kwargs):
""" Shuffle an XSpectrum1D dataset to an array of
observed wavelengths and rebin to dv pixels.
Note: This works on the unmasked data array and returns
unmasked spectra
Parameters
----------
spec : XSpectrum1D
zarr : nd.array
Array of redshifts
dv : Quantity
Velocity width of the new pixels
**kwargs :
Passed to spec.rebin()
Returns
-------
new_spec : XSpectrum1D
Not masked
"""
from linetools.spectra.xspectrum1d import XSpectrum1D
# Error checking
if spec.nspec <= 1:
raise IOError("Use spec.rebin instead")
if spec.nspec != len(zarr):
raise IOError("Input redshift array must have same dimension as nspec")
# Generate final wave array
dlnlamb = np.log(1 + dv / const.c)
z2d = np.outer(zarr, np.ones(spec.totpix))
wvmin, wvmax = (np.min(spec.data['wave'] / (1 + z2d)) * spec.units['wave'],
np.max(spec.data['wave'] / (1 + z2d)) * spec.units['wave'])
npix = int(np.round(np.log(wvmax / wvmin) / dlnlamb)) + 1
new_wv = wvmin * np.exp(dlnlamb * np.arange(npix))
# Final image
f_flux = np.zeros((spec.nspec, npix))
f_sig = np.zeros((spec.nspec, npix))
f_wv = np.outer(np.ones(spec.nspec), new_wv.value)
# Let's loop
for ispec in range(spec.nspec):
if debug:
print("ispec={:d}".format(ispec))
# Select
spec.select = ispec
# Rebin in obs frame
tspec = spec.rebin(new_wv * (1 + zarr[ispec]),
do_sig=True,
masking='none',
**kwargs)
# Save in rest-frame (worry about flambda)
f_flux[ispec, :] = tspec.flux.value
f_sig[ispec, :] = tspec.sig.value
# Finish
new_spec = XSpectrum1D(f_wv,
f_flux,
sig=f_sig,
masking='none',
units=spec.units.copy())
new_spec.meta = spec.meta.copy()
# Return
return new_spec
def collate(spectra, **kwargs):
""" Generate a single XSpectrum1D instance containing an array of
spectra from a list of individual XSpectrum1D spectra.
Each spectrum is padded with extra pixels so that the
wavelength ranges of all spectra are covered.
Padded pixels are masked.
Also note that masked pixels in the original data are ignored!
Parameters
----------
spectra : list
of XSpectrum1D
**kwargs : optional
Passed to the XSpectrum1D object generated
Returns
-------
new_spec : XSpectrum1D
"""
from linetools.spectra.xspectrum1d import XSpectrum1D
# Init
maxpix = 0
flg_co, flg_sig = False, False
nspec = 0
units = None
# Setup
for spec in spectra:
nspec += spec.nspec
for ii in range(spec.nspec):
spec.select = ii
maxpix = max(maxpix, spec.npix)
if spec.co_is_set:
flg_co = True
if spec.sig_is_set:
flg_sig = True
# Check for identical units
if units is None:
units = spec.units
else:
assert spec.units == units
# Generate data arrays
wave = np.zeros((nspec, maxpix), dtype='float64')
flux = np.zeros_like(wave, dtype='float32')
if flg_sig:
sig = np.zeros_like(wave, dtype='float32')
else:
sig = None
if flg_co:
co = np.zeros_like(flux)
else:
co = None
# Load
meta = dict(headers=[])
idx = 0
for xspec in spectra:
# Allow for multiple spectra in the XSpectrum1D object
for jj in range(xspec.nspec):
xspec.select = jj
wave[idx, :xspec.npix] = xspec.wavelength.value
flux[idx, :xspec.npix] = xspec.flux.value
if flg_sig:
try: # This will fail for spectra with fully masked arrays
sig[idx, :xspec.npix] = xspec.sig.value
except AttributeError:
sig[idx, :] = 0.
if flg_co:
if xspec.co_is_set: # Allow for a mix of continua (mainly for specdb)
co[idx, :xspec.npix] = xspec.co.value
idx += 1
# Meta
meta['headers'] += xspec.meta['headers']
# Finish
new_spec = XSpectrum1D(wave,
flux,
sig=sig,
co=co,
units=units.copy(),
meta=meta,
**kwargs)
# Return
return new_spec
def weight_mean_coadd(cubes,
vars,
stackwcs,
varwcs,
outfile=None,
outvarfile=None,
air_to_vac=True):
# Go through each cube and find the brightest pixels
buff = 200
levels = np.zeros(len(cubes))
for i, cc in enumerate(cubes):
var = vars[i]
partdat = cc[buff:-buff, :, :]
sumdat = np.sum(partdat, axis=0)
partvar = var[buff:-buff, :, :]
sumvar = np.sum(partvar, axis=0)
datnonan = sumdat[(~np.isnan(sumdat) & (sumdat != 0))]
varnonan = sumvar[(~np.isnan(sumdat) & (sumdat != 0))]
mednonan = np.median(datnonan)
stddev = np.std(datnonan)
#bright = np.where((datnonan > (mednonan+stddev))&(datnonan>0))[0]
bright = np.where((datnonan > (mednonan)) & (datnonan > 0))[0]
sumbright = np.sum(datnonan[bright])
varbright = np.sum(varnonan[bright])
cc[np.isnan(cc)] = 0.
var[np.isnan(var)] = 0.
sn = sumbright / np.sqrt(varbright)
#import pdb; pdb.set_trace()
print(sumbright, mednonan, stddev, sn)
if i == 0:
combined = cc * sn
combvar = var * sn**2
else:
combined += cc * sn
combvar += var * sn**2
levels[i] = sn
finaldat = combined / np.sum(levels)
finalvar = combvar / np.sum(levels)**2
newhdulist = stackwcs.to_fits()
newhdulist[0].data = finaldat
if air_to_vac is True:
dims = np.shape(cubes[0])
lastwaveidx = np.max(dims) - 1
wave1 = stackwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2]
wave2 = stackwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2]
newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10)
newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr))
newspec.meta['airvac'] = 'air'
newspec.airtovac()
diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1]
newhdulist[0].header['CRVAL3'] = newspec.wvmin.value
newhdulist[0].header['CRPIX3'] = 1
newhdulist[0].header['CD3_3'] = np.median(diffs.value)
newhdulist[0].header['PC3_3'] = np.median(diffs.value)
newhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths')
newhdulist[0].header['CUNIT3'] = 'Angstrom'
if outfile is not None:
newhdulist.writeto(outfile, overwrite=True)
varhdulist = varwcs.to_fits()
varhdulist[0].data = finalvar
if air_to_vac is True:
dims = np.shape(cubes[0])
lastwaveidx = np.max(dims) - 1
wave1 = varwcs.wcs_pix2world([[0, 0, 0]], 1)[0][2]
wave2 = varwcs.wcs_pix2world([[0, 0, lastwaveidx]], 1)[0][2]
newwavearr = np.arange(wave1, wave2 + 1e-10, 1e-10)
newspec = XSpectrum1D(newwavearr * 1e10, np.zeros_like(newwavearr))
newspec.meta['airvac'] = 'air'
newspec.airtovac()
diffs = newspec.wavelength[1:] - newspec.wavelength[0:-1]
varhdulist[0].header['CRVAL3'] = newspec.wvmin.value
newhdulist[0].header['CRPIX3'] = 1
varhdulist[0].header['CD3_3'] = np.median(diffs.value)
varhdulist[0].header['PC3_3'] = np.median(diffs.value)
varhdulist[0].header.set('CTYPE3', 'VAC', 'Vacuum wavelengths')
varhdulist[0].header['CUNIT3'] = 'Angstrom'
if outvarfile is not None:
varhdulist.writeto(outvarfile, overwrite=True)
return newhdulist
def extract_spectrum(cube,pixels,wvslice=None,method='median',varcube=None, spatial_scale_xy=None):
## KHRR: if spatial_scale_xy is not None, flux in units of cgs / sq. arcsec are assumed
## This is now only implemented with the 'sum' method
from linetools.spectra.xspectrum1d import XSpectrum1D
if isinstance(cube,str):
cube = DataCube(cube)
if wvslice is not None:
cube = kt.slice_cube(cube,wvslice[0],wvslice[1])
if varcube is not None:
varcube = kt.slice_cube(varcube, wvslice[0], wvslice[1])
##import pdb;
#pdb.set_trace()
dat = cube.data
# Get rid of NaNs
dc = dat.copy()
nans = np.where(np.isnan(dc))
dc[nans]=0
# Perform the extraction
if method == 'median':
if np.shape(np.array(pixels))==(2,):
extspec = dc[:,pixels[0],pixels[1]]
else:
spaxels = []
for i,px in enumerate(pixels):
thisspec = dc[:,px[0],px[1]]
spaxels.append(thisspec)
spaxels=np.array(spaxels)
extspec = np.median(spaxels, axis=0)
sigspec = -99. * np.ones(np.shape(dc)[0])
elif method == 'mean':
if np.shape(np.array(pixels))==(2,):
extspec = dc[:,pixels[0],pixels[1]]
else:
spaxels = []
for i,px in enumerate(pixels):
thisspec = dc[:,px[0],px[1]]
spaxels.append(thisspec)
spaxels=np.array(spaxels)
extspec = np.mean(spaxels, axis=0)
sigspec = -99. * np.ones(np.shape(dc)[0])
elif method == 'sum':
if np.shape(np.array(pixels))==(2,):
extspec = dc[:,pixels[0],pixels[1]]
else:
spaxels = []
for i,px in enumerate(pixels):
thisspec = dc[:,px[0],px[1]]
spaxels.append(thisspec)
spaxels=np.array(spaxels)
sigspec = -99. * np.ones(np.shape(dc)[0])
if spatial_scale_xy is None:
extspec = np.sum(spaxels, axis=0)
else:
spxsz = spatial_scale_xy[0] * spatial_scale_xy[1]
extspec = spxsz * np.sum(spaxels, axis=0)
else: # do the weighted mean
vardat = varcube.data
vdc = vardat.copy()
if np.shape(np.array(pixels))==(2,):
extspec = dc[:,pixels[0],pixels[1]]
else:
spaxels = []
varspaxels = []
for i, px in enumerate(pixels):
thisspec = dc[:, px[0], px[1]]
thisvar = vdc[:, px[0], px[1]]
spaxels.append(thisspec)
varspaxels.append(thisvar)
spaxels = np.array(spaxels)
ivarspaxels = 1./np.array(varspaxels)
extspec = np.sum(spaxels*ivarspaxels, axis=0)/np.sum(ivarspaxels,axis=0)
sigspec = 1./np.sqrt(np.sum(ivarspaxels,axis=0))
# Get the wavelength array
newwavearr = cube.wavelength
# Create the spectrum
try:
spec = XSpectrum1D(wave = newwavearr,flux=extspec,sig=sigspec)
except:
import pdb; pdb.set_trace()
return spec
def miss_id_check(path, idt, resol):
'''
Function for requesting spectra and meta for undetected DLAs
:param path object: path for the spectra
:param idt float: undetected id
:param resol: resolution for spectr
:return: two class object: 1. meta for spectra 2. spectra for undetected DLAs
'''
# define the variable
dataz = []
z = []
zerr = []
fluxt = []
wavetotal = []
ivar = []
mockid = []
ra = []
dec = []
mockid = []
filename = []
wvmin = []
wvmax = []
npix = []
date = []
# loop through the whole directory for requesting spectra
specf = []
mag = []
item1 = os.listdir(str(path))
for k in item1:
item = os.listdir(str(path) + str(k))
# print (item)
for j in item:
if os.listdir(str(path) + str(k) + '/' + str(j) + '/'):
dataz = fits.open(
str(path) + str(k) + '/' + str(j) + '/truth-16-' + str(
j) + '.fits')
data = fits.open(str(path) + str(k) + '/' + str(
j) + '/spectra-16-' + str(j) + '.fits') # k j j
indexcut = np.isin(data[1].data['TARGETID'], idt)
wavecut = (np.array(data[2].data) < 5730)
wavecut2 = ((np.array(data[7].data) > 5730) & (np.array(data[7].data) < 7560))
wavecut3 = (np.array(data[12].data) > 7560)
z.append(dataz[1].data['Z'][indexcut])
zerr.append(dataz[1].data['TRUEZ'][indexcut])
ra.append(data[1].data['TARGET_RA'][indexcut])
dec.append(data[1].data['TARGET_DEC'][indexcut])
mockid.append(data[1].data['TARGETID'][indexcut])
# combining the spectra from three channels
wavetotal.append(np.repeat(
[np.concatenate([data[2].data[wavecut], data[7].data[wavecut2], data[12].data[wavecut3]])],
len(dataz[1].data['Z']), axis=0)[indexcut])
fluxt.append(
np.concatenate((data[3].data[:, wavecut], data[8].data[:, wavecut2], data[13].data[:, wavecut3]),
axis=1)[indexcut])
ivar.append(
np.concatenate((data[4].data[:, wavecut], data[9].data[:, wavecut2], data[14].data[:, wavecut3]),
axis=1)[indexcut])
sp = XSpectrum1D(np.vstack(np.array(wavetotal)), np.vstack(np.array(fluxt)), np.sqrt(1 / np.vstack(np.array(ivar))),
verbose=False)
# ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr
binspec = ltsu.rebin_to_rest(sp, np.zeros(len(np.array(np.concatenate(z)))), resol * u.km / u.s)
specf = SPEC.spec_meta(np.concatenate(ra), np.concatenate(dec), np.concatenate(mockid),
np.min(binspec.wavelength),
np.max(binspec.wavelength), len(binspec.wavelength),
np.concatenate(z), np.concatenate(zerr))
# newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso)
return specf, binspec
