def mean_templ_zi(zimag, debug=False, i_wind=0.1, z_wind=0.05,
boss_pca_fil=None):
'''
Generate 'mean' templates at given z,i
Parameters
----------
zimag: list of tuples
Redshift, imag pairs for the templates
i_wind: float (0.1 mag)
Window for smoothing imag
z_wind: float (0.05 mag)
Window for smoothing redshift
'''
# PCA values
if boss_pca_fil is None:
boss_pca_fil = 'BOSS_DR10Lya_PCA_values_nocut.fits.gz'
hdu = fits.open(boss_pca_fil)
pca_coeff = hdu[1].data
# BOSS Eigenvectors
eigen, eigen_wave = fbq.read_qso_eigen()
npix = len(eigen_wave)
# Open the BOSS catalog file
boss_cat_fil = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
zQSO = t_boss['z_pipe']
tmp = t_boss['PSFMAG']
imag = tmp[:,3] # i-band mag
# Output array
ntempl = len(zimag)
out_spec = np.zeros( (ntempl, npix) )
# Iterate on z,imag
for izi in zimag:
tt = zimag.index(izi)
# Find matches
idx = np.where( (np.fabs(imag-izi[1]) < i_wind) &
(np.fabs(zQSO-izi[0]) < z_wind))[0]
if len(idx) < 50:
raise ValueError('mean_templ_zi: Not enough QSOs! {:d}'.format(len(idx)))
# Calculate median PCA values
PCA0 = np.median(pca_coeff['PCA0'][idx])
PCA1 = np.median(pca_coeff['PCA1'][idx])
PCA2 = np.median(pca_coeff['PCA2'][idx])
PCA3 = np.median(pca_coeff['PCA3'][idx])
acoeff = np.array( [PCA0, PCA1, PCA2, PCA3] )
# Make the template
out_spec[tt,:] = np.dot(eigen.T,acoeff)
if debug is True:
xdb.xplot(eigen_wave*(1.+izi[0]), out_spec[tt,:])
xdb.set_trace()
# Return
return out_spec
def mean_templ_zi(zimag, debug=False, i_wind=0.1, z_wind=0.05,
boss_pca_fil=None):
'''
Generate 'mean' templates at given z,i
Parameters
----------
zimag: list of tuples
Redshift, imag pairs for the templates
i_wind: float (0.1 mag)
Window for smoothing imag
z_wind: float (0.05 mag)
Window for smoothing redshift
'''
# PCA values
if boss_pca_fil is None:
boss_pca_fil = 'BOSS_DR10Lya_PCA_values_nocut.fits.gz'
hdu = fits.open(boss_pca_fil)
pca_coeff = hdu[1].data
# BOSS Eigenvectors
eigen, eigen_wave = fbq.read_qso_eigen()
npix = len(eigen_wave)
# Open the BOSS catalog file
boss_cat_fil = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
zQSO = t_boss['z_pipe']
tmp = t_boss['PSFMAG']
imag = tmp[:,3] # i-band mag
# Output array
ntempl = len(zimag)
out_spec = np.zeros( (ntempl, npix) )
# Iterate on z,imag
for izi in zimag:
tt = zimag.index(izi)
# Find matches
idx = np.where( (np.fabs(imag-izi[1]) < i_wind) &
(np.fabs(zQSO-izi[0]) < z_wind))[0]
if len(idx) < 50:
raise ValueError('mean_templ_zi: Not enough QSOs! {:d}'.format(len(idx)))
# Calculate median PCA values
PCA0 = np.median(pca_coeff['PCA0'][idx])
PCA1 = np.median(pca_coeff['PCA1'][idx])
PCA2 = np.median(pca_coeff['PCA2'][idx])
PCA3 = np.median(pca_coeff['PCA3'][idx])
acoeff = np.array( [PCA0, PCA1, PCA2, PCA3] )
# Make the template
out_spec[tt,:] = np.dot(eigen.T,acoeff)
if debug is True:
xdb.xplot(eigen_wave*(1.+izi[0]), out_spec[tt,:])
xdb.set_trace()
# Return
return out_spec
def mk_pix_stau(self, spec, kbin=22.*u.km/u.s, debug=False, **kwargs):
""" Generate the smoothed tau array for kinematic tests
Parameters
----------
spec: Spectrum1D class
Input spectrum
velo is expected to have been filled already
fill: bool (True)
Fill the dictionary with some items that other kin programs may need
Returns
-------
out_kin : dict
Dictionary of kinematic measurements
JXP on 11 Dec 2014
"""
# Calcualte dv
imn = np.argmin( np.fabs(spec.velo) )
dv = np.abs( spec.velo[imn] - spec.velo[imn+1] )
# Test for bad pixels
pixmin = np.argmin( np.fabs( spec.velo-self.vmnx[0] ) )
pixmax = np.argmin( np.fabs( spec.velo-self.vmnx[1] ) )
pix = np.arange(pixmin, pixmax+1)
npix = len(pix)
badzero=np.where((spec.flux[pix] == 0) & (spec.sig[pix] <= 0))[0]
if len(badzero) > 0:
if np.max(badzero)-np.min(badzero) >= 5:
raise ValueError('orig_kin: too many or too large sections of bad data')
spec.flux[pix[badzero]] = np.mean(np.array([spec.flux[pix[np.min(badzero)-1]],
spec.flux[pix[np.max(badzero)+1]]]))
xdb.set_trace() # Should add sig too
# Generate the tau array
tau = np.zeros(npix)
gd = np.where((spec.flux[pix] > spec.sig[pix]/2.) &
(spec.sig[pix] > 0.) )
if len(gd) == 0:
raise ValueError('orig_kin: Profile too saturated.')
tau[gd] = np.log(1./spec.flux[pix[gd]])
sat = (pix == pix)
sat[gd] = False
tau[sat] = np.log(2./spec.sig[pix[sat]])
# Smooth
nbin = (np.round(kbin/dv)).value
kernel = Box1DKernel(nbin, mode='center')
stau = convolve(tau, kernel, boundary='fill', fill_value=0.)
if debug is True:
xdb.xplot(spec.velo[pix], tau, stau)
# Fill
self.stau = stau
self.pix = pix
def plot(self):
''' Plot the spectrum
Parameters
----------
'''
if self.sig is not None:
xdb.xplot(self.dispersion, self.flux, self.sig)
else:
xdb.xplot(self.dispersion, self.flux)
def plot(self):
''' Plot the spectrum
Parameters
----------
'''
if self.sig is not None:
xdb.xplot(self.dispersion, self.flux, self.sig)
else:
xdb.xplot(self.dispersion, self.flux)
# Make EW spline file
mk_ew_lyman_spline(24.)
"""
from xastropy.igm.fN import model as xifm
import multiprocessing
# xdb.set_trace()
# read f(N)
fN_model = xifm.default_model()
print(fN_model)
# tau_eff
# tst_wv = tau_eff_llist()
tst_wv = np.arange(915.0, 1255, 1.0)
# lamb = 1215.6701*(1+2.4)
adict = []
for wrest in tst_wv:
tdict = dict(ilambda=wrest * (1 + 2.4), zem=2.5, fN_model=fN_model)
adict.append(tdict)
pool = multiprocessing.Pool(4) # initialize thread pool N threads
ateff = pool.map(map_etl, adict)
# Plot
xdb.xplot(tst_wv, np.exp(-np.array(ateff)))
# xdb.set_trace()
# teff = ew_teff_lyman(lamb, 2.5, fN_model, NHI_MIN=12., NHI_MAX=17.)
# print('teff at z=2.4 :: %g' % teff)
# teff = ew_teff_lyman(3400., 2.4, fN_model)
# print('teff at 3400A = %g' % teff)
def do_boss_lya_parallel(istart, iend, cut_Lya, output, debug=False):
'''
Generate PCA coeff for the BOSS Lya DR10 dataset, v2.1
Parameters
----------
cut_Lya: boolean (True)
Avoid using the Lya forest in the analysis
'''
# Eigen
eigen, eigen_wave = read_qso_eigen()
# Open the BOSS catalog file
boss_cat_fil = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
nqso = len(t_boss)
pca_val = np.zeros((iend-istart, 4))
if cut_Lya is False:
print('do_boss: Not cutting the Lya Forest in the fit')
# Loop us -- Should spawn on multiple CPU
#for ii in range(nqso):
datdir = os.environ.get('BOSSPATH')+'/DR10/BOSSLyaDR10_spectra_v2.1/'
jj = 0
print('istart = {:d}'.format(istart))
for ii in range(istart,iend):
if (ii % 100) == 0:
print('ii = {:d}'.format(ii))
#print('ii = {:d}'.format(ii))
# Spectrum file
pnm = str(t_boss['PLATE'][ii])
fnm = str(t_boss['FIBERID'][ii]).rjust(4,str('0'))
mjd = str(t_boss['MJD'][ii])
sfil = datdir+pnm+'/speclya-'
sfil = sfil+pnm+'-'+mjd+'-'+fnm+'.fits.gz'
# Read spectrum
spec_hdu = fits.open(sfil)
t = spec_hdu[1].data
flux = t['flux']
wave = 10.**t['loglam']
ivar = t['ivar']
zqso = t_boss['z_pipe'][ii]
wrest = wave / (1+zqso)
wlya = 1215.
# Cut Lya forest?
if cut_Lya is True:
Ly_imn = np.argmin(np.abs(wrest-wlya))
else:
Ly_imn = 0
# Pack
imn = np.argmin(np.abs(wrest[Ly_imn]-eigen_wave))
npix = len(wrest[Ly_imn:])
imx = npix+imn
eigen_flux = eigen[:,imn:imx]
# FIT
tflux = flux[Ly_imn:]
tivar = ivar[Ly_imn:]
acoeff = fit_eigen(tflux, ivar, eigen_flux)
pca_val[jj,:] = acoeff
jj += 1
# Check
if debug is True:
model = np.dot(eigen.T,acoeff)
if flg_xdb is True:
xdb.xplot(wrest, flux, xtwo=eigen_wave, ytwo=model)
xdb.set_trace()
#xdb.set_trace()
print('Done with my subset {:d}, {:d}'.format(istart,iend))
if output is not None:
output.put((istart,iend,pca_val))
#output.put(None)
else:
return pca_val
flg_test += 2**5
wave = np.linspace(1260.0,1285.0,10000)
wave = wave * u.AA
# Single line (Lya)
zabs = 0.05
line = abs_line.Abs_Line(1215.6701*u.AA)
line.z = zabs
line.attrib['N'] = 20.0
line.attrib['b'] = 50.0
if flg_test % 2 == 1:
print('voigt: Single line test')
vmodel = voigt_model(wave, line, Npix=None) # No smoothing
xdb.xplot(vmodel.dispersion, vmodel.flux)
# Two lines (Lya)
line.attrib['N'] = 13.0
line.attrib['b'] = 20.0
line2 = abs_line.Abs_Line(1215.6701*u.AA)
line2.z = zabs + 5e-3
line2.attrib['N'] = 13.5
line2.attrib['b'] = 20.0
lines = [line,line2]
if flg_test % 4 >= 2:
for item in lines: print('z = %g' % item.z)
vmodel = voigt_model(wave, lines, Npix=None) # No smoothing
print('voigt: Multiple lines test')
xdb.xplot(vmodel.dispersion, vmodel.flux)
def do_sdss_lya_parallel(istart, iend, cut_Lya, output, debug=False):
'''
Generate PCA coeff for the SDSS DR7 dataset, 0.5<z<2
Parameters
----------
cut_Lya: boolean (True)
Avoid using the Lya forest in the analysis
'''
# Eigen
eigen, eigen_wave = read_qso_eigen()
# Open the BOSS catalog file
sdss_cat_fil = os.environ.get('SDSSPATH') + '/DR7_QSO/dr7_qso.fits.gz'
bcat_hdu = fits.open(sdss_cat_fil)
t_sdss = bcat_hdu[1].data
nqso = len(t_sdss)
pca_val = np.zeros((iend - istart, 4))
if cut_Lya is False:
print('do_sdss: Not cutting the Lya Forest in the fit')
# Loop us -- Should spawn on multiple CPU
#for ii in range(nqso):
datdir = os.environ.get('SDSSPATH') + '/DR7_QSO/spectro/1d_26/'
jj = 0
for ii in range(istart, iend):
if (ii % 1000) == 0:
print('SDSS ii = {:d}'.format(ii))
# Spectrum file
pnm = str(t_sdss['PLATE'][ii]).rjust(4, str('0'))
fnm = str(t_sdss['FIBERID'][ii]).rjust(3, str('0'))
mjd = str(t_sdss['MJD'][ii])
sfil = datdir + pnm + '/1d/spSpec-'
sfil = sfil + mjd + '-' + pnm + '-' + fnm + '.fit.gz'
# Read spectrum
spec_hdu = fits.open(sfil)
head = spec_hdu[0].header
iwave = head['CRVAL1']
cdelt = head['CD1_1']
t = spec_hdu[0].data
flux = t[0, :]
sig = t[2, :]
npix = len(flux)
wave = 10.**(iwave + np.arange(npix) * cdelt)
ivar = np.zeros(npix)
gd = np.where(sig > 0.)[0]
ivar[gd] = 1. / sig[gd]**2
zqso = t_sdss['z'][ii]
wrest = wave / (1 + zqso)
wlya = 1215.
# Cut Lya forest?
if cut_Lya is True:
Ly_imn = np.argmin(np.abs(wrest - wlya))
else:
Ly_imn = 0
# Pack
imn = np.argmin(np.abs(wrest[Ly_imn] - eigen_wave))
npix = len(wrest[Ly_imn:])
imx = npix + imn
eigen_flux = eigen[:, imn:imx]
# FIT
acoeff = fit_eigen(flux[Ly_imn:], ivar[Ly_imn:], eigen_flux)
pca_val[jj, :] = acoeff
jj += 1
# Check
if debug is True:
model = np.dot(eigen.T, acoeff)
if flg_xdb is True:
xdb.xplot(wrest, flux, xtwo=eigen_wave, ytwo=model)
xdb.set_trace()
#xdb.set_trace()
print('Done with my subset {:d}, {:d}'.format(istart, iend))
if output is not None:
output.put((istart, iend, pca_val))
#output.put(None)
else:
return pca_val
def ew_teff_lyman(ilambda,
zem,
fN_model,
NHI_MIN=11.5,
NHI_MAX=22.0,
N_eval=5000,
EW_spline=None,
bval=24.,
fNz=False,
cosmo=None,
debug=False,
cumul=None,
verbose=False):
""" tau effective (follows ew_teff_lyman.pro from XIDL)
teff = ew_teff_lyman(3400., 2.4)
Parameters:
-------------
ilambda: float
Observed wavelength
zem: float
Emission redshift of the source [sets which Lyman lines are included]
bva: float
-- Characteristics Doppler parameter for the Lya forest
-- [Options: 24, 35 km/s]
NHI_MIN: float
-- Minimum log HI column for integration [default = 11.5]
NHI_MAX: float
-- Maximum log HI column for integration [default = 22.0]
fNz: Boolean (False)
-- Inputs f(N,z) instead of f(N,X)
cosmo: astropy.cosmology (None)
-- Cosmological model to adopt (as needed)
cumul: List of cumulative sums
-- Recorded only if cumul is not None
Returns:
teff:
Total effective opacity of all lines contributing
ToDo:
1. Parallelize the Lyman loop
JXP 07 Nov 2014
"""
# Lambda
if not isinstance(ilambda, float):
raise ValueError('igm.tau_eff: ilambda must be a float for now')
Lambda = ilambda
if not isinstance(Lambda, u.quantity.Quantity):
Lambda = Lambda * u.AA # Ang
# Read in EW spline (if needed)
if EW_spline == None:
if int(bval) == 24:
EW_FIL = xa_path + '/igm/EW_SPLINE_b24.p'
elif int(bval) == 35:
EW_FIL = os.environ.get('XIDL_DIR') + '/IGM/EW_SPLINE_b35.fits'
else:
raise ValueError('igm.tau_eff: Not ready for this bvalue %g' %
bval)
EW_spline = pickle.load(open(EW_FIL, "rb"))
# Lines
wrest = tau_eff_llist()
# Find the lines
gd_Lyman = wrest[(Lambda / (1 + zem)) < wrest]
nlyman = len(gd_Lyman)
if nlyman == 0:
if verbose:
print('igm.tau_eff: No Lyman lines covered at this wavelength')
return 0
# N_HI grid
lgNval = NHI_MIN + (NHI_MAX - NHI_MIN) * np.arange(N_eval) / (N_eval - 1
) # Base 10
dlgN = lgNval[1] - lgNval[0]
Nval = 10.**lgNval
teff_lyman = np.zeros(nlyman)
# For cumulative
if not cumul is None:
cumul.append(lgNval)
# Loop on the lines
for qq, line in enumerate(
gd_Lyman): # Would be great to do this in parallel...
# (Can pack together and should)
# Redshift
zeval = ((Lambda / line) - 1).value
if zeval < 0.:
teff_lyman[qq] = 0.
continue
# Cosmology
if fNz is False:
if cosmo not in locals():
cosmo = FlatLambdaCDM(H0=70, Om0=0.3) # Vanilla
#dxdz = (np.fabs(xigmu.cosm_xz(zeval-0.1, cosmo=cosmo)-
# xigmu.cosm_xz(zeval+0.1,cosmo=cosmo)) / 0.2 )
#xdb.set_trace()
dxdz = xigmu.cosm_xz(zeval, cosmo=cosmo, flg=1)
else:
dxdz = 1. # Code is using f(N,z)
#print('dxdz = %g' % dxdz)
# Get EW values (could pack these all together)
idx = np.where(EW_spline['wrest'] == line)[0]
if len(idx) != 1:
raise ValueError(
'tau_eff: Line %g not included or over included?!' % line)
restEW = interpolate.splev(lgNval, EW_spline['tck'][idx], der=0)
# dz
dz = ((restEW * u.AA) * (1 + zeval) / line).value
# Evaluate f(N,X) at zeval
log_fnX = fN_model.eval(lgNval, zeval).flatten()
#xdb.set_trace()
# Sum
intgrnd = 10.**(log_fnX) * dxdz * dz * Nval
teff_lyman[qq] = np.sum(intgrnd) * dlgN * np.log(10.)
if not cumul is None:
cumul.append(np.cumsum(intgrnd) * dlgN * np.log(10.))
#xdb.set_trace()
# Debug
if debug == True:
xdb.xplot(lgNval, np.log10(10.**(log_fnX) * dxdz * dz * Nval))
#x_splot, lgNval, total(10.d^(log_fnX) * dxdz * dz * Nval,/cumul) * dlgN * alog(10.) / teff_lyman[qq], /bloc
#printcol, lgnval, log_fnx, dz, alog10(10.d^(log_fnX) * dxdz * dz * Nval)
#writecol, 'debug_file'+strtrim(qq,2)+'.dat', lgNval, restEW, log_fnX
xdb.set_trace()
#xdb.set_trace()
return np.sum(teff_lyman)
# Write
myspec.write_to_fits('tmp.fits')
if (flg_test % 2**2) >= 2**1:
# Now 2D
fil = '/Users/xavier/Dropbox/QSOPairs/data/LRIS_redux/SDSSJ231254.65-025403.1_b400_F.fits.gz'
myspec = xsr.readspec(fil)
myspec.write_to_fits('tmp.fits')
if (flg_test % 2**3) >= 2**2: # Boxcar
fil = '~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits'
myspec = xsr.readspec(fil)
newspec = myspec.box_smooth(3)
#
newspec2 = myspec.box_smooth(3, preserve=True)
xdb.xplot(myspec.dispersion, myspec.flux, newspec2.flux)
if (flg_test % 2**4) >= 2**3: # Rebin array
fil = '~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits'
myspec = xsr.readspec(fil)
new_wv = np.arange(3000., 9000., 5) * u.AA
newspec = myspec.rebin(new_wv)
#xdb.xplot(myspec.dispersion, myspec.flux,
# xtwo=new_wv, ytwo=newspec.flux)
# Test EW
wvmnx = np.array((4859., 4961.))*u.AA
gd1 = np.where( (myspec.dispersion > wvmnx[0]) &
(myspec.dispersion < wvmnx[1]))[0]
dwv1 = myspec.dispersion - np.roll(myspec.dispersion,1)
EW1 = np.sum(dwv1[gd1]*(1.-myspec.flux[gd1].value))
I don't think the documentation of this module is correct.
"""
from __future__ import print_function, absolute_import, division
import numpy as np
import os
from astropy.io import fits
flg_xdb = True
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
flg_xdb = False
"""
## Tinkering about
# JXP on 01 Dec 2014
hdu = fits.open('spEigenQSO-55732.fits')
eigen = hdu[0].data # There are 4 eigenvectors
wav = 10.**(2.6534 + np.arange(13637)*0.0001) # Rest-frame, Ang
xdb.xplot(wav,eigen[0,:])
# Looks sensible enough
# QSO sample used to generate the PCA eigenvectors
qsos = hdu[1].data
xdb.xhist(qsos['REDSHIFT']) # Skewed to high z
"""
# Return
if dat is not None:
return dat.flatten(), tag
else:
return dat, "NONE"
#### ###############################
# Testing
if __name__ == "__main__":
flg_test = 0
flg_test += 1 # MagE
flg_test += 2 ** 1 # LRIS LowRedux
# Standard log-linear read (MagE)
if (flg_test % 2 ** 1) >= 2 ** 0:
fil = "~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits"
# fil = '/Users/xavier/Dropbox/QSOPairs/data/MAGE_redux/SDSSJ085357.49-001106.1_F.fits.gz'
# efil = '~ers/xavier/PROGETTI/LLSZ3/data/normalize/UM669_nE.fits'
myspec = readspec(fil)
# xdb.xplot(myspec.dispersion, myspec.flux)
myspec.plot()
# xdb.xplot(myspec.dispersion, myspec.flux, myspec.uncertainty.array)
# LowRedux
if (flg_test % 2 ** 2) >= 2 ** 1:
fil = "/Users/xavier/Dropbox/QSOPairs/data/LRIS_redux/SDSSJ231254.65-025403.1_b400_F.fits.gz"
myspec = readspec(fil)
xdb.xplot(myspec.dispersion, myspec.flux, myspec.uncertainty.array)
# xdb.set_trace()
def lyman_ew(ilambda, zem, fN_model, NHI_MIN=11.5, NHI_MAX=22.0, N_eval=5000,
bval=24., cosmo=None, debug=False, cumul=None,
verbose=False, EW_spline=None, wrest=None):
""" tau effective from HI Lyman series absorption
Parameters
----------
ilambda : float
Observed wavelength (Ang)
zem : float
Emission redshift of the source [sets which Lyman lines are included]
fN_model : FNModel
NHI_MIN : float, optional
-- Minimum log HI column for integration [default = 11.5]
NHI_MAX : float, optional
-- Maximum log HI column for integration [default = 22.0]
N_eval : int, optional
Number of NHI evaluations
bval : float
-- Characteristics Doppler parameter for the Lya forest
-- [Options: 24, 35 km/s]
cosmo : astropy.cosmology (None)
-- Cosmological model to adopt (as needed)
cumul : List of cumulative sums
-- Recorded only if cumul is not None
EW_spline : spline, optional
Speeds up execution if input
HI : LineList, optional
HI line list. Speeds up execution
Returns
-------
teff : float
Total effective opacity of all lines contributing
ToDo:
1. Parallelize the Lyman loop
"""
# Cosmology
if cosmo is None:
cosmo = FlatLambdaCDM(H0=70, Om0=0.3)
# Lambda
if not isinstance(ilambda, float):
raise ValueError('tau_eff: ilambda must be a float for now')
Lambda = ilambda
if not isinstance(Lambda,u.quantity.Quantity):
Lambda = Lambda * u.AA # Ang
# Read in EW spline (if needed)
if EW_spline is None:
if int(bval) == 24:
EW_FIL = pyigm_path+'/data/fN/EW_SPLINE_b24.yml'
with open(EW_FIL, 'r') as infile:
EW_spline = yaml.load(infile) # dict from mk_ew_lyman_spline
else:
raise ValueError('tau_eff: Not ready for this bvalue %g' % bval)
# Lines
if wrest is None:
HI = LineList('HI')
wrest = HI._data['wrest']
# Find the lines
gd_Lyman = wrest[(Lambda/(1+zem)) < wrest]
nlyman = len(gd_Lyman)
if nlyman == 0:
if verbose:
print('igm.tau_eff: No Lyman lines covered at this wavelength')
return 0
# N_HI grid
lgNval = NHI_MIN + (NHI_MAX-NHI_MIN)*np.arange(N_eval)/(N_eval-1) # Base 10
dlgN = lgNval[1]-lgNval[0]
Nval = 10.**lgNval
teff_lyman = np.zeros(nlyman)
# For cumulative
if cumul is not None:
cumul.append(lgNval)
# Loop on the lines
for qq, line in enumerate(gd_Lyman): # Would be great to do this in parallel...
# (Can pack together and should)
# Redshift
zeval = ((Lambda / line) - 1).value
if zeval < 0.:
teff_lyman[qq] = 0.
continue
# dxdz
dxdz = pyigmu.cosm_xz(zeval, cosmo=cosmo, flg_return=1)
# Get EW values (could pack these all together)
idx = np.where(EW_spline['wrest']*u.AA == line)[0]
if len(idx) != 1:
raise ValueError('tau_eff: Line %g not included or over included?!' % line)
restEW = interpolate.splev(lgNval, EW_spline['tck'][idx], der=0)
# dz
dz = ((restEW*u.AA) * (1+zeval) / line).value
# Evaluate f(N,X) at zeval
log_fnX = fN_model.evaluate(lgNval, zeval, cosmo=cosmo).flatten()
# Sum
intgrnd = 10.**(log_fnX) * dxdz * dz * Nval
teff_lyman[qq] = np.sum(intgrnd) * dlgN * np.log(10.)
if cumul is not None:
cumul.append(np.cumsum(intgrnd) * dlgN * np.log(10.))
# Debug
if debug:
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
break
xdb.xplot(lgNval, np.log10(10.**(log_fnX) * dxdz * dz * Nval))
#x_splot, lgNval, total(10.d^(log_fnX) * dxdz * dz * Nval,/cumul) * dlgN * alog(10.) / teff_lyman[qq], /bloc
#printcol, lgnval, log_fnx, dz, alog10(10.d^(log_fnX) * dxdz * dz * Nval)
#writecol, 'debug_file'+strtrim(qq,2)+'.dat', lgNval, restEW, log_fnX
xdb.set_trace()
# Return
return np.sum(teff_lyman)
def generate_stau(velo, flux, sig, kbin=22.*u.km/u.s, debug=False):
""" Generate the smoothed tau array for kinematic tests
Parameters
----------
velo : Quantity array (usually km/s)
flux : Quantity array (flux)
sig : Quantity array (sig)
kbin : Quantity (velocity), optional
Kernel size for Gaussian smoothing of optical depth array
debug : bool, optional
Returns
-------
stau : array
Smoothed tau array
"""
# Velocity array
npix = len(velo)
pix = np.arange(npix).astype(int)
# Calculate dv
dv = np.abs(np.median(velo-np.roll(velo,1)))
# Test for bad pixels
badzero = np.where((flux == 0) | (sig <= 0))[0]
if len(badzero) > 0:
if np.max(badzero)-np.min(badzero) >= 5:
raise ValueError('orig_kin: too many or too large sections of bad data')
flux[badzero] = np.mean(np.array([flux[np.min(badzero)-1],
flux[np.max(badzero)+1]]))
pdb.set_trace() # Should add sig too
# Generate the tau array
tau = np.zeros(npix)
gd = np.where((flux > sig/2.) & (sig > 0.) )
if len(gd) == 0:
raise ValueError('orig_kin: Profile too saturated.')
tau[gd] = np.log(1./flux[gd])
sat = (pix == pix)
sat[gd] = False
tau[sat] = np.log(2./sig[sat])
# Smooth
nbin = (np.round(kbin/dv)).value
try:
kernel = Box1DKernel(nbin, mode='center')
except:
pdb.set_trace()
stau = convolve(tau, kernel, boundary='fill', fill_value=0.)
if debug is True:
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
pdb.set_trace()
else:
xdb.xplot(velo, tau, stau)
xdb.set_trace()
# Return
return stau
def lyman_ew(ilambda, zem, fN_model, NHI_MIN=11.5, NHI_MAX=22.0, N_eval=5000,
bval=24., cosmo=None, debug=False, cumul=None,
verbose=False, EW_spline=None, wrest=None):
""" tau effective from HI Lyman series absorption
Parameters
----------
ilambda : float
Observed wavelength (Ang)
zem : float
Emission redshift of the source [sets which Lyman lines are included]
fN_model : FNModel
NHI_MIN : float, optional
-- Minimum log HI column for integration [default = 11.5]
NHI_MAX : float, optional
-- Maximum log HI column for integration [default = 22.0]
N_eval : int, optional
Number of NHI evaluations
bval : float
-- Characteristics Doppler parameter for the Lya forest
-- [Options: 24, 35 km/s]
cosmo : astropy.cosmology (None)
-- Cosmological model to adopt (as needed)
cumul : List of cumulative sums
-- Recorded only if cumul is not None
EW_spline : spline, optional
Speeds up execution if input
HI : LineList, optional
HI line list. Speeds up execution
Returns
-------
teff : float
Total effective opacity of all lines contributing
ToDo:
1. Parallelize the Lyman loop
"""
# Cosmology
if cosmo is None:
cosmo = FlatLambdaCDM(H0=70, Om0=0.3)
# Lambda
if not isinstance(ilambda, float):
raise ValueError('tau_eff: ilambda must be a float for now')
Lambda = ilambda
if not isinstance(Lambda,u.quantity.Quantity):
Lambda = Lambda * u.AA # Ang
# Read in EW spline (if needed)
if EW_spline is None:
if int(bval) == 24:
EW_FIL = pyigm_path+'/data/fN/EW_SPLINE_b24.yml'
with open(EW_FIL, 'r') as infile:
EW_spline = yaml.load(infile) # dict from mk_ew_lyman_spline
else:
raise ValueError('tau_eff: Not ready for this bvalue %g' % bval)
# Lines
if wrest is None:
HI = LineList('HI')
wrest = HI._data['wrest']
# Find the lines
gd_Lyman = wrest[(Lambda/(1+zem)) < wrest]
nlyman = len(gd_Lyman)
if nlyman == 0:
if verbose:
print('igm.tau_eff: No Lyman lines covered at this wavelength')
return 0
# N_HI grid
lgNval = NHI_MIN + (NHI_MAX-NHI_MIN)*np.arange(N_eval)/(N_eval-1) # Base 10
dlgN = lgNval[1]-lgNval[0]
Nval = 10.**lgNval
teff_lyman = np.zeros(nlyman)
# For cumulative
if cumul is not None:
cumul.append(lgNval)
# Loop on the lines
for qq, line in enumerate(gd_Lyman): # Would be great to do this in parallel...
# (Can pack together and should)
# Redshift
zeval = ((Lambda / line) - 1).value
if zeval < 0.:
teff_lyman[qq] = 0.
continue
# dxdz
dxdz = pyigmu.cosm_xz(zeval, cosmo=cosmo, flg_return=1)
# Get EW values (could pack these all together)
idx = np.where(EW_spline['wrest']*u.AA == line)[0]
if len(idx) != 1:
raise ValueError('tau_eff: Line %g not included or over included?!' % line)
restEW = interpolate.splev(lgNval, EW_spline['tck'][idx[0]], der=0)
# dz
dz = ((restEW*u.AA) * (1+zeval) / line).value
# Evaluate f(N,X) at zeval
log_fnX = fN_model.evaluate(lgNval, zeval, cosmo=cosmo).flatten()
# Sum
intgrnd = 10.**(log_fnX) * dxdz * dz * Nval
teff_lyman[qq] = np.sum(intgrnd) * dlgN * np.log(10.)
if cumul is not None:
cumul.append(np.cumsum(intgrnd) * dlgN * np.log(10.))
# Debug
if debug:
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
break
xdb.xplot(lgNval, np.log10(10.**(log_fnX) * dxdz * dz * Nval))
#x_splot, lgNval, total(10.d^(log_fnX) * dxdz * dz * Nval,/cumul) * dlgN * alog(10.) / teff_lyman[qq], /bloc
#printcol, lgnval, log_fnx, dz, alog10(10.d^(log_fnX) * dxdz * dz * Nval)
#writecol, 'debug_file'+strtrim(qq,2)+'.dat', lgNval, restEW, log_fnX
xdb.set_trace()
# Return
return np.sum(teff_lyman)
#;------------------------------------------------------------------------------
"""
from __future__ import print_function, absolute_import, division, unicode_literals
import numpy as np
import os
from astropy.io import fits
flg_xdb = True
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
flg_xdb = False
"""
## Tinkering about
# JXP on 01 Dec 2014
hdu = fits.open('spEigenQSO-55732.fits')
eigen = hdu[0].data # There are 4 eigenvectors
wav = 10.**(2.6534 + np.arange(13637)*0.0001) # Rest-frame, Ang
xdb.xplot(wav,eigen[0,:])
# Looks sensible enough
# QSO sample used to generate the PCA eigenvectors
qsos = hdu[1].data
xdb.xhist(qsos['REDSHIFT']) # Skewed to high z
"""
# Make EW spline file
mk_ew_lyman_spline(24.)
'''
from xastropy.igm.fN import model as xifm
import multiprocessing
#xdb.set_trace()
# read f(N)
fN_model = xifm.default_model()
print(fN_model)
# tau_eff
#tst_wv = tau_eff_llist()
tst_wv = np.arange(915., 1255, 1.)
#lamb = 1215.6701*(1+2.4)
adict = []
for wrest in tst_wv:
tdict = dict(ilambda=wrest * (1 + 2.4), zem=2.5, fN_model=fN_model)
adict.append(tdict)
pool = multiprocessing.Pool(4) # initialize thread pool N threads
ateff = pool.map(map_etl, adict)
# Plot
xdb.xplot(tst_wv, np.exp(-np.array(ateff)))
#xdb.set_trace()
#teff = ew_teff_lyman(lamb, 2.5, fN_model, NHI_MIN=12., NHI_MAX=17.)
#print('teff at z=2.4 :: %g' % teff)
#teff = ew_teff_lyman(3400., 2.4, fN_model)
#print('teff at 3400A = %g' % teff)
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
slf
det
obj_skyspec
arx_skyspec
Returns
-------
flex_dict
"""
#Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)
#Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
#Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(arx_skyspec.wavelength.value[1]-arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:]-arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(obj_skyspec.wavelength.value[1]-obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:]-obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
#Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
arx_idx = (arx_cent+0.5).astype(np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent+0.5).astype(np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
np.median(arx_res), np.median(obj_res)))
#Determine sigma of gaussian for smoothing
arx_sig2 = (arx_disp[arx_idx]*arx_wid[arx_w][arx_keep])**2.
obj_sig2 = (obj_disp[obj_idx]*obj_wid[obj_w][obj_keep])**2.
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2-arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
if np.median(obj_res) >= np.median(arx_res):
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value>=min_wave) &
(obj_skyspec.wavelength.value<=max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.error("Not enough overlap between sky spectra")
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
#deal with bad pixels
msgs.work("Need to mask bad pixels")
#deal with underlying continuum
msgs.work("Need to deal with underlying continuum")
#Cross correlation of spectra
corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within max_shift of zero lag
lag0 = corr.size/2
mxshft = slf._argflag['reduce']['flexure']['max_shift']
max_corr = np.argmax(corr[lag0-mxshft:lag0+mxshft]) + lag0-mxshft
subpix_grid = np.linspace(max_corr-3., max_corr+3., 7.)
#Fit a 2-degree polynomial to peak of correlation function
fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2)
max_fit = -0.5*fit[1]/fit[2]
#Calculate and apply shift in wavelength
shift = float(max_fit)-lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
if msgs._debug['flexure']:
debugger.xplot(arx_skyspec.wavelength, arx_skyspec.flux, xtwo=obj_skyspec.wavelength, ytwo=obj_skyspec.flux)
#debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
debugger.set_trace()
flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size/2, smooth=smooth_sig_pix)
# Return
return flex_dict
# Return
if dat is not None:
return dat.flatten(), tag
else:
return dat, 'NONE'
#### ###############################
# Testing
if __name__ == '__main__':
flg_test = 0
flg_test += 1 # MagE
flg_test += 2**1 # LRIS LowRedux
# Standard log-linear read (MagE)
if (flg_test % 2**1) >= 2**0:
fil = '~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits'
#fil = '/Users/xavier/Dropbox/QSOPairs/data/MAGE_redux/SDSSJ085357.49-001106.1_F.fits.gz'
#efil = '~ers/xavier/PROGETTI/LLSZ3/data/normalize/UM669_nE.fits'
myspec = readspec(fil)
#xdb.xplot(myspec.dispersion, myspec.flux)
myspec.plot()
#xdb.xplot(myspec.dispersion, myspec.flux, myspec.uncertainty.array)
# LowRedux
if (flg_test % 2**2) >= 2**1:
fil = '/Users/xavier/Dropbox/QSOPairs/data/LRIS_redux/SDSSJ231254.65-025403.1_b400_F.fits.gz'
myspec = readspec(fil)
xdb.xplot(myspec.dispersion, myspec.flux, myspec.uncertainty.array)
#xdb.set_trace()
def flex_shift(slf, det, obj_skyspec, arx_skyspec):
""" Calculate shift between object sky spectrum and archive sky spectrum
Parameters
----------
slf
det
obj_skyspec
arx_skyspec
Returns
-------
flex_dict
"""
#Determine the brightest emission lines
msgs.warn("If we use Paranal, cut down on wavelength early on")
arx_amp, arx_cent, arx_wid, arx_w, arx_satsnd, arx_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=arx_skyspec.flux.value, MK_SATMASK=False)
obj_amp, obj_cent, obj_wid, obj_w, obj_satsnd, obj_yprep = ararc.detect_lines(slf, det, msarc=None, censpec=obj_skyspec.flux.value, MK_SATMASK=False)
#Keep only 5 brightest amplitude lines (xxx_keep is array of indices within arx_w of the 5 brightest)
arx_keep = np.argsort(arx_amp[arx_w])[-5:]
obj_keep = np.argsort(obj_amp[obj_w])[-5:]
#Calculate wavelength (Angstrom per pixel)
arx_disp = np.append(arx_skyspec.wavelength.value[1]-arx_skyspec.wavelength.value[0],
arx_skyspec.wavelength.value[1:]-arx_skyspec.wavelength.value[:-1])
#arx_disp = (np.amax(arx_sky.wavelength.value)-np.amin(arx_sky.wavelength.value))/arx_sky.wavelength.size
obj_disp = np.append(obj_skyspec.wavelength.value[1]-obj_skyspec.wavelength.value[0],
obj_skyspec.wavelength.value[1:]-obj_skyspec.wavelength.value[:-1])
#obj_disp = (np.amax(obj_sky.wavelength.value)-np.amin(obj_sky.wavelength.value))/obj_sky.wavelength.size
#Calculate resolution (lambda/delta lambda_FWHM)..maybe don't need this? can just use sigmas
arx_idx = (arx_cent+0.5).astype(np.int)[arx_w][arx_keep] # The +0.5 is for rounding
arx_res = arx_skyspec.wavelength.value[arx_idx]/\
(arx_disp[arx_idx]*(2*np.sqrt(2*np.log(2)))*arx_wid[arx_w][arx_keep])
obj_idx = (obj_cent+0.5).astype(np.int)[obj_w][obj_keep] # The +0.5 is for rounding
obj_res = obj_skyspec.wavelength.value[obj_idx]/ \
(obj_disp[obj_idx]*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
#obj_res = (obj_sky.wavelength.value[0]+(obj_disp*obj_cent[obj_w][obj_keep]))/(
# obj_disp*(2*np.sqrt(2*np.log(2)))*obj_wid[obj_w][obj_keep])
msgs.info("Resolution of Archive={:g} and Observation={:g}".format(
np.median(arx_res), np.median(obj_res)))
#Determine sigma of gaussian for smoothing
arx_sig2 = (arx_disp[arx_idx]*arx_wid[arx_w][arx_keep])**2.
obj_sig2 = (obj_disp[obj_idx]*obj_wid[obj_w][obj_keep])**2.
arx_med_sig2 = np.median(arx_sig2)
obj_med_sig2 = np.median(obj_sig2)
if obj_med_sig2 >= arx_med_sig2:
smooth_sig = np.sqrt(obj_med_sig2-arx_med_sig2) # Ang
smooth_sig_pix = smooth_sig / np.median(arx_disp[arx_idx])
else:
msgs.warn("Prefer archival sky spectrum to have higher resolution")
smooth_sig_pix = 0.
#smooth_sig = np.sqrt(arx_med_sig**2-obj_med_sig**2)
#Determine region of wavelength overlap
min_wave = max(np.amin(arx_skyspec.wavelength.value), np.amin(obj_skyspec.wavelength.value))
max_wave = min(np.amax(arx_skyspec.wavelength.value), np.amax(obj_skyspec.wavelength.value))
#Smooth higher resolution spectrum by smooth_sig (flux is conserved!)
if np.median(obj_res) >= np.median(arx_res):
msgs.warn("New Sky has higher resolution than Archive. Not smoothing")
#obj_sky_newflux = ndimage.gaussian_filter(obj_sky.flux, smooth_sig)
else:
#tmp = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
arx_skyspec = arx_skyspec.gauss_smooth(smooth_sig_pix*2*np.sqrt(2*np.log(2)))
#arx_sky.flux = ndimage.gaussian_filter(arx_sky.flux, smooth_sig)
# Define wavelengths of overlapping spectra
keep_idx = np.where((obj_skyspec.wavelength.value>=min_wave) &
(obj_skyspec.wavelength.value<=max_wave))[0]
#keep_wave = [i for i in obj_sky.wavelength.value if i>=min_wave if i<=max_wave]
#Rebin both spectra onto overlapped wavelength range
if len(keep_idx) <= 50:
msgs.error("Not enough overlap between sky spectra")
else: #rebin onto object ALWAYS
keep_wave = obj_skyspec.wavelength[keep_idx]
arx_skyspec = arx_skyspec.rebin(keep_wave)
obj_skyspec = obj_skyspec.rebin(keep_wave)
#deal with bad pixels
msgs.work("Need to mask bad pixels")
#deal with underlying continuum
msgs.work("Need to deal with underlying continuum")
#Cross correlation of spectra
corr = np.correlate(arx_skyspec.flux, obj_skyspec.flux, "same")
#Create array around the max of the correlation function for fitting for subpixel max
# Restrict to pixels within max_shift of zero lag
lag0 = corr.size/2
mxshft = slf._argflag['reduce']['flexure']['max_shift']
max_corr = np.argmax(corr[lag0-mxshft:lag0+mxshft]) + lag0-mxshft
subpix_grid = np.linspace(max_corr-3., max_corr+3., 7.)
#Fit a 2-degree polynomial to peak of correlation function
fit = arutils.func_fit(subpix_grid, corr[subpix_grid.astype(np.int)], 'polynomial', 2)
max_fit = -0.5*fit[1]/fit[2]
#Calculate and apply shift in wavelength
shift = float(max_fit)-lag0
msgs.info("Flexure correction of {:g} pixels".format(shift))
#model = (fit[2]*(subpix_grid**2.))+(fit[1]*subpix_grid)+fit[0]
if msgs._debug['flexure']:
debugger.xplot(arx_skyspec.wavelength, arx_skyspec.flux, xtwo=obj_skyspec.wavelength, ytwo=obj_skyspec.flux)
#debugger.xplot(arx_sky.wavelength, arx_sky.flux, xtwo=np.roll(obj_sky.wavelength.value,9), ytwo=obj_sky.flux*100)
debugger.set_trace()
flex_dict = dict(polyfit=fit, shift=shift, subpix=subpix_grid,
corr=corr[subpix_grid.astype(np.int)],
sky_spec=obj_skyspec,
arx_spec=arx_skyspec,
corr_cen=corr.size/2, smooth=smooth_sig_pix)
# Return
return flex_dict
def do_boss_lya_parallel(istart, iend, cut_Lya, output, debug=False):
'''
Generate PCA coeff for the BOSS Lya DR10 dataset, v2.1
Parameters
----------
cut_Lya: boolean (True)
Avoid using the Lya forest in the analysis
'''
# Eigen
eigen, eigen_wave = read_qso_eigen()
# Open the BOSS catalog file
boss_cat_fil = os.environ.get(
'BOSSPATH') + '/DR10/BOSSLyaDR10_cat_v2.1.fits.gz'
bcat_hdu = fits.open(boss_cat_fil)
t_boss = bcat_hdu[1].data
nqso = len(t_boss)
pca_val = np.zeros((iend - istart, 4))
if cut_Lya is False:
print('do_boss: Not cutting the Lya Forest in the fit')
# Loop us -- Should spawn on multiple CPU
#for ii in range(nqso):
datdir = os.environ.get('BOSSPATH') + '/DR10/BOSSLyaDR10_spectra_v2.1/'
jj = 0
print('istart = {:d}'.format(istart))
for ii in range(istart, iend):
if (ii % 100) == 0:
print('ii = {:d}'.format(ii))
#print('ii = {:d}'.format(ii))
# Spectrum file
pnm = str(t_boss['PLATE'][ii])
fnm = str(t_boss['FIBERID'][ii]).rjust(4, str('0'))
mjd = str(t_boss['MJD'][ii])
sfil = datdir + pnm + '/speclya-'
sfil = sfil + pnm + '-' + mjd + '-' + fnm + '.fits.gz'
# Read spectrum
spec_hdu = fits.open(sfil)
t = spec_hdu[1].data
flux = t['flux']
wave = 10.**t['loglam']
ivar = t['ivar']
zqso = t_boss['z_pipe'][ii]
wrest = wave / (1 + zqso)
wlya = 1215.
# Cut Lya forest?
if cut_Lya is True:
Ly_imn = np.argmin(np.abs(wrest - wlya))
else:
Ly_imn = 0
# Pack
imn = np.argmin(np.abs(wrest[Ly_imn] - eigen_wave))
npix = len(wrest[Ly_imn:])
imx = npix + imn
eigen_flux = eigen[:, imn:imx]
# FIT
tflux = flux[Ly_imn:]
tivar = ivar[Ly_imn:]
acoeff = fit_eigen(tflux, ivar, eigen_flux)
pca_val[jj, :] = acoeff
jj += 1
# Check
if debug is True:
model = np.dot(eigen.T, acoeff)
if flg_xdb is True:
xdb.xplot(wrest, flux, xtwo=eigen_wave, ytwo=model)
xdb.set_trace()
#xdb.set_trace()
print('Done with my subset {:d}, {:d}'.format(istart, iend))
if output is not None:
output.put((istart, iend, pca_val))
#output.put(None)
else:
return pca_val
def mk_pix_stau(self, spec, kbin=22. * u.km / u.s, debug=False, **kwargs):
""" Generate the smoothed tau array for kinematic tests
Parameters
----------
spec: Spectrum1D class
Input spectrum
velo is expected to have been filled already
fill: bool (True)
Fill the dictionary with some items that other kin programs may need
Returns
-------
out_kin : dict
Dictionary of kinematic measurements
JXP on 11 Dec 2014
"""
# Calcualte dv
imn = np.argmin(np.fabs(spec.velo))
dv = np.abs(spec.velo[imn] - spec.velo[imn + 1])
# Test for bad pixels
pixmin = np.argmin(np.fabs(spec.velo - self.vmnx[0]))
pixmax = np.argmin(np.fabs(spec.velo - self.vmnx[1]))
pix = np.arange(pixmin, pixmax + 1)
npix = len(pix)
badzero = np.where((spec.flux[pix] == 0) & (spec.sig[pix] <= 0))[0]
if len(badzero) > 0:
if np.max(badzero) - np.min(badzero) >= 5:
raise ValueError(
'orig_kin: too many or too large sections of bad data')
spec.flux[pix[badzero]] = np.mean(
np.array([
spec.flux[pix[np.min(badzero) - 1]],
spec.flux[pix[np.max(badzero) + 1]]
]))
xdb.set_trace() # Should add sig too
# Generate the tau array
tau = np.zeros(npix)
gd = np.where((spec.flux[pix] > spec.sig[pix] / 2.)
& (spec.sig[pix] > 0.))
if len(gd) == 0:
raise ValueError('orig_kin: Profile too saturated.')
tau[gd] = np.log(1. / spec.flux[pix[gd]])
sat = (pix == pix)
sat[gd] = False
tau[sat] = np.log(2. / spec.sig[pix[sat]])
# Smooth
nbin = (np.round(kbin / dv)).value
kernel = Box1DKernel(nbin, mode='center')
stau = convolve(tau, kernel, boundary='fill', fill_value=0.)
if debug is True:
xdb.xplot(spec.velo[pix], tau, stau)
# Fill
self.stau = stau
self.pix = pix
# Plot with Data
if (flg_test % 2**3) >= 2**2:
fN_model = xifm.default_model()
fN_data.tst_fn_data(fN_model=fN_model)
# Check l(X)
if (flg_test % 16) >= 8:
fN_model = xifm.default_model()
lX = fN_model.calc_lox(2.4, 17.19 + np.log10(2.), 23.)
print('l(X) = %g' % lX)
# Check teff_LL
if (flg_test % 32) >= 16:
fN_model = xifm.default_model()
zval, teff_LL = fN_model.teff_ll(0.5, 2.45)
xdb.xplot(zval, teff_LL) #,xlabel='z', ylabel=r'$\tau_{\rm LL}$')
# Check MFP
if (flg_test % 64) >= 32:
fN_model = xifm.default_model()
z = 2.44
mfp = fN_model.mfp(z)
print('MFP at z={:g} is {:g} Mpc'.format(z, mfp.value))
# Check Inoue+14
if (flg_test % 2**7) >= 2**6:
print('Testing Akio Model')
fN_model = fN_Model('Gamma')
NHI = [12., 14., 17., 21.]
z = 2.5
dXdz = igmu.cosm_xz(z, flg=1)
#flg_fig += 2 # NHI plot
flg_fig += 2**2 # Simple Kinematics
# Load FITS
if (flg_fig % 2) == 1:
cos_halos = COSHalos()
cos_halos.load_mega()
print(cos_halos)
# Simple rho vs NHI plot
if (flg_fig % 2**2) >= 2**1:
cos_halos = COSHalos()
cos_halos.load_mega()
x= cos_halos.rho
y= cos_halos.NHI
xdb.xplot(x, y, scatter=True)
#
# Simple kinematics
if (flg_fig % 2**3) >= 2**2:
cos_halos = COSHalos()
cos_halos.load_mega()#test=True)
cos_halos.load_abskin()
# Plot
mtl_kin = cos_halos.abs_kin('Metal')
gd = np.where(mtl_kin['flg'] > 0)
xdb.xplot(cos_halos.NHI[gd], mtl_kin['Dv'][gd], scatter=True)
HI_kin = cos_halos.abs_kin('HI')
gd = np.where(HI_kin['flg'] > 0)
xdb.xplot(cos_halos.NHI[gd], HI_kin['Dv'][gd], scatter=True)
print('All done')
def do_sdss_lya_parallel(istart, iend, cut_Lya, output, debug=False):
'''
Generate PCA coeff for the SDSS DR7 dataset, 0.5<z<2
Parameters
----------
cut_Lya: boolean (True)
Avoid using the Lya forest in the analysis
'''
# Eigen
eigen, eigen_wave = read_qso_eigen()
# Open the BOSS catalog file
sdss_cat_fil = os.environ.get('SDSSPATH')+'/DR7_QSO/dr7_qso.fits.gz'
bcat_hdu = fits.open(sdss_cat_fil)
t_sdss = bcat_hdu[1].data
nqso = len(t_sdss)
pca_val = np.zeros((iend-istart, 4))
if cut_Lya is False:
print('do_sdss: Not cutting the Lya Forest in the fit')
# Loop us -- Should spawn on multiple CPU
#for ii in range(nqso):
datdir = os.environ.get('SDSSPATH')+'/DR7_QSO/spectro/1d_26/'
jj = 0
for ii in range(istart,iend):
if (ii % 1000) == 0:
print('SDSS ii = {:d}'.format(ii))
# Spectrum file
pnm = str(t_sdss['PLATE'][ii]).rjust(4,str('0'))
fnm = str(t_sdss['FIBERID'][ii]).rjust(3,str('0'))
mjd = str(t_sdss['MJD'][ii])
sfil = datdir+pnm+'/1d/spSpec-'
sfil = sfil+mjd+'-'+pnm+'-'+fnm+'.fit.gz'
# Read spectrum
spec_hdu = fits.open(sfil)
head = spec_hdu[0].header
iwave = head['CRVAL1']
cdelt = head['CD1_1']
t = spec_hdu[0].data
flux = t[0,:]
sig = t[2,:]
npix = len(flux)
wave = 10.**(iwave + np.arange(npix)*cdelt)
ivar = np.zeros(npix)
gd = np.where(sig>0.)[0]
ivar[gd] = 1./sig[gd]**2
zqso = t_sdss['z'][ii]
wrest = wave / (1+zqso)
wlya = 1215.
# Cut Lya forest?
if cut_Lya is True:
Ly_imn = np.argmin(np.abs(wrest-wlya))
else:
Ly_imn = 0
# Pack
imn = np.argmin(np.abs(wrest[Ly_imn]-eigen_wave))
npix = len(wrest[Ly_imn:])
imx = npix+imn
eigen_flux = eigen[:,imn:imx]
# FIT
acoeff = fit_eigen(flux[Ly_imn:], ivar[Ly_imn:], eigen_flux)
pca_val[jj,:] = acoeff
jj += 1
# Check
if debug is True:
model = np.dot(eigen.T,acoeff)
if flg_xdb is True:
xdb.xplot(wrest, flux, xtwo=eigen_wave, ytwo=model)
xdb.set_trace()
#xdb.set_trace()
print('Done with my subset {:d}, {:d}'.format(istart,iend))
if output is not None:
output.put((istart,iend,pca_val))
#output.put(None)
else:
return pca_val
#flg_fig += 2 # NHI plot
flg_fig += 2**2 # Simple Kinematics
# Load FITS
if (flg_fig % 2) == 1:
cos_halos = COS_Halos()
cos_halos.load_mega()
print(cos_halos)
# Simple rho vs NHI plot
if (flg_fig % 2**2) >= 2**1:
cos_halos = COS_Halos()
cos_halos.load_mega()
x = cos_halos.rho
y = cos_halos.NHI
xdb.xplot(x, y, scatter=True)
#
# Simple kinematics
if (flg_fig % 2**3) >= 2**2:
cos_halos = COS_Halos()
cos_halos.load_mega() #test=True)
cos_halos.load_abskin()
# Plot
mtl_kin = cos_halos.abs_kin('Metal')
gd = np.where(mtl_kin['flg'] > 0)
xdb.xplot(cos_halos.NHI[gd], mtl_kin['Dv'][gd], scatter=True)
HI_kin = cos_halos.abs_kin('HI')
gd = np.where(HI_kin['flg'] > 0)
xdb.xplot(cos_halos.NHI[gd], HI_kin['Dv'][gd], scatter=True)
print('All done')
def generate_stau(velo, flux, sig, kbin=22. * u.km / u.s, debug=False):
""" Generate the smoothed tau array for kinematic tests
Parameters
----------
velo : Quantity array (usually km/s)
flux : Quantity array (flux)
sig : Quantity array (sig)
kbin : Quantity (velocity), optional
Kernel size for Gaussian smoothing of optical depth array
debug : bool, optional
Returns
-------
stau : array
Smoothed tau array
"""
# Velocity array
npix = len(velo)
pix = np.arange(npix).astype(int)
# Calculate dv
dv = np.abs(np.median(velo - np.roll(velo, 1)))
# Test for bad pixels
badzero = np.where((flux == 0) | (sig <= 0))[0]
if len(badzero) > 0:
if np.max(badzero) - np.min(badzero) >= 5:
raise ValueError(
'orig_kin: too many or too large sections of bad data')
flux[badzero] = np.mean(
np.array([flux[np.min(badzero) - 1], flux[np.max(badzero) + 1]]))
pdb.set_trace() # Should add sig too
# Generate the tau array
tau = np.zeros(npix)
gd = np.where((flux > sig / 2.) & (sig > 0.))
if len(gd) == 0:
raise ValueError('orig_kin: Profile too saturated.')
tau[gd] = np.log(1. / flux[gd])
sat = (pix == pix)
sat[gd] = False
tau[sat] = np.log(2. / sig[sat])
# Smooth
nbin = (np.round(kbin / dv)).value
try:
kernel = Box1DKernel(nbin, mode='center')
except:
pdb.set_trace()
stau = convolve(tau, kernel, boundary='fill', fill_value=0.)
if debug is True:
try:
from xastropy.xutils import xdebug as xdb
except ImportError:
pdb.set_trace()
else:
xdb.xplot(velo, tau, stau)
xdb.set_trace()
# Return
return stau
# Plot with Data
if (flg_test % 2**3) >= 2**2:
fN_model = xifm.default_model()
fN_data.tst_fn_data(fN_model=fN_model)
# Check l(X)
if (flg_test % 16) >= 8:
fN_model = xifm.default_model()
lX = fN_model.calc_lox(2.4, 17.19+np.log10(2.), 23.)
print('l(X) = %g' % lX)
# Check teff_LL
if (flg_test % 32) >= 16:
fN_model = xifm.default_model()
zval,teff_LL = fN_model.teff_ll(0.5, 2.45)
xdb.xplot(zval,teff_LL)#,xlabel='z', ylabel=r'$\tau_{\rm LL}$')
# Check MFP
if (flg_test % 64) >= 32:
fN_model = xifm.default_model()
z = 2.44
mfp = fN_model.mfp(z)
print('MFP at z={:g} is {:g} Mpc'.format(z,mfp.value))
# Check Inoue+14
if (flg_test % 2**7) >= 2**6:
print('Testing Akio Model')
fN_model = fN_Model('Gamma')
NHI = [12.,14.,17.,21.]
z = 2.5
dXdz = igmu.cosm_xz(z, flg=1)
def ew_teff_lyman(
ilambda,
zem,
fN_model,
NHI_MIN=11.5,
NHI_MAX=22.0,
N_eval=5000,
EW_spline=None,
bval=24.0,
fNz=False,
cosmo=None,
debug=False,
cumul=None,
verbose=False,
):
""" tau effective (follows ew_teff_lyman.pro from XIDL)
teff = ew_teff_lyman(3400., 2.4)
Parameters:
-------------
ilambda: float
Observed wavelength
zem: float
Emission redshift of the source [sets which Lyman lines are included]
bva: float
-- Characteristics Doppler parameter for the Lya forest
-- [Options: 24, 35 km/s]
NHI_MIN: float
-- Minimum log HI column for integration [default = 11.5]
NHI_MAX: float
-- Maximum log HI column for integration [default = 22.0]
fNz: Boolean (False)
-- Inputs f(N,z) instead of f(N,X)
cosmo: astropy.cosmology (None)
-- Cosmological model to adopt (as needed)
cumul: List of cumulative sums
-- Recorded only if cumul is not None
Returns:
teff:
Total effective opacity of all lines contributing
ToDo:
1. Parallelize the Lyman loop
JXP 07 Nov 2014
"""
# Lambda
if not isinstance(ilambda, float):
raise ValueError("igm.tau_eff: ilambda must be a float for now")
Lambda = ilambda
if not isinstance(Lambda, u.quantity.Quantity):
Lambda = Lambda * u.AA # Ang
# Read in EW spline (if needed)
if EW_spline == None:
if int(bval) == 24:
EW_FIL = xa_path + "/igm/EW_SPLINE_b24.p"
elif int(bval) == 35:
EW_FIL = os.environ.get("XIDL_DIR") + "/IGM/EW_SPLINE_b35.fits"
else:
raise ValueError("igm.tau_eff: Not ready for this bvalue %g" % bval)
EW_spline = pickle.load(open(EW_FIL, "rb"))
# Lines
wrest = tau_eff_llist()
# Find the lines
gd_Lyman = wrest[(Lambda / (1 + zem)) < wrest]
nlyman = len(gd_Lyman)
if nlyman == 0:
if verbose:
print("igm.tau_eff: No Lyman lines covered at this wavelength")
return 0
# N_HI grid
lgNval = NHI_MIN + (NHI_MAX - NHI_MIN) * np.arange(N_eval) / (N_eval - 1) # Base 10
dlgN = lgNval[1] - lgNval[0]
Nval = 10.0 ** lgNval
teff_lyman = np.zeros(nlyman)
# For cumulative
if not cumul is None:
cumul.append(lgNval)
# Loop on the lines
for qq, line in enumerate(gd_Lyman): # Would be great to do this in parallel...
# (Can pack together and should)
# Redshift
zeval = ((Lambda / line) - 1).value
if zeval < 0.0:
teff_lyman[qq] = 0.0
continue
# Cosmology
if fNz is False:
if cosmo not in locals():
cosmo = FlatLambdaCDM(H0=70, Om0=0.3) # Vanilla
# dxdz = (np.fabs(xigmu.cosm_xz(zeval-0.1, cosmo=cosmo)-
# xigmu.cosm_xz(zeval+0.1,cosmo=cosmo)) / 0.2 )
# xdb.set_trace()
dxdz = xigmu.cosm_xz(zeval, cosmo=cosmo, flg=1)
else:
dxdz = 1.0 # Code is using f(N,z)
# print('dxdz = %g' % dxdz)
# Get EW values (could pack these all together)
idx = np.where(EW_spline["wrest"] == line)[0]
if len(idx) != 1:
raise ValueError("tau_eff: Line %g not included or over included?!" % line)
restEW = interpolate.splev(lgNval, EW_spline["tck"][idx], der=0)
# dz
dz = ((restEW * u.AA) * (1 + zeval) / line).value
# Evaluate f(N,X) at zeval
log_fnX = fN_model.eval(lgNval, zeval).flatten()
# xdb.set_trace()
# Sum
intgrnd = 10.0 ** (log_fnX) * dxdz * dz * Nval
teff_lyman[qq] = np.sum(intgrnd) * dlgN * np.log(10.0)
if not cumul is None:
cumul.append(np.cumsum(intgrnd) * dlgN * np.log(10.0))
# xdb.set_trace()
# Debug
if debug == True:
xdb.xplot(lgNval, np.log10(10.0 ** (log_fnX) * dxdz * dz * Nval))
# x_splot, lgNval, total(10.d^(log_fnX) * dxdz * dz * Nval,/cumul) * dlgN * alog(10.) / teff_lyman[qq], /bloc
# printcol, lgnval, log_fnx, dz, alog10(10.d^(log_fnX) * dxdz * dz * Nval)
# writecol, 'debug_file'+strtrim(qq,2)+'.dat', lgNval, restEW, log_fnX
xdb.set_trace()
# xdb.set_trace()
return np.sum(teff_lyman)
# Write
myspec.write_to_fits('tmp.fits')
if (flg_test % 2**2) >= 2**1:
# Now 2D
fil = '/Users/xavier/Dropbox/QSOPairs/data/LRIS_redux/SDSSJ231254.65-025403.1_b400_F.fits.gz'
myspec = xsr.readspec(fil)
myspec.write_to_fits('tmp.fits')
if (flg_test % 2**3) >= 2**2: # Boxcar
fil = '~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits'
myspec = xsr.readspec(fil)
newspec = myspec.box_smooth(3)
#
newspec2 = myspec.box_smooth(3, preserve=True)
xdb.xplot(myspec.dispersion, myspec.flux, newspec2.flux)
if (flg_test % 2**4) >= 2**3: # Rebin array
fil = '~/PROGETTI/LLSZ3/data/normalize/UM669_nF.fits'
myspec = xsr.readspec(fil)
new_wv = np.arange(3000., 9000., 5) * u.AA
newspec = myspec.rebin(new_wv)
#xdb.xplot(myspec.dispersion, myspec.flux,
# xtwo=new_wv, ytwo=newspec.flux)
# Test EW
wvmnx = np.array((4859., 4961.)) * u.AA
gd1 = np.where((myspec.dispersion > wvmnx[0])
& (myspec.dispersion < wvmnx[1]))[0]
dwv1 = myspec.dispersion - np.roll(myspec.dispersion, 1)
EW1 = np.sum(dwv1[gd1] * (1. - myspec.flux[gd1].value))
