def test_rebin_to_rest(specmr):
zarr = np.array([2.1, 2.2])
# Build spectra array
rest_spec = ltsu.rebin_to_rest(specmr, zarr, 100 * u.km / u.s, debug=False)
# Test
assert rest_spec.totpix == 3716
np.testing.assert_allclose(rest_spec.wvmin.value, 986.3506403, rtol=1e-5)
def test_rebin_to_rest(specmr):
zarr = np.array([2.1,2.2])
# Build spectra array
rest_spec = ltsu.rebin_to_rest(specmr, zarr, 100*u.km/u.s, debug=False)
# Test
assert rest_spec.totpix == 3716
np.testing.assert_allclose(rest_spec.wvmin.value, 986.3506403, rtol=1e-5)
def test_smash_spectra(specmr):
# Try to stack 2 spectra with different wavelengths
with pytest.raises(AssertionError):
stack = ltsu.smash_spectra(specmr)
# Stack rebinned
zarr = np.array([2.1, 2.2])
rest_spec = ltsu.rebin_to_rest(specmr, zarr, 100 * u.km / u.s, debug=False)
stack = ltsu.smash_spectra(rest_spec, method='average')
# Test
assert stack.totpix == 3716
np.testing.assert_allclose(stack.flux[1].value, -3.32135105133, rtol=1e-5)
def test_smash_spectra(specmr):
# Try to stack 2 spectra with different wavelengths
with pytest.raises(AssertionError):
stack = ltsu.smash_spectra(specmr)
# Stack rebinned
zarr = np.array([2.1,2.2])
rest_spec = ltsu.rebin_to_rest(specmr, zarr, 100*u.km/u.s, debug=False)
stack = ltsu.smash_spectra(rest_spec, method='average')
# Test
assert stack.totpix == 3716
np.testing.assert_allclose(stack.flux[1].value, -3.32135105133, rtol=1e-5)
def clean(spec, red):
"""
:param spec: (numpy array) XSpectrum1D objects
:param red: (numpy array) their redshift values (z)
:returns:
rest_spec: (numpy array) truncatted, normalized and restframed XSpectrum1D objects
"""
import numpy as np
import astropy.units as u
from linetools.spectra import utils as ltsu
from linetools.spectra.xspectrum1d import XSpectrum1D
r = range(len(spec))
temp = [np.asarray(spec[i].wavelength / (1 + red[i])) for i in r]
# truncating each spectra to only include wavelength values from 1000 to 1450
wv_coverage = [(1000 < entry) & (entry < 1450) for entry in temp]
wave = np.asarray([spec[i].wavelength[wv_coverage[i]] for i in r])
flux = np.asarray([spec[i].flux[wv_coverage[i]] for i in r])
error = np.asarray([spec[i].sig[wv_coverage[i]] for i in r])
temp = [np.asarray(wave[i] / (1 + red[i])) for i in r]
# normalizing just between the SiII lines
wv_norm = [(1260 < entry) & (entry < 1304)
for entry in temp] # just between the SiII lines
flux_range = np.asarray([flux[i][wv_norm[i]] for i in r])
medians = np.asarray([np.median(flux_range[i]) for i in r])
norm_flux = np.asarray([(flux[i] / medians[i]) for i in r])
# getting a single XSpec object to feed into the stack
spec = [XSpectrum1D(wave[i], norm_flux[i], error[i]) for i in r]
collate = ltsu.collate(spec)
rest_spec = ltsu.rebin_to_rest(collate,
red,
300 * u.km / u.s,
grow_bad_sig=True)
return rest_spec
def stack_spec(spec_file, dv=100 * u.km / u.s, cut_on_rho=4.):
# Load
xspec, tpe, spec_tbl = load_spec(spec_file)
# Cut on separation (should do this much earlier in the process)
if cut_on_rho is not None:
warnings.warn("Cutting on rho in stack. Should do this earlier")
b_coords = SkyCoord(ra=tpe['BG_RA'], dec=tpe['BG_DEC'], unit='deg')
f_coords = SkyCoord(ra=tpe['FG_RA'], dec=tpe['FG_DEC'], unit='deg')
kpc_amin = cosmo.kpc_comoving_per_arcmin(tpe['FG_Z']) # kpc per arcmin
ang_seps = b_coords.separation(f_coords)
rho = ang_seps.to('arcmin') * kpc_amin / (1 + tpe['FG_Z'])
cut_rho = rho.to('Mpc').value < cut_on_rho
print("We have {:d} spectra after the cut.".format(np.sum(cut_rho)))
xspec = xspec[cut_rho]
tpe = tpe[cut_rho]
spec_tbl = spec_tbl[cut_rho]
# Remove those without continua
has_co = spec_tbl['HAS_CO'].data
co_spec = xspec[has_co]
co_spec.normed = True # Apply continuum
# May also wish to isolate in wavelength to avoid rejected pixels
for ii in range(co_spec.nspec):
co_spec.select = ii
co = co_spec.co.value
sig = co_spec.sig.value
bad_pix = np.any([(co == 0.), (co == 1.), (sig <= 0.)], axis=0)
co_spec.add_to_mask(bad_pix, compressed=True)
# Rebin to rest
zarr = tpe['FG_Z'][has_co]
rebin_spec = lspu.rebin_to_rest(co_spec, zarr, dv)
# Stack
stack = lspu.smash_spectra(rebin_spec)
# Plot
tpep.plot_stack(stack, 'all_stack.pdf')
tpep.plot_spec_img(rebin_spec, 'spec_img.pdf')
pdb.set_trace()
# Return
return
def reading_data(path, resol, zlow, zhigh):
'''
function for requesting the spectra from desi mocks
:param resol float: dleta v for rebin
:param zlow float: lower limit for request redshift
:param zhigh float: higher limit for request redshift
:return: two class object: 1.meta data and unbinned spectra 2. xspectrum object
'''
#define the variable
dataz = []
z = []
zerr = []
fluxt = []
wavetotal = []
ivar = []
ra = []
dec = []
id = []
filename = []
wvmin = []
wvmax = []
npix = []
date = []
#loop through the whole directory for requesting spectra
specf = []
mag = []
item1 = os.listdir(str(path))
for k in item1:
item = os.listdir(str(path) + str(k))
# print (item)
for j in item:
if os.listdir(str(path) + str(k) + '/' + str(j) + '/'):
dataz = fits.open(
str(path) + str(k) + '/' + str(j) + '/zbest-16-' + str(j) +
'.fits')
data = fits.open(
str(path) + str(k) + '/' + str(j) + '/spectra-16-' +
str(j) + '.fits') # k j j
zcut = (zlow < np.array(dataz[1].data['Z'])) & (np.array(
dataz[1].data['Z']) < zhigh)
wavecut = (np.array(data[2].data) < 5730)
wavecut2 = ((np.array(data[7].data) > 5730) &
(np.array(data[7].data) < 7560))
wavecut3 = (np.array(data[12].data) > 7560)
z.append(dataz[1].data['Z'][zcut])
zerr.append(dataz[1].data['ZERR'][zcut])
ra.append(data[1].data['TARGET_RA'][zcut])
dec.append(data[1].data['TARGET_DEC'][zcut])
id.append(data[1].data['TARGETID'][zcut])
filename.append(
np.tile([k, j, j], (len(dataz[1].data['Z'][zcut]), 1)))
mag.append(1.0)
date.append(data[1].data['NIGHT'][zcut])
#combining the spectra from three channels
wavetotal.append(
np.repeat([
np.concatenate([
data[2].data[wavecut], data[7].data[wavecut2],
data[12].data[wavecut3]
])
],
len(dataz[1].data['Z']),
axis=0)[zcut])
fluxt.append(
np.concatenate(
(data[3].data[:, wavecut], data[8].data[:, wavecut2],
data[13].data[:, wavecut3]),
axis=1)[zcut])
ivar.append(
np.concatenate(
(data[4].data[:, wavecut], data[9].data[:, wavecut2],
data[14].data[:, wavecut3]),
axis=1)[zcut])
sp = XSpectrum1D(np.vstack(np.array(wavetotal)),
np.vstack(np.array(fluxt)),
np.sqrt(1 / np.vstack(np.array(ivar))),
verbose=False)
#ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr
binspec = ltsu.rebin_to_rest(sp,
np.zeros(len(np.array(np.concatenate(z)))),
resol * u.km / u.s)
specf = spec(np.concatenate(ra), np.concatenate(dec), np.concatenate(id),
np.concatenate(filename), np.min(binspec.wavelength),
np.max(binspec.wavelength), len(binspec.wavelength),
np.concatenate(date), np.array(mag), np.concatenate(z),
np.concatenate(zerr))
# newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso)
return specf, binspec
def miss_id_check(path, idt, resol):
'''
Function for requesting spectra and meta for undetected DLAs
:param path object: path for the spectra
:param idt float: undetected id
:param resol: resolution for spectr
:return: two class object: 1. meta for spectra 2. spectra for undetected DLAs
'''
# define the variable
dataz = []
z = []
zerr = []
fluxt = []
wavetotal = []
ivar = []
mockid = []
ra = []
dec = []
mockid = []
filename = []
wvmin = []
wvmax = []
npix = []
date = []
# loop through the whole directory for requesting spectra
specf = []
mag = []
item1 = os.listdir(str(path))
for k in item1:
item = os.listdir(str(path) + str(k))
# print (item)
for j in item:
if os.listdir(str(path) + str(k) + '/' + str(j) + '/'):
dataz = fits.open(
str(path) + str(k) + '/' + str(j) + '/truth-16-' + str(
j) + '.fits')
data = fits.open(str(path) + str(k) + '/' + str(
j) + '/spectra-16-' + str(j) + '.fits') # k j j
indexcut = np.isin(data[1].data['TARGETID'], idt)
wavecut = (np.array(data[2].data) < 5730)
wavecut2 = ((np.array(data[7].data) > 5730) & (np.array(data[7].data) < 7560))
wavecut3 = (np.array(data[12].data) > 7560)
z.append(dataz[1].data['Z'][indexcut])
zerr.append(dataz[1].data['TRUEZ'][indexcut])
ra.append(data[1].data['TARGET_RA'][indexcut])
dec.append(data[1].data['TARGET_DEC'][indexcut])
mockid.append(data[1].data['TARGETID'][indexcut])
# combining the spectra from three channels
wavetotal.append(np.repeat(
[np.concatenate([data[2].data[wavecut], data[7].data[wavecut2], data[12].data[wavecut3]])],
len(dataz[1].data['Z']), axis=0)[indexcut])
fluxt.append(
np.concatenate((data[3].data[:, wavecut], data[8].data[:, wavecut2], data[13].data[:, wavecut3]),
axis=1)[indexcut])
ivar.append(
np.concatenate((data[4].data[:, wavecut], data[9].data[:, wavecut2], data[14].data[:, wavecut3]),
axis=1)[indexcut])
sp = XSpectrum1D(np.vstack(np.array(wavetotal)), np.vstack(np.array(fluxt)), np.sqrt(1 / np.vstack(np.array(ivar))),
verbose=False)
# ra,dec,id,filename,wvmin,wvmax,npix, date, mag,z, zerr
binspec = ltsu.rebin_to_rest(sp, np.zeros(len(np.array(np.concatenate(z)))), resol * u.km / u.s)
specf = SPEC.spec_meta(np.concatenate(ra), np.concatenate(dec), np.concatenate(mockid),
np.min(binspec.wavelength),
np.max(binspec.wavelength), len(binspec.wavelength),
np.concatenate(z), np.concatenate(zerr))
# newwave = np.linspace(np.min(wavetotal[0]),np.max(wavetotal[0]),reso)
return specf, binspec
def tpe_stack_lris(dv=100 * u.km / u.s):
""" Testing stacks with LRIS
"""
# Load sample
ipos = this_file.rfind('/')
if ipos == -1:
path = './'
else:
path = this_file[0:ipos]
tpe = Table.read(path + '/../TPE_DR12_31.2_spec.fits')
# Load spectra
# Coordiantes
b_coords = SkyCoord(ra=tpe['BG_RA'], dec=tpe['BG_DEC'], unit='deg')
f_coords = SkyCoord(ra=tpe['FG_RA'], dec=tpe['FG_DEC'], unit='deg')
# Cut on impact parameter and BOSS
kpc_amin = cosmo.kpc_comoving_per_arcmin(tpe['FG_Z']) # kpc per arcmin
ang_seps = b_coords.separation(f_coords)
rho = ang_seps.to('arcmin') * kpc_amin / (1 + tpe['FG_Z'])
cut_Rlris = (rho.to('Mpc').value < 4) & (tpe['BG_LYA_INSTRUMENT'] == 'LRIS'
) # & (
#tpe['FG_Z'] > 2.) # Some of these have too low z (just barely)
# Cut
gd_b_coords = b_coords[cut_Rlris]
gd_tpe = tpe[cut_Rlris]
# Grab these spectra from QPQ
# For boss, we are ok taking the first entry of each
# The returned set is aligned with the input coords
qpq = IgmSpec(db_file=qpq_file, skip_test=True)
IDs = qpq.qcat.match_coord(gd_b_coords, group='LRIS')
meta = qpq['LRIS'].meta
gcut = meta['GRATING'] == '1200/3400' # There is one with B400
B1200 = np.in1d(IDs, meta['PRIV_ID'][gcut])
print("There are {:d} sources without B1200".format(np.sum(~B1200)))
# Cut again
gd_b_coords = gd_b_coords[B1200]
gd_tpe = gd_tpe[B1200]
gd_IDs = IDs[B1200]
# Find the rows
idx = cat_utils.match_ids(gd_IDs, meta['PRIV_ID'])
rows = meta['GROUP_ID'][idx]
pdb.set_trace()
spec, meta = qpq.coords_to_spectra(gd_b_coords, 'LRIS', all_spec=False)
# Check for continua
has_co = np.array([True] * spec.nspec)
for ii in range(spec.nspec):
# Select
spec.select = ii
# Match to lya
lya = (1 + gd_tpe['FG_Z'][ii]) * 1215.67 * u.AA
iwave = np.argmin(np.abs(spec.wavelength - lya))
# Check for co
#coval = spec.co[iwave]
#print('spec: {:d} with co={:g}'.format(ii, coval))
if np.isclose(spec.co[iwave], 0.) or np.isclose(spec.co[iwave], 1.):
has_co[ii] = False
# Slice to good co
print("{:d} BOSS spectra with a continuum".format(np.sum(has_co)))
co_spec = spec[has_co]
co_spec.normed = True # Apply continuum
# NEED TO ZERO OUT REGIONS WITHOUT CONTINUUM
# May also wish to isolate in wavelength to avoid rejected pixels
for ii in range(co_spec.nspec):
co_spec.select = ii
co = co_spec.co.value
bad_pix = np.any([(co == 0.), (co == 1.)], axis=0)
co_spec.add_to_mask(bad_pix, compressed=True)
# Rebin to rest
zarr = gd_tpe['FG_Z'][has_co]
rebin_spec = lspu.rebin_to_rest(co_spec, zarr, dv)
# Stack
stack = lspu.smash_spectra(rebin_spec)
# Plot
plot_stack(stack, 'LRIS_stack.pdf')
return stack
def tpe_stack_boss(dv=100 * u.km / u.s):
""" Testing stacks with BOSS
"""
# Load sample
ipos = this_file.rfind('/')
if ipos == -1:
path = './'
else:
path = this_file[0:ipos]
tpe = Table.read(path + '/../TPE_DR12_31.2_spec.fits')
# Load spectra
igmsp = IgmSpec()
# Coordiantes
b_coords = SkyCoord(ra=tpe['BG_RA'], dec=tpe['BG_DEC'], unit='deg')
f_coords = SkyCoord(ra=tpe['FG_RA'], dec=tpe['FG_DEC'], unit='deg')
# Cut on impact parameter and BOSS
kpc_amin = cosmo.kpc_comoving_per_arcmin(tpe['FG_Z']) # kpc per arcmin
ang_seps = b_coords.separation(f_coords)
rho = ang_seps.to('arcmin') * kpc_amin / (1 + tpe['FG_Z'])
cut_Rboss = (rho.to('Mpc').value < 4) & (
tpe['BG_LYA_INSTRUMENT'] == 'BOSS') & (
tpe['FG_Z'] > 2.) # Some of these have too low z (just barely)
# Cut
gd_b_coords = b_coords[cut_Rboss]
gd_f_coords = f_coords[cut_Rboss]
gd_tpe = tpe[cut_Rboss]
# Grab these spectra from igmsp
# For boss, we are ok taking the first entry of each
# The returned set is aligned with the input coords
spec, meta = igmsp.coords_to_spectra(gd_b_coords,
'BOSS_DR12',
all_spec=False)
# Check for continua
has_co = np.array([True] * spec.nspec)
for ii in range(spec.nspec):
# Select
spec.select = ii
# Match to lya
lya = (1 + gd_tpe['FG_Z'][ii]) * 1215.67 * u.AA
iwave = np.argmin(np.abs(spec.wavelength - lya))
# Check for co
#coval = spec.co[iwave]
#print('spec: {:d} with co={:g}'.format(ii, coval))
if np.isclose(spec.co[iwave], 0.) or np.isclose(spec.co[iwave], 1.):
has_co[ii] = False
# Slice to good co
print("{:d} BOSS spectra with a continuum".format(np.sum(has_co)))
co_spec = spec[has_co]
co_spec.normed = True # Apply continuum
# NEED TO ZERO OUT REGIONS WITHOUT CONTINUUM
# May also wish to isolate in wavelength to avoid rejected pixels
for ii in range(co_spec.nspec):
co_spec.select = ii
co = co_spec.co.value
bad_pix = np.any([(co == 0.), (co == 1.)], axis=0)
co_spec.add_to_mask(bad_pix, compressed=True)
# Rebin to rest
zarr = gd_tpe['FG_Z'][has_co]
rebin_spec = lspu.rebin_to_rest(co_spec, zarr, dv)
# Check 2D
check_td = True
if check_td:
fx = rebin_spec.data['flux']
sig = rebin_spec.data['sig']
gds = sig > 0.
fx[~gds] = 0.
xdb.set_trace() # xdb.ximshow(fx)
# Stack
stack = lspu.smash_spectra(rebin_spec)
# Plot
plot_stack(stack, 'BOSS_stack.pdf')
print('Wrote')
return stack