def __next__(self):
filename = None
filename2 = None
# this sets up the first call
if self.tile is None:
(self.tile,self.date)= self.it0.__next__()
self.it1 = TileDate_PreDate_Iterator(self.tile, self.date, subdir=self.subdir) # TileDate_PreDate_Iterator
while self.pdate is None:
try:
self.pdate = self.it1.__next__()
except StopIteration:
try:
(self.tile,self.date)= self.it0.__next__()
except StopIteration:
raise StopIteration
self.it1 = TileDate_PreDate_Iterator(self.tile, self.date, subdir=self.subdir)
self.it2 = fs_utils.panels.flat # np.nditer(fs_utils.panels)
while filename is None or filename2 is None:
# all calls go through this
try:
self.panel = self.it2.__next__()
except StopIteration:
try:
self.pdate=self.it1.__next__()
self.it2 = fs_utils.panels.flat
self.panel=self.it2.__next__()
except StopIteration:
self.pdate = None
while self.pdate is None:
(self.tile,self.date)= self.it0.__next__()
self.it1 = TileDate_PreDate_Iterator(self.tile, self.date, subdir=self.subdir)
try:
self.pdate = self.it1.__next__()
except StopIteration:
self.pdate = None
self.it2 = fs_utils.panels.flat
self.panel=self.it2.__next__()
filename = fs_utils.fitsfile(self.tile, self.date, self.panel, subdir=self.subdir,trunk=self.trunk)
# filename2 = fs_utils.fitsfile(self.tile, self.pdate, self.panel, subdir='recent',trunk=self.trunk)
filename2 = fs_utils.fitsfile(self.tile, self.pdate, self.panel, subdir=self.subdir,trunk=self.trunk)
if self.verbose:
print("Iterator: Tile {}, Panel {}, Date {}, Date 2 {}".format(self.tile, self.panel,self.date,self.pdate))
log.debug("{} {}".format(filename,filename2))
return((read_spectra(filename) , read_spectra(filename2)), ({'tile': self.tile, 'date': self.date, 'panel': self.panel},{'tile': self.tile, 'date': self.pdate, 'panel': self.panel}))
def prospector_LGAL_desiSpec_i(galid, mask=False, infer_method='dynesty'):
''' run prospector on L-Gal DESI-like spectra
'''
# read in source spectra (only to get the redshift)
f_name = 'gal_spectrum_'+str(galid)+'_BGS_template_BC03_Stelib.fits'
f_inspec = fits.open(''.join([UT.dat_dir(), 'Lgal/templates/', f_name]))
zred = f_inspec[0].header['REDSHIFT']
# read desi-like spectra
f_name = 'gal_spectrum_'+str(galid)+'_BGS_template_BC03_Stelib.fits'
f_outspec = ''.join([UT.dat_dir(), 'Lgal/spectra/', 'desi_out_', f_name])
spec_desi = desiIO.read_spectra(f_outspec)
wave = np.concatenate([spec_desi.wave[b] for b in ['b', 'r', 'z']])
flux = np.concatenate([spec_desi.flux[b][0] for b in ['b', 'r', 'z']]) # 10-17 ergs/s/cm2/AA
flux_unc = np.concatenate([spec_desi.ivar[b][0]**-0.5 for b in ['b', 'r', 'z']])
prosp = Fitters.Prospector()
str_mask = ''
if mask: str_mask = '.masked'
f_name = ''.join([UT.dat_dir(), 'Lgal/spectra/',
'prospector.', infer_method, str_mask,
'.desi_out_gal_spectrum_', str(galid), '_BGS_template_BC03_Stelib.h5'])
if infer_method == 'dynesty': # run dynamic nested sampling
prosp.dynesty_spec(wave, flux, flux_unc, zred, mask=mask,
nested=True, maxcall_init=25000, maxcall=50000,
write=True, output_file=f_name)
elif infer_method == 'emcee': # emcee
prosp.emcee_spec(wave, flux, flux_unc, zred, mask=mask,
write=True, output_file=f_name, silent=False)
return None
def __next__(self):
while True:
if self.tile is None:
self.tile = self.date_tile_iterator.__next__()
try:
self.panel = self.panel_iterator.__next__()
fname = fs_utils.fitsfile(self.tile,
self.date,
self.panel,
subdir=self.subdir,
trunk=self.trunk)
if fname is not None:
break
except StopIteration:
if self.date_tile_iterator.hasNext():
self.tile = self.date_tile_iterator.__next__()
self.panel_iterator = fs_utils.panels.flat #Panel_Iterator()
self.panel = self.panel_iterator.__next__()
fname = fs_utils.fitsfile(self.tile,
self.date,
self.panel,
subdir=self.subdir,
trunk=self.trunk)
if fname is not None:
break
else:
raise StopIteration
if self.verbose:
print("Filename {}".format(fname))
return read_spectra(fname), fname
def get_spectra(spectra_name, targetid):
spectra = read_spectra(spectra_name)
spectra = spectra.select(targets=[targetid])
if 'brz' not in spectra.bands:
spectra = coadd_cameras(spectra)
return spectra.wave['brz'], spectra.flux['brz'].T, spectra.ivar['brz'].T
def __next__(self):
filename = None
filename2 = None
#handle the first case
if self.it0 is None:
try:
self.it0 = Date_Tile_Iterator(self.date, subdir=self.subdir)
self.tile = self.it0.__next__()
self.it1 = TileDate_PreDate_Iterator(self.tile, self.date, subdir=self.subdir)
self.pdate = self.it1.__next__()
self.it2 = fs_utils.panels.flat
self.panel=self.it2.__next__()
filename = fs_utils.fitsfile(self.tile, self.date, self.panel, subdir=self.subdir,trunk=self.trunk)
filename2 = fs_utils.fitsfile(self.tile, self.pdate, self.panel, subdir=self.subdir,trunk=self.trunk)
except StopIteration:
raise StopIteration
while filename is None or filename2 is None:
try:
self.panel = self.it2.__next__()
except StopIteration:
try:
self.pdate=self.it1.__next__()
self.it2 = fs_utils.panels.flat
self.panel=self.it2.__next__()
except StopIteration:
try:
self.tile = self.it0.__next__()
self.it1 = TileDate_PreDate_Iterator(self.tile, self.date, subdir=self.subdir)
self.pdate=self.it1.__next__()
self.it2 = fs_utils.panels.flat
self.panel=self.it2.__next__()
except StopIteration:
raise StopIteration
filename = fs_utils.fitsfile(self.tile, self.date, self.panel, subdir=self.subdir,trunk=self.trunk)
filename2 = fs_utils.fitsfile(self.tile, self.pdate, self.panel, subdir=self.subdir,trunk=self.trunk)
if self.verbose:
print("Iterator: Tile {}, Panel {}, Date {}, Date 2 {}".format(self.tile, self.panel,self.date,self.pdate))
return (read_spectra(filename) , read_spectra(filename2))
def firefly_LGAL_desiSpec_i(galid,
model='m11',
model_lib='MILES',
imf='cha',
hpf_mode='on'):
''' run firelfy on L-Gal DESI-like spectra
'''
t0 = time.time()
# read in source spectra (only to get the redshift)
f_name = 'gal_spectrum_' + str(galid) + '_BGS_template_BC03_Stelib.fits'
f_inspec = fits.open(''.join([UT.dat_dir(), 'Lgal/templates/', f_name]))
redshift = f_inspec[0].header['REDSHIFT']
# read desi-like spectra
f_name = 'gal_spectrum_' + str(galid) + '_BGS_template_BC03_Stelib.fits'
f_outspec = ''.join([UT.dat_dir(), 'Lgal/spectra/', 'desi_out_', f_name])
spec_desi = desiIO.read_spectra(f_outspec)
gspec = Spec.GSfirefly()
gspec.DESIlike(spec_desi, redshift=redshift)
gspec.path_to_spectrum = UT.dat_dir()
# output firefly file
f_out = ''.join([
'firefly.', model, '.', model_lib, '.imf_', imf, '.dust_', hpf_mode,
'.', 'desi_out_',
f_name.split('.fits')[0], '.hdf5'
])
f_firefly = ''.join([UT.dat_dir(), 'Lgal/spectra/', f_out])
firefly = Fitters.Firefly(
gspec,
f_firefly, # output file
co.Planck13, # comsology
models=model, # model ('m11', 'bc03', 'm09')
model_libs=[model_lib], # model library for M11
imfs=[imf], # IMF used ('ss', 'kr', 'cha')
hpf_mode=
hpf_mode, # uses HPF to dereden the spectrum
age_limits=[0, 15],
Z_limits=[-3., 5.],
wave_limits=[3350., 9000.],
suffix=None,
downgrade_models=False,
data_wave_medium='vacuum',
use_downgraded_models=False,
write_results=True)
bestfit = firefly.fit_models_to_data()
print('galid %i took %f' % (galid, (time.time() - t0) / 60.))
return None
def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
print(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
print(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
log.info("reading spectra ...")
spectra = read_spectra(args.infile)
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
def firefly_lgal_bgsSpec(galid, iobs, lib='bc03', obs_sampling='spacefill',
model='m11', model_lib='MILES', imf='cha', hpf_mode='on'):
''' run firefly on simulated DESI BGS spectra from testGalIDs LGal
'''
obscond = obs_condition(sampling=obs_sampling)
if iobs >= obscond.shape[0]: raise ValueError
# read in source spectra
f_inspec = fits.open(f_Source(galid, lib=lib))
redshift = f_inspec[0].header['REDSHIFT']
# read in simulated DESI bgs spectra
f_bgsspec = f_BGSspec(galid, iobs, lib=lib, obs_sampling=obs_sampling, nobs=obscond.shape[0])
if not os.path.isfile(f_bgsspec): raise ValueError('spectra file does not exist')
spec_desi = read_spectra(f_bgsspec)
gspec = Spec.GSfirefly()
gspec.DESIlike(spec_desi, redshift=redshift)
gspec.path_to_spectrum = UT.dat_dir()
# output firefly file
f_bgs = f_BGSspec(galid, iobs, lib=lib, obs_sampling=obs_sampling, nobs=obscond.shape[0])
f_firefly = ''.join([UT.dat_dir(), 'spectral_challenge/bgs/',
'firefly.', model, '.', model_lib, '.imf_', imf, '.dust_', hpf_mode, '__',
f_bgs.rsplit('/', 1)[1].rsplit('.fits', 1)[0], '.hdf5'])
firefly = Fitters.Firefly(gspec,
f_firefly, # output file
co.Planck13, # comsology
models = model, # model ('m11', 'bc03', 'm09')
model_libs = [model_lib], # model library for M11
imfs = [imf], # IMF used ('ss', 'kr', 'cha')
hpf_mode = hpf_mode, # uses HPF to dereden the spectrum
age_limits = [0, np.log10(co.Planck13.age(redshift).value * 1e9)],
Z_limits = [0.001, 4.],
wave_limits = [3350., 9000.],
suffix=None,
downgrade_models = False,
data_wave_medium = 'vacuum',
use_downgraded_models = False,
write_results = True)
bestfit = firefly.fit_models_to_data()
return None
def lgal_bgsSpec(galid, iobs, lib='bc03', obs_sampling='spacefill', overwrite=False):
''' simulate DESI BGS spectra from testGalIDs LGal
'''
obscond = obs_condition(sampling=obs_sampling)
if iobs >= obscond.shape[0]:
raise ValueError
f_bgsspec = f_BGSspec(galid, iobs, lib=lib, obs_sampling=obs_sampling, nobs=obscond.shape[0])
if os.path.isfile(f_bgsspec) and not overwrite:
bgs_spectra = read_spectra(f_bgsspec)
else:
# get source spectra
spec_source = lgal_sourceSpectra(galid, lib=lib)
wavemin, wavemax = 3523.0, 9923.0
wave_source = np.arange(wavemin, wavemax, 0.2)
flux_source_interp = sp.interpolate.interp1d(
spec_source['wave'], spec_source['flux'], fill_value='extrapolate')
flux_source = flux_source_interp(wave_source)
# observing condition
airmass, seeing, exptime, _, _, _, _, _ = obscond[iobs,:]
# read in sky surface brightness
w_sky, skybrights = obs_SkyBrightness(sampling=obs_sampling)
skybright = skybrights[iobs,:]
u_surface_brightness = 1e-17 * u.erg / u.angstrom / u.arcsec**2 / u.cm**2 / u.second
# simulate BGS spectra
fdesi = FM.fakeDESIspec()
# BGS spectra output file
bgs_spectra = fdesi.simExposure(
wave_source,
np.atleast_2d(flux_source),
exptime=exptime,
airmass=airmass,
seeing=seeing,
skycondition={'name': 'input',
'sky': np.clip(skybright, 0, None) * u_surface_brightness,
'wave': w_sky},
filename=f_bgsspec)
return bgs_spectra
def Spectra(self, galid, type='source', lib='bc03'):
''' Read in spectra given galid
'''
# source spectra file with meta data
f_source = fits.open(self._Fspec(galid, 'source', lib))
# get meta data of spectra
hdr = f_source[0].header
meta = {}
for k in hdr.keys():
meta[k] = hdr[k]
spec = {}
if type == 'source':
# source spectra
specin = f_source[1].data
spec['wave'] = specin['wave']
spec['flux'] = specin['flux_nodust_nonoise'] * 1e20
spec['flux_unc'] = None
elif type == 'desibgs':
# desi-like spectra (in desiIO format)
import desispec.io as desiIO
f_desi = self._Fspec(
galid, type, lib
) #''.join([self._dir_lgal, 'spectra/', 'desi_out_', f_spec])
spec_desi = desiIO.read_spectra(f_desi)
spec['wave'] = np.concatenate(
[spec_desi.wave[b] for b in ['b', 'r', 'z']])
spec['flux'] = np.concatenate([
spec_desi.flux[b][0] for b in ['b', 'r', 'z']
]) # 10-17 ergs/s/cm2/AA
spec['flux_unc'] = np.concatenate(
[spec_desi.ivar[b][0]**-0.5 for b in ['b', 'r', 'z']])
return spec, meta
def prospector_lgal_bgsSpec(galid, iobs, lib='bc03', obs_sampling='spacefill', mask=False, infer_method='dynesty'):
''' run prospector on simulated DESI BGS spectra from testGalIDs LGal
'''
obscond = obs_condition(sampling=obs_sampling)
if iobs >= obscond.shape[0]: raise ValueError
# read in source spectra
f_inspec = fits.open(f_Source(galid, lib=lib))
redshift = f_inspec[0].header['REDSHIFT']
# read in simulated DESI bgs spectra
f_bgsspec = f_BGSspec(galid, iobs, lib=lib, obs_sampling=obs_sampling, nobs=obscond.shape[0])
if not os.path.isfile(f_bgsspec): raise ValueError('spectra file does not exist')
spec_desi = read_spectra(f_bgsspec)
wave = np.concatenate([spec_desi.wave[b] for b in ['b', 'r', 'z']])
flux = np.concatenate([spec_desi.flux[b][0] for b in ['b', 'r', 'z']]) # 10-17 ergs/s/cm2/AA
flux_unc = np.concatenate([spec_desi.ivar[b][0]**-0.5 for b in ['b', 'r', 'z']])
# output firefly file
str_mask = ''
if mask: str_mask = '.masked'
f_bgs = f_BGSspec(galid, iobs, lib=lib, obs_sampling=obs_sampling, nobs=obscond.shape[0])
f_prosp = ''.join([UT.dat_dir(), 'spectral_challenge/bgs/',
'prospector.', infer_method, str_mask, '__',
f_bgs.rsplit('/', 1)[1].rsplit('.fits', 1)[0], '.h5'])
prosp = Fitters.Prospector()
if infer_method == 'dynesty': # run dynamic nested sampling
prosp.dynesty_spec(wave, flux, flux_unc, redshift, mask=mask,
nested=True, maxcall_init=25000, maxcall=50000,
write=True, output_file=f_prosp)
elif infer_method == 'emcee': # emcee
prosp.emcee_spec(wave, flux, flux_unc, redshift, mask=mask,
write=True, output_file=f_prosp, silent=False)
return None
def get_rr_model(coadd_fn, index, redrock_fn=None, ith_bestfit=1, use_targetid=False, coadd_cameras=False, restframe=False, z=None, return_z=False):
'''
Return redrock model spectrum.
Args:
coadd_fn: str, path of coadd FITS file
index: int, index of coadd FITS file if use_targetid=False, or TARGETID if use_targetid=True
Options:
redrock_fn, str, path of redrock FITS file
use_targetid: bool, if True, index is TARGETID
coadd_cameras: bool, if True, the BRZ cameras are coadded together
restframe: bool, if True, return restframe spectrum in template wavelength grid; if False,
return spectrum in three cameras in observed frame
z: bool, if None, use redrock best-fit redshift
return_z: bool, if true, include redshift in output
'''
# If use_targetid=False, index is the index of coadd file; if True, index is TARGETID.
from desispec.interpolation import resample_flux
import redrock.templates
from desispec.io import read_spectra
templates = dict()
for filename in redrock.templates.find_templates():
tx = redrock.templates.Template(filename)
templates[(tx.template_type, tx.sub_type)] = tx
spec = read_spectra(coadd_fn)
if redrock_fn is None:
redrock_fn = coadd_fn.replace('/coadd-', '/redrock-')
redshifts = Table(fitsio.read(redrock_fn, ext='REDSHIFTS'))
if use_targetid:
tid = index
coadd_index = np.where(redshifts['TARGETID']==index)[0][0]
else:
tid = redshifts['TARGETID'][index]
coadd_index = index
if ith_bestfit==1:
spectype, subtype = redshifts['SPECTYPE'][coadd_index], redshifts['SUBTYPE'][coadd_index]
tx = templates[(spectype, subtype)]
coeff = redshifts['COEFF'][coadd_index][0:tx.nbasis]
if z is None:
z = redshifts['Z'][coadd_index]
else:
import h5py
rrdetails_fn = redrock_fn.replace('/redrock-', '/rrdetails-').replace('.fits', '.h5')
f = h5py.File(rrdetails_fn)
entry = f['zfit'][str(tid)]["zfit"]
spectype, subtype = entry['spectype'][ith_bestfit].decode("utf-8"), entry['subtype'][ith_bestfit].decode("utf-8")
tx = templates[(spectype, subtype)]
coeff = entry['coeff'][ith_bestfit][0:tx.nbasis]
if z is None:
z = entry['z'][ith_bestfit]
if restframe==False:
wave = dict()
model_flux = dict()
if coadd_cameras:
cameras = ['BRZ']
else:
cameras = ['B', 'R', 'Z']
for camera in cameras:
wave[camera] = spec.wave[camera.lower()]
model_flux[camera] = np.zeros(wave[camera].shape)
model = tx.flux.T.dot(coeff).T
mx = resample_flux(wave[camera], tx.wave*(1+z), model)
model_flux[camera] = spec.R[camera.lower()][coadd_index].dot(mx)
else:
wave = tx.wave
model_flux = tx.flux.T.dot(coeff).T
if return_z:
return wave, model_flux, z
else:
return wave, model_flux
def sim_source_spectra(allinfo, allzbest, infofile='source-truth.fits', debug=False):
"""Build the residual (source) spectra. No redshift-fitting.
"""
from desispec.io import read_spectra, write_spectra
from desispec.spectra import Spectra
from desispec.interpolation import resample_flux
from desispec.resolution import Resolution
from redrock.external.desi import DistTargetsDESI
from redrock.templates import find_templates, Template
assert(np.all(allinfo['TARGETID'] == allzbest['TARGETID']))
nsim = len(allinfo)
# Select the subset of objects for which we got the correct lens (BGS)
# redshift.
these = np.where((allzbest['SPECTYPE'] == 'GALAXY') *
(np.abs(allzbest['Z'] - allinfo['LENS_Z']) < 0.003))[0]
print('Selecting {}/{} lenses with the correct redshift'.format(len(these), nsim))
if len(these) == 0:
raise ValueError('No spectra passed the cuts!')
allinfo = allinfo[these]
allzbest = allzbest[these]
print('Writing {}'.format(infofile))
allinfo.write(infofile, overwrite=True)
tempfile = find_templates()[0]
rrtemp = Template(tempfile)
# loop on each chunk of lens+source spectra
nchunk = len(set(allinfo['CHUNK']))
for ichunk in set(allinfo['CHUNK']):
I = np.where(allinfo['CHUNK'] == ichunk)[0]
info = allinfo[I]
zbest = allzbest[I]
specfile = 'lenssource-spectra-chunk{:03d}.fits'.format(ichunk)
sourcefile = 'source-spectra-chunk{:03d}.fits'.format(ichunk)
spectra = read_spectra(specfile).select(targets=info['TARGETID'])
for igal, zz in enumerate(zbest):
zwave = rrtemp.wave * (1 + zz['Z'])
zflux = rrtemp.flux.T.dot(zz['COEFF']).T #/ (1 + zz['Z'])
if debug:
fig, ax = plt.subplots()
for band in spectra.bands:
R = Resolution(spectra.resolution_data[band][igal])
# use fastspecfit here
modelflux = R.dot(resample_flux(spectra.wave[band], zwave, zflux))
if debug:
ax.plot(spectra.wave[band], spectra.flux[band][igal, :])
ax.plot(spectra.wave[band], modelflux)
ax.set_ylim(np.median(spectra.flux['r'][igal, :]) + np.std(spectra.flux['r'][igal, :]) * np.array([-1.5, 3]))
#ax.set_xlim(4500, 5500)
spectra.flux[band][igal, :] -= modelflux # subtract
if debug:
qafile = 'source-spectra-chunk{:03d}-{}.png'.format(ichunk, igal)
fig.savefig(qafile)
plt.close()
print('Writing {} spectra to {}'.format(len(zbest), sourcefile))
write_spectra(outfile=sourcefile, spec=spectra)
return allinfo
def __next__(self):
filename = None
# the first
if self.it0 is None:
self.it0 = iter(self.list)
(self.tile,self.date)= self.it0.__next__()
if self.verbose:
print(self.tile,self.date)
self.it1 = fs_utils.panels.flat
# get the first file that exists
while filename is None:
try:
self.panel = self.it1.__next__()
filename = fs_utils.fitsfile(self.tile, self.date, self.panel, subdir=self.subdir,trunk=self.trunk)
except StopIteration:
(self.tile,self.date)= self.it0.__next__()
if self.verbose:
print(self.tile,self.date)
self.it1 = fs_utils.panels.flat
self.spectra = read_spectra(filename)
if self.verbose:
print('first', filename)
filename = None
self.it2 = Spectra_Subspectra_Iterator(self.spectra, verbose=False)
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(), verbose=False)
while True:
try:
ans=self.it3.__next__()
break
except StopIteration:
try:
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(), verbose=False)
except StopIteration: # it2 throws exception
while filename is None:
try:
self.panel = self.it1.__next__()
filename = fs_utils.fitsfile(self.tile, self.date, self.panel, subdir=self.subdir,trunk=self.trunk)
except StopIteration:
(self.tile,self.date)= self.it0.__next__()
if self.verbose:
print(self.tile,self.date)
self.it1 = fs_utils.panels.flat
# print(" In Stop iteration and got filename")
self.spectra = read_spectra(filename)
if self.verbose:
print('after', filename)
filename=None
self.it2 = Spectra_Subspectra_Iterator(self.spectra, verbose=False)
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(), verbose=False)
# print(" Made new iterators ")
# if self.verbose:
# print(self.tile, self.date)
return ans, (self.tile,self.date)
def sim_lenssource_spectra(BGSmags, fratios, seed=None, exptime=1000., nperchunk=500,
infofile='lenssource-truth.fits', debug=False):
"""Build the (noisy) lens+source spectra. No redshift-fitting.
"""
from astropy.io import fits
from desisim.templates import BGS, ELG
from desisim.scripts.quickspectra import sim_spectra
from desisim.io import read_basis_templates
from desispec.io import read_spectra
rand = np.random.RandomState(seed)
nsim = len(BGSmags)
assert(nsim == len(fratios))
if nperchunk > 500:
raise ValueError('nperchunk={} exceeds the maximum number allowed by redrock'.format(nperchunk))
nchunk = np.ceil(nsim / nperchunk).astype(int)
# [1] Build the noise-less lens (BGS) spectra.
# Read one healpix of the Buzzard mocks for redshift distribution.
mockfile = os.path.join(os.getenv('DESI_ROOT'), 'mocks', 'buzzard', 'buzzard_v1.6_desicut',
'8', '0', '0', 'Buzzard_v1.6_lensed-8-0.fits')
print('Reading {}'.format(mockfile))
mock_BGS = Table(fitsio.read(mockfile)) #columns='lmag z'.split()
# From the BGS template library, select a reddish galaxy
tflux, twave, tmeta_BGS = read_basis_templates('BGS')
i = np.argmin(np.abs(2.0 - tmeta_BGS['D4000']))
iredBGS = i
redspecBGS = tflux[i, :]
Itempl_BGS = np.array([iredBGS])
# LMAG: observed mag, DECam grizY
mock_mag_r = mock_BGS['LMAG'][:, 1] # r-band
dm = 0.01
zz_BGS = np.zeros_like(BGSmags)
for ii, mag in enumerate(BGSmags):
I = np.flatnonzero(np.abs(mock_mag_r - mag) <= dm)
zz_BGS[ii] = mock_BGS['Z'][rand.choice(I, size=1, replace=False)]
input_meta_BGS = Table()
input_meta_BGS['TEMPLATEID'] = [Itempl_BGS]*nsim
input_meta_BGS['SEED'] = np.arange(nsim) # [seed]*nsim #
input_meta_BGS['REDSHIFT'] = zz_BGS
input_meta_BGS['MAG'] = BGSmags
input_meta_BGS['MAGFILTER'] = ['decam2014-r']*nsim
BGSflux, BGSwave, BGSmeta, BGSobjmeta = BGS().make_templates(
input_meta=input_meta_BGS, nocolorcuts=True, seed=seed)
# [2] Build the noise-less source (ELG) spectra.
#ELGmags = maggen(BGSmags[:, np.newaxis], fratios[np.newaxis, :])
ELGmags = maggen(BGSmags, fratios)
# Select a single ELG template.
tflux, twave, tmeta_ELG = read_basis_templates('ELG')
i = np.argmin(np.abs(1.0 - tmeta_ELG['D4000'])) # MIGHT NEED TO ADJUST THIS LINE
iblueELG = i
bluespecELG = tflux[i, :]
Itempl_ELG = np.array([iblueELG])
# uncorrelated redshifts
zmin_ELG, zmax_ELG = 0.8, 1.4
zz_ELG = rand.uniform(zmin_ELG, zmax_ELG, nsim)
input_meta_ELG = Table()
input_meta_ELG['TEMPLATEID'] = [Itempl_ELG]*nsim
input_meta_ELG['SEED'] = [3]*nsim # [seed]*nsim # np.arange(nsim) hack!
input_meta_ELG['REDSHIFT'] = zz_ELG
input_meta_ELG['MAG'] = ELGmags
input_meta_ELG['MAGFILTER'] = ['decam2014-r']*nsim
ELGflux, ELGwave, ELGmeta, ELGobjmeta = ELG().make_templates(
input_meta=input_meta_ELG, nocolorcuts=True, seed=seed)
assert(np.all(BGSwave == ELGwave))
# Pack the simulation info into a table, for convenience.
siminfo = Table()
siminfo['TARGETID'] = np.arange(nsim, dtype=np.int64)
siminfo['LENS_Z'] = input_meta_BGS['REDSHIFT'].astype('f4')
siminfo['LENS_MAG'] = input_meta_BGS['MAG'].astype('f4')
siminfo['SOURCE_Z'] = input_meta_ELG['REDSHIFT'].astype('f4')
siminfo['SOURCE_MAG'] = input_meta_ELG['MAG'].astype('f4')
siminfo['FRATIO'] = fratios.astype('f4')
siminfo['CHUNK'] = np.zeros(nsim, dtype=np.int32)
# Generate simulated DESI spectra given real spectra and observing
# conditions. Divide the sample into chunks with a fixed number of
# spectra per chunk (but no more than 500).
obscond = {'AIRMASS': 1.3, 'EXPTIME': exptime, 'SEEING': 1.1,
'MOONALT': -60, 'MOONFRAC': 0.0, 'MOONSEP': 180}
simflux = BGSflux + ELGflux
simwave = BGSwave
for ichunk in np.arange(nchunk):
specfile = 'lenssource-spectra-chunk{:03d}.fits'.format(ichunk)
print('Writing chunk {}/{} to {}'.format(ichunk, nchunk-1, specfile))
i1 = ichunk * nperchunk
i2 = (ichunk+1) * nperchunk
siminfo['CHUNK'][i1:i2] = ichunk
sim_spectra(simwave, simflux[i1:i2, :], 'dark', specfile, obsconditions=obscond,
sourcetype='bgs', seed=seed, targetid=siminfo['TARGETID'][i1:i2],
redshift=siminfo['LENS_Z'][i1:i2])
if debug:
spectra = read_spectra(specfile)
for igal in np.arange(spectra.num_targets()):
qafile = 'lenssource-spectra-chunk{:03d}-{}.png'.format(ichunk, igal)
fig, ax = plt.subplots()
for band in spectra.bands:
ax.plot(spectra.wave[band], spectra.flux[band][igal, :])
ax.plot(simwave, simflux[i1:i2, :][igal, :], color='k', lw=2)
ax.set_ylim(np.median(simflux[i1:i2, :][igal, :]) + np.std(spectra.flux['r'][igal, :]) * np.array([-1.5, 3]))
fig.savefig(qafile)
plt.close()
# write out and return
hduflux = fits.PrimaryHDU(simflux)
hduflux.header['EXTNAME'] = 'FLUX'
hduflux.header['BUNIT'] = '10^(-17) erg/(s cm2 Angstrom)'
hdubgs = fits.ImageHDU(BGSflux)
hdubgs.header['EXTNAME'] = 'BGSFLUX'
hduelg = fits.ImageHDU(ELGflux)
hduelg.header['EXTNAME'] = 'ELGFLUX'
hduwave = fits.ImageHDU(simwave)
hduwave.header['EXTNAME'] = 'WAVE'
hduwave.header['BUNIT'] = 'Angstrom'
hduwave.header['AIRORVAC'] = ('vac', 'vacuum wavelengths')
hdutable = fits.convenience.table_to_hdu(siminfo)
hdutable.header['EXTNAME'] = 'METADATA'
hx = fits.HDUList([hduflux, hdubgs, hduelg, hduwave, hdutable])
print('Writing {}'.format(infofile))
hx.writeto(infofile, overwrite=True)
return siminfo
def main(args=None):
log = get_logger()
if isinstance(args, (list, tuple, type(None))):
args = parse(args)
thedate = datetime.now().strftime('%Y-%m-%d')
# Generate transient model if one is specified.
trans_model = None
if args.transient is not None:
trans_model = transients.get_model(args.transient)
# Generate list of HEALPix pixels to randomly sample the mocks.
rng = np.random.RandomState(args.seed)
nside = args.nside
healpixels = _get_healpixels_in_footprint(nside=args.nside)
npix = np.minimum(10 * args.nsim, len(healpixels))
pixels = rng.choice(healpixels, size=npix, replace=False)
ipix = iter(pixels)
# Set up the template generator.
fluxratio_range = 10**(-np.sort(args.magrange)[::-1] / 2.5)
epoch_range = np.sort(args.epochrange)
if args.host == 'bgs':
maker = BGSMaker(seed=args.seed)
tmpl_maker = BGS
elif args.host == 'elg':
maker = ELGMaker(seed=args.seed)
tmpl_maker = ELG
elif args.host == 'lrg':
maker = LRGMaker(seed=args.seed)
tmpl_maker = LRG
else:
raise ValueError('Unusable host type {}'.format(args.host))
maker.template_maker = tmpl_maker(transient=trans_model,
tr_fluxratio=fluxratio_range,
tr_epoch=epoch_range)
for j in range(args.nsim):
# Loop until finding a non-empty healpixel with mock galaxies.
tdata = []
while len(tdata) == 0:
pixel = next(ipix)
tdata = maker.read(healpixels=pixel, nside=args.nside)
# Generate spectral templates and write them to truth files.
# Keep producing templates until we have enough to pass brightness cuts.
wave = None
flux, targ, truth, objtr = [], [], [], []
ntosim = np.min([args.nspec, len(tdata['RA'])])
ngood = 0
while ngood < args.nspec:
idx = rng.choice(len(tdata['RA']), ntosim)
tflux, twave, ttarg, ttruth, tobj = maker.make_spectra(tdata,
indx=idx)
g, r, z, w1, w2 = [
ttruth['FLUX_{}'.format(_)]
for _ in ['G', 'R', 'Z', 'W1', 'W2']
]
rfib = ttarg['FIBERFLUX_R']
# print(g, r, z, w1, w2, rfib)
# Apply color cuts.
is_bright = isBGS_colors(rfib, g, r, z, w1, w2, targtype='bright')
is_faint = isBGS_colors(rfib, g, r, z, w1, w2, targtype='faint')
is_wise = isBGS_colors(rfib, g, r, z, w1, w2, targtype='wise')
keep = np.logical_or.reduce([is_bright, is_faint, is_wise])
_ngood = np.count_nonzero(keep)
if _ngood > 0:
ngood += _ngood
flux.append(tflux[keep, :])
targ.append(ttarg[keep])
truth.append(ttruth[keep])
objtr.append(tobj[keep])
wave = maker.wave
flux = np.vstack(flux)[:args.nspec, :]
targ = vstack(targ)[:args.nspec]
truth = vstack(truth)[:args.nspec]
objtr = vstack(objtr)[:args.nspec]
# Set up and verify the TARGETID across all truth tables.
n = len(truth)
new_id = 10000 * pixel + 100 * j + np.arange(1, n + 1)
targ['OBJID'][:] = new_id
truth['TARGETID'][:] = new_id
objtr['TARGETID'][:] = new_id
assert (len(truth) == args.nspec)
assert (np.all(targ['OBJID'] == truth['TARGETID']))
assert (len(targ) == len(np.unique(targ['OBJID'])))
assert (len(truth) == len(np.unique(truth['TARGETID'])))
assert (len(objtr) == len(np.unique(objtr['TARGETID'])))
truthfile = os.path.join(
args.outdir,
'{}_{}_{:04d}s_{:03d}_truth.fits'.format(args.host, thedate,
int(args.exptime), j + 1))
write_templates(truthfile, flux, wave, targ, truth, objtr)
log.info('Wrote {}'.format(truthfile))
# Get observing conditions and generate spectra.
obs = dict(AIRMASS=args.airmass,
EXPTIME=args.exptime,
MOONALT=args.moonalt,
MOONFRAC=args.moonfrac,
MOONSEP=args.moonsep,
SEEING=args.seeing)
fcols = dict(BRICKID=targ['BRICKID'],
BRICK_OBJID=targ['OBJID'],
FLUX_G=targ['FLUX_G'],
FLUX_R=targ['FLUX_R'],
FLUX_Z=targ['FLUX_Z'],
FLUX_W1=targ['FLUX_W1'],
FLUX_W2=targ['FLUX_W2'],
FLUX_IVAR_G=targ['FLUX_IVAR_G'],
FLUX_IVAR_R=targ['FLUX_IVAR_R'],
FLUX_IVAR_Z=targ['FLUX_IVAR_Z'],
FLUX_IVAR_W1=targ['FLUX_IVAR_W1'],
FLUX_IVAR_W2=targ['FLUX_IVAR_W2'],
FIBERFLUX_G=targ['FIBERFLUX_G'],
FIBERFLUX_R=targ['FIBERFLUX_R'],
FIBERFLUX_Z=targ['FIBERFLUX_Z'],
FIBERTOTFLUX_G=targ['FIBERTOTFLUX_G'],
FIBERTOTFLUX_R=targ['FIBERTOTFLUX_R'],
FIBERTOTFLUX_Z=targ['FIBERTOTFLUX_Z'],
MW_TRANSMISSION_G=targ['MW_TRANSMISSION_G'],
MW_TRANSMISSION_R=targ['MW_TRANSMISSION_R'],
MW_TRANSMISSION_Z=targ['MW_TRANSMISSION_Z'],
EBV=targ['EBV'])
specfile = os.path.join(
args.outdir,
'{}_{}_{:04d}s_{:03d}_spect.fits'.format(args.host, thedate,
int(args.exptime), j + 1))
# redshifts = truth['TRUEZ'] if args.host=='bgs' else None
redshifts = None
sim_spectra(wave,
flux,
args.host,
specfile,
sourcetype=args.host,
obsconditions=obs,
meta=obs,
fibermap_columns=fcols,
targetid=truth['TARGETID'],
redshift=redshifts,
ra=targ['RA'],
dec=targ['DEC'],
seed=args.seed,
expid=j)
if args.coadd_cameras:
coaddfile = specfile.replace('spect', 'coadd')
spectra = read_spectra(specfile)
spectra = coadd_cameras(spectra)
write_spectra(coaddfile, spectra)
log.info('Wrote {}'.format(coaddfile))
def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
log.critical(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
log.critical(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
if len(args.infile) == 0:
log.critical("You must specify input files")
return 12
log.info("reading input ...")
# inspect headers
input_is_frames = False
input_is_spectra = False
for filename in args.infile:
ifile = fitsio.FITS(filename)
head = ifile[0].read_header()
identified = False
if "EXTNAME" in head and head["EXTNAME"] == "FLUX":
print(filename, "is a frame")
input_is_frames = True
identified = True
ifile.close()
continue
for hdu in ifile:
head = hdu.read_header()
if "EXTNAME" in head and head["EXTNAME"].find("_FLUX") >= 0:
print(filename, "is a spectra")
input_is_spectra = True
identified = True
break
ifile.close()
if not identified:
log.error(
"{} not identified as frame of spectra file".format(filename))
sys.exit(1)
if input_is_frames and input_is_spectra:
log.error("cannot combine input spectra and frames")
sys.exit(1)
if input_is_spectra:
spectra = read_spectra(args.infile[0])
for filename in args.infile[1:]:
log.info("append {}".format(filename))
spectra.update(read_spectra(filename))
else: # frames
frames = dict()
cameras = {}
for filename in args.infile:
frame = read_frame(filename)
night = frame.meta['NIGHT']
expid = frame.meta['EXPID']
camera = frame.meta['CAMERA']
frames[(night, expid, camera)] = frame
if args.coadd_cameras:
cam, spec = camera[0], camera[1]
# Keep a list of cameras (b,r,z) for each exposure + spec
if (night, expid) not in cameras.keys():
cameras[(night, expid)] = {spec: [cam]}
elif spec not in cameras[(night, expid)].keys():
cameras[(night, expid)][spec] = [cam]
else:
cameras[(night, expid)][spec].append(cam)
if args.coadd_cameras:
# If not all 3 cameras are available, remove the incomplete sets
for (night, expid), camdict in cameras.items():
for spec, camlist in camdict.items():
log.info("Found {} for SP{} on NIGHT {} EXP {}".format(
camlist, spec, night, expid))
if len(camlist) != 3 or np.any(
np.sort(camlist) != np.array(['b', 'r', 'z'])):
for cam in camlist:
frames.pop((night, expid, cam + spec))
log.warning(
"Removing {}{} from Night {} EXP {}".format(
cam, spec, night, expid))
#import pdb
#pdb.set_trace()
spectra = frames2spectra(frames)
#- hacks to make SpectraLite like a Spectra
spectra.fibermap = Table(spectra.fibermap)
del frames #- maybe free some memory
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
#- Add input files to header
if spectra.meta is None:
spectra.meta = dict()
for i, filename in enumerate(args.infile):
spectra.meta['INFIL{:03d}'.format(i)] = os.path.basename(filename)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
def get_spectra(spectra_name,
zbest_name,
lambda_width,
index_to_fit,
template_dir=None,
archetypes_dir=None):
"""
Get the spectra and the best fit model from a given spectra and zbest file.
Args:
spectra_name (str): The name of the spectra file.
zbest_name (str): The name of the zbest file associated to the spectra
file.
lambda_width (float): The width in wavelength (in Angstrom) considered
for the fitting arount the MgII peak
index_to_fit (boolean numpy array): boolean array of size 500 specifing
which spectra have to be used
add_linear_term (boolean): Add a linear term to the Gaussian peak term
to fit the continuum.
template_dir (str): If the redrock template variable is not loaded by
the desi environment, specify the template path
archetypes_dir (str): If not None, use the archetypes templates in the
path specified
Returns:
target_id (numpy array): Array containing target id of the the object
to fit
redshift_redrock (numpy array): Array containing the redshift of the the
object to fit
flux (numpy array): Array containing the full flux arrays of every
object to fit
ivar_flux (numpy array): Array containing the inverse variance arrays
of every object to fit
model_flux (numpy array): Array containing the best fit redrock model
for every object to fit
wavelength (numpy array): Array containing the wavelength
index_with_fit (boolean numpy array): boolean array of index_to_fit size
masking index where mgII fitter is
not apply
"""
spectra = read_spectra(spectra_name)
spectra = spectra.select(
targets=spectra.fibermap["TARGETID"][index_to_fit])
if 'brz' not in spectra.bands:
spectra = coadd_cameras(spectra)
zbest = Table.read(zbest_name, 'ZBEST')[index_to_fit]
if archetypes_dir is not None:
model_wave, model_flux = create_model(spectra,
zbest,
archetype_fit=True,
archetypes_dir=archetypes_dir,
template_dir=template_dir)
else:
model_wave, model_flux = create_model(spectra,
zbest,
archetype_fit=False,
archetypes_dir=None,
template_dir=template_dir)
redshift_redrock = zbest["Z"]
wavelength = spectra.wave['brz']
mgii_peak_1, mgii_peak_2 = 2803.5324, 2796.3511
mean_mgii_peak = (mgii_peak_1 + mgii_peak_2) / 2
non_visible_peak = (redshift_redrock + 1) * mean_mgii_peak < np.min(
wavelength) + lambda_width / 2
non_visible_peak |= (redshift_redrock + 1) * mean_mgii_peak > np.max(
wavelength) - lambda_width / 2
index_with_fit = ~non_visible_peak
target_id = spectra.fibermap["TARGETID"][index_with_fit]
redshift_redrock = redshift_redrock[index_with_fit]
flux = spectra.flux['brz'][index_with_fit]
ivar_flux = spectra.ivar['brz'][index_with_fit]
model_flux = model_flux[index_with_fit]
return target_id, redshift_redrock, flux, ivar_flux, model_flux, wavelength, index_with_fit
allzbest = None
allfmap = None
allwave = None
allflux = None
allivar = None
allmask = None
allres = None
print('Tile: {} - {}'.format(tile_number, obsdate))
for cafile, zbfile in match_files(cafiles, zbfiles):
print(' - Petal {}'.format(get_petal_id(cafile)))
# Access data per petal.
zbest = Table.read(zbfile, 'ZBEST')
pspectra = read_spectra(cafile)
cspectra = coadd_cameras(pspectra)
fibermap = cspectra.fibermap
# Apply standard event selection.
isTGT = fibermap['OBJTYPE'] == 'TGT'
isGAL = zbest['SPECTYPE'] == 'GALAXY'
isBGS = fibermap['SV1_BGS_TARGET'] & bgs_mask.mask(
sv1_bgs_bits) != 0
isGoodFiber = fibermap['FIBERSTATUS'] == 0
isGoodZbest = (zbest['DELTACHI2'] > 25.) & (zbest['ZWARN']
== 0)
select = isTGT & isGAL & isBGS & isGoodFiber & isGoodZbest
print(' + selected: {}'.format(np.sum(select)))
def get_spectra(spectra_name,
zbest_name,
lambda_width,
qso_target=True,
template_dir=None,
archetypes_dir=None):
spectra = read_spectra(spectra_name)
if 'brz' not in spectra.bands:
spectra = coadd_cameras(spectra)
zbest = Table.read(zbest_name, 'ZBEST')
if archetypes_dir is not None:
model_wave, model_flux = create_model(spectra,
zbest,
archetype_fit=True,
archetypes_dir=archetypes_dir,
template_dir=template_dir)
else:
model_wave, model_flux = create_model(spectra,
zbest,
archetype_fit=False,
archetypes_dir=None,
template_dir=template_dir)
fiber_status = spectra.fibermap["FIBERSTATUS"]
target_id = spectra.fibermap["TARGETID"]
sv1_desi_target = spectra.fibermap["SV1_DESI_TARGET"]
wavelength = spectra.wave['brz']
flux = spectra.flux['brz']
ivar_flux = spectra.ivar['brz']
redshift_redrock = zbest["Z"]
spec_type = zbest["SPECTYPE"]
fiber_ok = fiber_status == 0
redrock_galaxies = (spec_type == "GALAXY")
mgii_peak_1 = 2803.5324
mgii_peak_2 = 2796.3511
mean_mgii_peak = (mgii_peak_1 + mgii_peak_2) / 2
non_visible_peak = (redshift_redrock + 1) * mean_mgii_peak < np.min(
wavelength) + lambda_width / 2
non_visible_peak |= (redshift_redrock + 1) * mean_mgii_peak > np.max(
wavelength) - lambda_width / 2
galaxies_to_test = redrock_galaxies & fiber_ok & (~non_visible_peak)
if (qso_target):
galaxies_to_test &= ((sv1_desi_target & 4) != 0)
print("nb galaxies to test: ",
len(galaxies_to_test[galaxies_to_test == True]))
target_id_to_test = target_id[galaxies_to_test]
redshift_redrock_to_test = redshift_redrock[galaxies_to_test]
flux_to_test = flux[galaxies_to_test]
ivar_flux_to_test = ivar_flux[galaxies_to_test]
model_flux_to_test = model_flux[galaxies_to_test]
print("Flux and RR best fit obtained")
return (target_id_to_test, redshift_redrock_to_test, flux_to_test,
ivar_flux_to_test, model_flux_to_test, wavelength)
def __next__(self):
# the first
if self.it0 is None:
try:
self.it0 = iter(self.list)
(self.tile, self.date) = self.it0.__next__()
self.it1 = fs_utils.panels.flat
self.panel = self.it1.__next__()
filename = fs_utils.fitsfile(self.tile,
self.date,
self.panel,
subdir=self.subdir,
trunk=self.trunk)
self.spectra = read_spectra(filename)
self.it2 = Spectra_Subspectra_Iterator(self.spectra,
verbose=True)
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(),
verbose=True)
ans = self.it3.__next__()
except:
raise StopIteration
while True:
try:
ans = self.it3.__next__()
break
except StopIteration:
try:
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(),
verbose=True)
except StopIteration:
try:
self.panel = self.it1.__next__()
filename = fs_utils.fitsfile(self.tile,
self.date,
self.panel,
subdir=self.subdir,
trunk=self.trunk)
self.spectra = read_spectra(filename)
self.it2 = Spectra_Subspectra_Iterator(self.spectra,
verbose=True)
self.it3 = Spectra_Pairs_Iterator(self.it2.__next__(),
verbose=True)
except StopIteration:
try:
(self.tile, self.date) = self.it0.__next__()
self.it1 = fs_utils.panels.flat
self.panel = self.it1.__next__()
filename = fs_utils.fitsfile(self.tile,
self.date,
self.panel,
subdir=self.subdir,
trunk=self.trunk)
self.spectra = read_spectra(filename)
self.it2 = Spectra_Subspectra_Iterator(
self.spectra, verbose=True)
self.it3 = Spectra_Pairs_Iterator(
self.it2.__next__(), verbose=True)
except StopIteration:
raise StopIteration
if self.verbose:
print(self.tile, self.date)
return ans
def main(cmdargs):
""" Load DESI spectra using the Spectra and DESISpectrum Classes
defined in squeze_boss_spectra.py and squeze_desi_spectrum.py
respectively.
"""
# load options
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"-i",
"--input-filename",
type=str,
required=True,
help="""Name of the filename to be loaded to be loaded.""")
parser.add_argument(
"-m",
"--model",
type=str,
required=True,
help="""Name of the file containing the trained model.""")
parser.add_argument("-o",
"--output-filename",
type=str,
required=True,
help="""Name of the output fits file.""")
parser.add_argument(
"-e",
"--single-exp",
action="store_true",
help="""Load only the first reobservation for each spectrum""")
parser.add_argument(
"--metadata",
nargs='+',
required=False,
default=["TARGETID"],
help="""White-spaced list of the list of columns to keep as metadata"""
)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="""Print messages""")
args = parser.parse_args(cmdargs)
# prepare variables
assert args.output_filename.endswith(
"fits") or args.output_filename.endswith("fits.gz")
if args.verbose:
userprint = verboseprint
else:
userprint = quietprint
args.keep_cols = [col.upper() for col in args.keep_cols]
# read desi spectra
userprint("Reading spectra")
desi_spectra = read_spectra(args.input_filename)
# initialize squeze Spectra class
squeze_spectra = Spectra()
# get targetids
targetid = np.unique(desi_spectra.fibermap["TARGETID"])
# loop over targeid
for targid in targetid:
# select objects
pos = np.where(desi_spectra.fibermap["TARGETID"] == targid)
# prepare metadata
metadata = {
col.upper(): desi_spectra.fibermap[col][pos[0][0]]
for col in args.metadata
}
# add specid
metadata["SPECID"] = targid
# Extract-2 data
flux = {}
wave = {}
ivar = {}
mask = {}
for band in desi_spectra.bands:
flux[band] = desi_spectra.flux[band][pos]
wave[band] = desi_spectra.wave[band]
ivar[band] = desi_spectra.ivar[band][pos]
mask[band] = desi_spectra.mask[band][pos]
# format spectrum
spectrum = DesiSpectrum(flux, wave, ivar, mask, metadata,
args.single_exp)
# append to list
squeze_spectra.append(spectrum)
# load model
userprint("Reading model")
if args.model.endswith(".json"):
model = Model.from_json(load_json(args.model))
else:
model = Model.from_fits(args.model)
# initialize candidates object
userprint("Initialize candidates object")
candidates = Candidates(mode="operation",
model=model,
name=args.output_filename)
# look for candidates
userprint('Looking for candidates')
candidates.find_candidates(squeze_spectra.spectra_list())
columns_candidates = squeze_spectra.spectra_list()[0].metadata_names()
candidates.candidates_list_to_dataframe(columns_candidates, save=False)
# compute probabilities
userprint("Computing probabilities")
candidates.classify_candidates(save=False)
# filter results
data_frame = candidates.candidates()
data_frame = data_frame[~data_frame["DUPLICATED"]]
# save results
data_out = np.zeros(len(data_frame),
dtype=[('TARGETID', 'int64'), ('Z_SQ', 'float64'),
('Z_SQ_CONF', 'float64')])
data_out['TARGETID'] = data_frame['TARGETID'].values
data_out['Z_SQ'] = data_frame['Z_TRY'].values
data_out['Z_SQ_CONF'] = data_frame['PROB'].values
data_hdu = fits.BinTableHDU.from_columns(data_out, name='SQZ_CAT')
data_hdu.writeto(args.output_filename)
tile = 80869
nlbg = 0
compwave = np.arange(1000., 5000., 1.)
comp = []
cframes = {}
for petal in [0, 1, 2, 3, 4, 5, 7, 8, 9]:
dat = Table.read(
'/global/cscratch1/sd/mjwilson/DESILBGSPEC/{:d}/deep/zbest-{:d}-{:d}-deep.fits'
.format(tile, petal, tile), 'ZBEST')
dat = dat[dat['DELTACHI2'] > 25.]
spectra = read_spectra(
'/global/cscratch1/sd/mjwilson/DESILBGSPEC/{:d}/deep/coadd-{:d}-{:d}-deep.fits'
.format(tile, petal, tile))
fmap = spectra.fibermap
fmap['ROW'] = np.arange(len(fmap))
isin = (fmap['SV1_SCND_TARGET'] & scnd_mask['HETDEX_MAIN']) != 0
isin |= (fmap['SV1_SCND_TARGET'] & scnd_mask['HETDEX_HP']) != 0
isin &= (fmap['FIBERSTATUS'] == 0)
fmap = fmap[isin]
fmap = join(fmap, dat, keys='TARGETID', join_type='left')
# Cut on dchisq
def main(cmdargs):
""" Load DESI spectra using the Spectra and DESISpectrum Classes
defined in squeze_boss_spectra.py and squeze_desi_spectrum.py
respectively.
"""
# load options
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"--input-filename",
type=str,
required=True,
help="""Name of the filename to be loaded to be loaded.""")
parser.add_argument("--output-filename",
type=str,
required=True,
help="""Name of the output filename.""")
parser.add_argument(
"--single-exp",
action="store_true",
help="""Load only the first reobservation for each spectrum""")
parser.add_argument(
"--metadata",
nargs='+',
required=False,
default=["TARGETID", "TARGET_RA", "TARGET_DEC", "CMX_TARGET"],
help="""White-spaced list of the list of columns to keep as metadata"""
)
args = parser.parse_args(cmdargs)
# read desi spectra
desi_spectra = read_spectra(args.input_filename)
# initialize squeze Spectra class
squeze_spectra = Spectra()
# get targetids
targetid = np.unique(desi_spectra.fibermap["TARGETID"])
# loop over targeid
for targid in targetid:
# select objects
pos = np.where(desi_spectra.fibermap["TARGETID"] == targid)
# prepare metadata
metadata = {
col.upper(): desi_spectra.fibermap[col][pos[0][0]]
for col in args.metadata
}
# add specid
metadata["SPECID"] = targid
# Extract-2 data
flux = {}
wave = {}
ivar = {}
mask = {}
for band in desi_spectra.bands:
flux[band] = desi_spectra.flux[band][pos]
wave[band] = desi_spectra.wave[band]
ivar[band] = desi_spectra.ivar[band][pos]
mask[band] = desi_spectra.mask[band][pos]
# format spectrum
spectrum = DesiSpectrum(flux, wave, ivar, mask, metadata,
args.single_exp)
# append to list
squeze_spectra.append(spectrum)
# save formated spectra
save_json(args.output_filename, squeze_spectra)
