def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files into one master QA file
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove
Returns:
"""
from desispec.qa import QA_Exposure
from desispec.io import write_qa_prod
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
# Reset
log.info("Resetting qa_exps in qa_prod")
self.qa_exps = []
# Loop
for night in nights:
# Loop on exposures
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
if len(frames_dict) == 0:
continue
# Load any frame (for the type and meta info)
key = list(frames_dict.keys())[0]
frame_fil = frames_dict[key]
frame_meta = read_meta_frame(frame_fil)
qa_exp = QA_Exposure(exposure,
night,
frame_meta['FLAVOR'],
specprod_dir=self.specprod_dir,
remove=remove)
qa_exp.load_meta(frame_meta)
# Append
self.qa_exps.append(qa_exp)
# Write
outroot = self.specprod_dir + '/QA/' + self.prod_name + '_qa'
write_qa_prod(outroot, self)
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files to generate
a list of QA_Exposure objects
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove the individual QA files?
Returns:
"""
from desispec.qa import QA_Exposure
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
# Reset
log.info("Resetting QA_Exposure objects")
self.qa_exps = []
# Loop
for night in self.mexp_dict.keys():
# Loop on exposures
for exposure in self.mexp_dict[night].keys():
frames_dict = self.mexp_dict[night][exposure]
if len(frames_dict) == 0:
continue
# Load any frame (for the type)
qa_exp = QA_Exposure(exposure, night,
specprod_dir=self.specprod_dir, remove=remove)
# Append
self.qa_exps.append(qa_exp)
def test_qa_exposure_load_write_data(self):
#- Test loading data
self._write_qaframes()
qaexp = QA_Exposure(self.expid,
self.night,
'dark',
specprod_dir=self.testDir)
assert 'b0' in qaexp.data['frames']
assert 'b1' in qaexp.data['frames']
# Write
write_qa_exposure(self.qafile_exp, qaexp)
def test_init_qa_exposure(self):
# Simple init
os.environ['SPECPROD'] = './'
os.environ['DESI_SPECTRO_REDUX'] = './'
qaexp = QA_Exposure(1, '20150211', 'arc')
assert qaexp.expid == 1
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = get_logger()
#- YEARMMDD string, rolls over at noon not midnight
if night is None:
night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
#- check for required environment variables
check_env()
#- parameter dictionary that will later be used for formatting commands
params = dict(night=night, nspec=nspec)
#-----
#- Input fibermaps, spectra, and pixel-level raw data
raw_dict = {0: 'flat', 1: 'arc', 2: 'dark'}
#for expid, flavor in zip([0,1,2], ['flat', 'arc', 'dark']):
for expid, flavor in raw_dict.items():
cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, **params)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))
cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
# outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
#-----
#- Extract
waverange = dict(
b = "3570,5940,1.0",
r = "5630,7740,1.0",
z = "7440,9830,1.0",
)
for expid in [0,1,2]:
for channel in ['b', 'r', 'z']:
camera = channel+'0'
pixfile = io.findfile('pix', night, expid, camera)
fiberfile = io.findfile('fibermap', night, expid)
psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel)
framefile = io.findfile('frame', night, expid, camera)
# cmd = "exspec -i {pix} -p {psf} --specmin 0 --nspec {nspec} -w {wave} -o {frame}".format(
# pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)
cmd = "desi_extract_spectra -i {pix} -p {psf} -f {fibermap} --specmin 0 --nspec {nspec} -o {frame}".format(
pix=pixfile, psf=psffile, frame=framefile, fibermap=fiberfile, **params)
inputs = [pixfile, psffile, fiberfile]
outputs = [framefile,]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid))
#-----
#- Fiber flat
expid = 0
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fiberflat = io.findfile('fiberflat', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid) # for QA
qafile = io.findfile('qa_calib', night, expid, camera)
qafig = io.findfile('qa_flat_fig', night, expid, camera)
cmd = "desi_compute_fiberflat --infile {frame} --outfile {fiberflat} --qafile {qafile} --qafig {qafig}".format(
frame=framefile, fiberflat=fiberflat, qafile=qafile, qafig=qafig, **params)
inputs = [framefile,fibermap,]
outputs = [fiberflat,qafile,qafig,]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('fiberflat failed for '+camera)
#-----
#- Sky model
flat_expid = 0
expid = 2
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
qafile = io.findfile('qa_data', night, expid, camera)
qafig = io.findfile('qa_sky_fig', night, expid, camera)
cmd="desi_compute_sky --infile {frame} --fiberflat {fiberflat} --outfile {sky} --qafile {qafile} --qafig {qafig}".format(
frame=framefile, fiberflat=fiberflat, sky=skyfile, qafile=qafile, qafig=qafig, **params)
inputs = [framefile, fibermap, fiberflat]
outputs = [skyfile, qafile, qafig,]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('sky model failed for '+camera)
#-----
#- Fit standard stars
if 'STD_TEMPLATES' in os.environ:
std_templates = os.getenv('STD_TEMPLATES')
else:
std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/star_templates/v1.1/star_templates_v1.1.fits'
stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
flats = list()
frames = list()
skymodels = list()
for channel in ['b', 'r', 'z']:
camera = channel+'0'
frames.append( io.findfile('frame', night, expid, camera) )
flats.append( io.findfile('fiberflat', night, flat_expid, camera) )
skymodels.append( io.findfile('sky', night, expid, camera) )
frames = ' '.join(frames)
flats = ' '.join(flats)
skymodels = ' '.join(skymodels)
cmd = """desi_fit_stdstars \
--frames {frames} \
--fiberflats {flats} \
--skymodels {skymodels} \
--starmodels {std_templates} \
-o {stdstars}""".format(
frames=frames, flats=flats, skymodels=skymodels,
std_templates=std_templates, stdstars=stdstarfile)
inputs = [fibermap, std_templates]
outputs = [stdstarfile,]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('fitting stdstars failed')
#-----
#- Flux calibration
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
calibfile = io.findfile('calib', night, expid, camera)
qafile = io.findfile('qa_data', night, expid, camera)
qafig = io.findfile('qa_flux_fig', night, expid, camera)
#- Compute flux calibration vector
cmd = """desi_compute_fluxcalibration \
--infile {frame} --fiberflat {fiberflat} --sky {sky} \
--models {stdstars} --outfile {calib} --qafile {qafile} --qafig {qafig}""".format(
frame=framefile, fiberflat=fiberflat, sky=skyfile,
stdstars=stdstarfile, calib=calibfile, qafile=qafile, qafig=qafig
)
inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
outputs = [calibfile, qafile, qafig]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('flux calibration failed for '+camera)
#- Apply the flux calibration to write a cframe file
cframefile = io.findfile('cframe', night, expid, camera)
cmd = """desi_process_exposure \
--infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
--outfile {cframe}""".format(frame=framefile, fibermap=fibermap,
fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile)
inputs = [framefile, fiberflat, skyfile, calibfile]
outputs = [cframefile, ]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('combining calibration steps failed for '+camera)
#-----
#- Collate QA
# Collate data QA
expid = 2
qafile = io.findfile('qa_data_exp', night, expid)
qaexp_data = QA_Exposure(expid, night, raw_dict[expid]) # Removes camera files
io.write_qa_exposure(qafile, qaexp_data)
# Collate calib QA
calib_expid = [0,1]
for expid in calib_expid:
qafile = io.findfile('qa_calib_exp', night, expid)
qaexp_calib = QA_Exposure(expid, night, raw_dict[expid])
io.write_qa_exposure(qafile, qaexp_calib)
#-----
#- Bricks
expid = 2
inputs = list()
for camera in ['b0', 'r0', 'z0']:
inputs.append( io.findfile('cframe', night, expid, camera) )
outputs = list()
fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
bricks = set(fibermap['BRICKNAME'])
for b in bricks:
for channel in ['b', 'r', 'z']:
outputs.append( io.findfile('brick', brickname=b, band=channel))
cmd = "desi_make_bricks --night "+night
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('brick generation failed')
#-----
#- Redshifts!
for b in bricks:
inputs = [io.findfile('brick', brickname=b, band=channel) for channel in ['b', 'r', 'z']]
zbestfile = io.findfile('zbest', brickname=b)
outputs = [zbestfile, ]
cmd = "desi_zfind --brick {} -o {}".format(b, zbestfile)
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('redshifts failed for brick '+b)
# ztruth QA
# qafile = io.findfile('qa_ztruth', night)
# qafig = io.findfile('qa_ztruth_fig', night)
# cmd = "desi_qa_zfind --night {night} --qafile {qafile} --qafig {qafig} --verbose".format(
# night=night, qafile=qafile, qafig=qafig)
# inputs = []
# outputs = [qafile, qafig]
# if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
# raise RuntimeError('redshift QA failed for night '+night)
#-----
#- Did it work?
#- (this combination of fibermap, simspec, and zbest is a pain)
simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickname=b))
for i in range(len(zbest.z)):
if zbest.spectype[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.spectype[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.spectype[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
oiiflux = siminfo['OIIFLUX'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5*(z-truez)/(1+truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL':
if abs(dv) < 150 or oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
else:
status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux)
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype in ('STD', 'FSTD') and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")