def test_quickgen_moonzenith_simspec(self):
night = self.night
camera = 'r0'
expid0 = 100
expid1 = 101
# generate exposures
obs.new_exposure('bgs',night=night,expid=expid0,nspec=1,seed=1)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obs.new_exposure('bgs',night=night,expid=expid1,nspec=1,seed=1)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
# generate quickgen output for each moon phase
cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 0".format(simspec0,fibermap0)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 90".format(simspec1,fibermap1)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cframe0=desispec.io.findfile("cframe",night,expid0,camera)
cframe1=desispec.io.findfile("cframe",night,expid1,camera)
cf0=desispec.io.read_frame(cframe0)
cf1=desispec.io.read_frame(cframe1)
self.assertLess(np.median(cf0.ivar),np.median(cf1.ivar))
def _write_qaframe(self, camera='b0', expid=1, night='20160101', ZPval=24., flavor='science'):
"""Write QA data frame files"""
frm = self._make_frame(camera=camera, expid=expid, night=night, flavor=flavor)
qafrm = QA_Frame(frm)
# SKY
qafrm.init_skysub()
qafrm.qa_data['SKYSUB']['METRICS'] = {}
qafrm.qa_data['SKYSUB']['METRICS']['NSKY_FIB'] = 10
# FLUX
qafrm.init_fluxcalib()
qafrm.qa_data['FLUXCALIB']['METRICS'] = {}
qafrm.qa_data['FLUXCALIB']['METRICS']['ZP'] = ZPval
qafrm.qa_data['FLUXCALIB']['METRICS']['RMS_ZP'] = 0.05
# Outfile
qafile = findfile('qa_data', night=night, expid=expid,
specprod_dir=self.testDir, camera=camera)
# WRITE
write_qa_frame(qafile, qafrm)
self.files_written.append(qafile)
# Generate frame too (for QA_Exposure)
frame = self._make_frame(camera=camera, flavor=flavor, night=night, expid=expid)
frame_file = findfile('frame', night=night, expid=expid, specprod_dir=self.testDir, camera=camera)
_ = write_frame(frame_file, frame)
self.files_written.append(frame_file)
#
return qafile
def io_qa_pa(self,paname):
"""
Specify the filenames: json and png of the pa level qa files"
"""
filemap={'Initialize': 'initial',
'Preproc': 'preproc',
'Flexure': 'flexure',
'BoxcarExtract': 'boxextract',
'Extract_QP': 'extractqp',
'ComputeFiberflat_QL': 'computeflat',
'ComputeFiberflat_QP': 'computeflatqp',
'ApplyFiberFlat_QL': 'fiberflat',
'ApplyFiberFlat_QP': 'fiberflatqp',
'SkySub_QL': 'skysub',
'SkySub_QP': 'skysubqp',
'ResolutionFit': 'resfit',
'ApplyFluxCalibration': 'fluxcalib'
}
if paname in filemap:
outfile=findfile('ql_file',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfile=outfile.replace('qlfile',filemap[paname])
outfig=findfile('ql_fig',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfig=outfig.replace('qlfig',filemap[paname])
else:
raise IOError("PA name does not match any file type. Check PA name in config for {}".format(paname))
return (outfile,outfig)
def test_quickgen_airmass_simspec(self):
night=self.night
camera = 'r0'
expid0 = 100
expid1 = 101
# generate exposures of varying airmass
obscond = simexp.reference_conditions['DARK']
obscond['AIRMASS'] = 1.5
obs.new_exposure('dark',night=night,expid=expid0,nspec=1,seed=1,obsconditions=obscond)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obscond['AIRMASS'] = 1.0
obs.new_exposure('dark',night=night,expid=expid1,nspec=1,seed=1,obsconditions=obscond)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
# generate quickgen output for each airmass
cmd = "quickgen --simspec {} --fibermap {}".format(simspec0,fibermap0)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {}".format(simspec1,fibermap1)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cframe0=desispec.io.findfile("cframe",night,expid0,camera)
cframe1=desispec.io.findfile("cframe",night,expid1,camera)
cf0=desispec.io.read_frame(cframe0)
cf1=desispec.io.read_frame(cframe1)
self.assertLess(np.median(cf0.ivar),np.median(cf1.ivar))
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('newexp-random failed for {} exposure {}'.format(program, expid))
cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
outputs.append(io.findfile('raw', night, expid))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(program, expid))
return
def io_qa(self,qaname):
"""
Specify the filenames: json and png for the given qa output
"""
filemap={'Check_HDUs':'checkHDUs',
'Trace_Shifts':'trace',
'Bias_From_Overscan': 'getbias',
'Get_RMS' : 'getrms',
'Count_Pixels': 'countpix',
'Calc_XWSigma': 'xwsigma',
'CountSpectralBins': 'countbins',
'Sky_Continuum': 'skycont',
'Sky_Rband': 'skyRband',
'Sky_Peaks': 'skypeak',
'Sky_Residual': 'skyresid',
'Integrate_Spec': 'integ',
'Calculate_SNR': 'snr',
'Check_Resolution': 'checkres',
'Check_FiberFlat': 'checkfibflat'
}
if qaname in filemap:
outfile=findfile('ql_file',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfile=outfile.replace('qlfile',filemap[qaname])
outfig=findfile('ql_fig',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfig=outfig.replace('qlfig',filemap[qaname])
else:
raise IOError("QA name does not match any file type. Check QA name in config for {}".format(qaname))
return (outfile,outfig)
def _write_flat_file(self, camera='b0', night=None, expid=None):
# Init
if night is None:
night = self.nights[0]
if expid is None:
expid = self.expids[0]
# Filename
frame_file = findfile('frame',
night=night,
expid=expid,
specprod_dir=self.testDir,
camera=camera)
fflat_file = findfile('fiberflat',
night=night,
expid=expid,
specprod_dir=self.testDir,
camera=camera)
# Frames
fb = self._make_frame(camera=camera, flavor='flat', nspec=10)
_ = write_frame(frame_file, fb)
self.files_written.append(frame_file)
# Fiberflats
ff = get_fiberflat_from_frame(fb)
write_fiberflat(fflat_file, ff)
self.files_written.append(fflat_file)
# Return
return frame_file, fflat_file
def _write_qaframe(self, camera='b0', expid=1, night='20160101', ZPval=24., flavor='science'):
"""Write QA data frame files"""
frm = self._make_frame(camera=camera, expid=expid, night=night, flavor=flavor)
qafrm = QA_Frame(frm)
# SKY
qafrm.init_skysub()
qafrm.qa_data['SKYSUB']['METRICS'] = {}
qafrm.qa_data['SKYSUB']['METRICS']['NSKY_FIB'] = 10
# FLUX
qafrm.init_fluxcalib()
qafrm.qa_data['FLUXCALIB']['METRICS'] = {}
qafrm.qa_data['FLUXCALIB']['METRICS']['ZP'] = ZPval
qafrm.qa_data['FLUXCALIB']['METRICS']['RMS_ZP'] = 0.05
# Outfile
qafile = findfile('qa_data', night=night, expid=expid,
specprod_dir=self.testDir, camera=camera)
# WRITE
write_qa_frame(qafile, qafrm)
self.files_written.append(qafile)
# Generate frame too (for QA_Exposure)
frame = self._make_frame(camera=camera, flavor=flavor, night=night, expid=expid)
frame_file = findfile('frame', night=night, expid=expid, specprod_dir=self.testDir, camera=camera)
_ = write_frame(frame_file, frame)
self.files_written.append(frame_file)
#
return qafile
def test_quickgen_moonzenith_simspec(self):
night = self.night
camera = 'r0'
expid0 = 100
expid1 = 101
# generate exposures
obs.new_exposure('bgs', night=night, expid=expid0, nspec=1, seed=1)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obs.new_exposure('bgs', night=night, expid=expid1, nspec=1, seed=1)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
# generate quickgen output for each moon phase
cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 0".format(
simspec0, fibermap0)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 90".format(
simspec1, fibermap1)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cframe0 = desispec.io.findfile("cframe", night, expid0, camera)
cframe1 = desispec.io.findfile("cframe", night, expid1, camera)
cf0 = desispec.io.read_frame(cframe0)
cf1 = desispec.io.read_frame(cframe1)
self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def test_quickgen_seed_simspec(self):
night=self.night
camera = 'r0'
expid0 = 100
expid1 = 101
expid2 = 102
# generate exposures seed 1 & 2
obs.new_exposure('dark',night=night,expid=expid0,nspec=1,seed=1)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obs.new_exposure('dark',night=night,expid=expid1,nspec=1,seed=1)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
obs.new_exposure('dark',night=night,expid=expid2,nspec=1,seed=2)
simspec2 = io.findfile('simspec', night, expid2)
fibermap2 = desispec.io.findfile('fibermap', night, expid2)
s0=fits.open(simspec0)
s1=fits.open(simspec1)
s2=fits.open(simspec2)
self.assertEqual(s0['TRUTH'].data['OBJTYPE'][0], s1['TRUTH'].data['OBJTYPE'][0])
self.assertEqual(s0['TRUTH'].data['REDSHIFT'][0], s1['TRUTH'].data['REDSHIFT'][0])
self.assertNotEqual(s0['TRUTH'].data['REDSHIFT'][0], s2['TRUTH'].data['REDSHIFT'][0])
self.assertTrue(np.all(s0['WAVE'].data == s1['WAVE'].data))
self.assertTrue(np.all(s0['FLUX'].data == s1['FLUX'].data))
# generate quickgen output for each exposure
# spawn with subprocess so that it will really be restarting fresh;
# using quickgen.main re-uses a specsim Simulator that consumes random
# state when using it.
# See https://github.com/desihub/specsim/issues/94
cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 1".format(
sys.executable, self.binDir, simspec0, fibermap0)
### quickgen.main(quickgen.parse(cmd.split()[1:]))
subprocess.check_call(cmd.split())
cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 1".format(
sys.executable, self.binDir, simspec1, fibermap1)
### quickgen.main(quickgen.parse(cmd.split()[1:]))
subprocess.check_call(cmd.split())
cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 2".format(
sys.executable, self.binDir, simspec2, fibermap2)
### quickgen.main(quickgen.parse(cmd.split()[1:]))
subprocess.check_call(cmd.split())
cframe0=desispec.io.findfile("cframe",night,expid0,camera)
cframe1=desispec.io.findfile("cframe",night,expid1,camera)
cframe2=desispec.io.findfile("cframe",night,expid2,camera)
cf0=desispec.io.read_frame(cframe0)
cf1=desispec.io.read_frame(cframe1)
cf2=desispec.io.read_frame(cframe2)
self.assertTrue(np.all(cf0.flux == cf1.flux)) #- same seed
self.assertTrue(np.all(cf0.ivar == cf1.ivar))
self.assertTrue(np.any(cf0.flux != cf2.flux)) #- different seed
def load_s2n_values(objtype, nights, channel, sub_exposures=None):
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night) #, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap',
night=night,
expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(
flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data, masked=True)
sps_hdu.close()
objs = sps_tab['TEMPLATETYPE'] == objtype
if np.sum(objs) == 0:
continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel + str(ii)
cframe_path = findfile(filetype='cframe',
night=night,
expid=exposure,
camera=camera)
try:
cframe = read_frame(cframe_path)
except:
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave, 1)
dwave[0] = dwave[1]
#
iobjs = objs[cframe.fibers]
if np.sum(iobjs) == 0:
continue
s2n = cframe.flux[iobjs, :] * np.sqrt(
cframe.ivar[iobjs, :]) / np.sqrt(dwave)
# Save
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]])
if objtype == 'ELG':
fdict['OII'].append(
sps_tab['OIIFLUX'][cframe.fibers[iobjs]])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
def test_quickgen_simspec(self):
night = self.night
expid = self.expid
camera = 'r0'
# flavors = ['flat','dark','gray','bright','bgs','mws','elg','lrg','qso','arc']
flavors = ['flat', 'dark', 'bright']
for i in range(len(flavors)):
flavor = flavors[i]
obs.new_exposure(flavor, night=night, expid=expid, nspec=2)
#- output to same directory as input
os.environ['DESI_SPECTRO_REDUX'] = os.path.join(
os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
#- run quickgen
simspec = io.findfile('simspec', night, expid)
fibermap = io.findfile('simfibermap', night, expid)
self.assertTrue(os.path.exists(simspec))
self.assertTrue(os.path.exists(fibermap))
if flavor == 'flat':
try:
cmd = "quickgen --simspec {} --fibermap {}".format(
simspec, fibermap)
quickgen.main(quickgen.parse(cmd.split()[1:]))
except SystemExit:
pass
#- verify flat outputs
fiberflatfile = desispec.io.findfile('fiberflat', night, expid,
camera)
self.assertTrue(os.path.exists(fiberflatfile))
else:
cmd = "quickgen --simspec {} --fibermap {}".format(
simspec, fibermap)
quickgen.main(quickgen.parse(cmd.split()[1:]))
#- verify outputs
framefile = desispec.io.findfile('frame', night, expid, camera)
self.assertTrue(os.path.exists(framefile))
cframefile = desispec.io.findfile('cframe', night, expid,
camera)
self.assertTrue(os.path.exists(cframefile))
skyfile = desispec.io.findfile('sky', night, expid, camera)
self.assertTrue(os.path.exists(skyfile))
fluxcalibfile = desispec.io.findfile('calib', night, expid,
camera)
self.assertTrue(os.path.exists(fluxcalibfile))
os.remove(simspec)
os.remove(fibermap)
def test_quickgen_seed_simspec(self):
night = self.night
camera = 'r0'
expid0 = 100
expid1 = 101
expid2 = 102
# generate exposures seed 1 & 2
obs.new_exposure('dark', night=night, expid=expid0, nspec=1, seed=1)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obs.new_exposure('dark', night=night, expid=expid1, nspec=1, seed=1)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
obs.new_exposure('dark', night=night, expid=expid2, nspec=1, seed=2)
simspec2 = io.findfile('simspec', night, expid2)
fibermap2 = desispec.io.findfile('fibermap', night, expid2)
# generate quickgen output for each exposure
cmd = "quickgen --simspec {} --fibermap {} --seed 1".format(
simspec0, fibermap0)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {} --seed 1".format(
simspec1, fibermap1)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {} --seed 2".format(
simspec2, fibermap2)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cframe0 = desispec.io.findfile("cframe", night, expid0, camera)
cframe1 = desispec.io.findfile("cframe", night, expid1, camera)
cframe2 = desispec.io.findfile("cframe", night, expid2, camera)
cf0 = desispec.io.read_frame(cframe0)
cf1 = desispec.io.read_frame(cframe1)
cf2 = desispec.io.read_frame(cframe2)
self.assertTrue(np.all(cf0.flux == cf1.flux)) #- same seed
self.assertTrue(np.all(cf0.ivar == cf1.ivar))
self.assertTrue(np.any(cf0.flux != cf2.flux)) #- different seed
s0 = fits.open(simspec0)
s1 = fits.open(simspec1)
s2 = fits.open(simspec2)
self.assertEqual(s0['TRUTH'].data['OBJTYPE'][0],
s1['TRUTH'].data['OBJTYPE'][0])
self.assertEqual(s0['TRUTH'].data['REDSHIFT'][0],
s1['TRUTH'].data['REDSHIFT'][0])
self.assertNotEqual(s0['TRUTH'].data['REDSHIFT'][0],
s2['TRUTH'].data['REDSHIFT'][0])
def find_most_recent(night, file_type='psfnight', cam='r', n_nights=30):
'''
Searches back in time for either psfnight or fiberflatnight (or anything supported by
desispec.calibfinder.findcalibfile. This only works on nightly-based files, so exposure id
information is not used.
Inputs:
night : str. YYYYMMDD night to look back from
file_type : str. psfnight or fiberflatnight
cam : str. camera (b, r, or z).
n_nights : int. number of nights to step back before giving up
returns:
nightfile : str. Full pathname to calibration file of interest.
If none found, None is returned.
'''
# Set the time as Noon on the day in question
today = time.strptime('{} 12'.format(night), '%Y%m%d %H')
one_day = 60 * 60 * 24 # seconds
test_night_struct = today
# Search a month in the past
for daysback in range(n_nights):
test_night_struct = time.strptime(
time.ctime(time.mktime(test_night_struct) - one_day))
test_night_str = time.strftime('%Y%m%d', test_night_struct)
nightfile = findfile(file_type, test_night_str, camera=cam)
if os.path.isfile(nightfile):
return nightfile
return None
def _write_qaframe(self,
camera='b0',
expid=1,
night='20160101',
ZPval=24.):
"""Write QA data frame files"""
frm = self._make_frame(camera=camera)
qafrm = QA_Frame(frm)
# SKY
qafrm.init_skysub()
qafrm.qa_data['SKYSUB']['METRICS'] = {}
qafrm.qa_data['SKYSUB']['METRICS']['NSKY_FIB'] = 10
# FLUX
qafrm.init_fluxcalib()
qafrm.qa_data['FLUXCALIB']['METRICS'] = {}
qafrm.qa_data['FLUXCALIB']['METRICS']['ZP'] = ZPval
qafrm.qa_data['FLUXCALIB']['METRICS']['RMS_ZP'] = 0.05
# Outfile
qafile = findfile('qa_data',
night=night,
expid=expid,
specprod_dir=self.testDir,
camera=camera)
# WRITE
write_qa_frame(qafile, qafrm)
self.files_written.append(qafile)
return qafile
def qaargs(self):
qaopts = {}
referencemetrics=[]
for PA in self.palist:
for qa in self.qalist[PA]: #- individual QA for that PA
pa_yaml = PA.upper()
params=self._qaparams(qa)
qaopts[qa]={'night' : self.night, 'expid' : self.expid,
'camera': self.camera, 'paname': PA, 'PSFFile': self.psf_filename,
'amps': self.amps, #'qafile': self.dump_qa()[0][qa],
'qafig': self.dump_qa()[qa], 'FiberMap': self.fibermap,
'param': params, 'refKey':self._qaRefKeys[qa],
'singleqa' : self.singqa,
'plotconf':self.plotconf, 'hardplots': self.hardplots
}
if qa == 'Calc_XWSigma':
qaopts[qa]['Peaks']=self.algorithms['Initialize']['PEAKS']
qaopts[qa]['Flavor']=self.flavor
qaopts[qa]['PSFFile']=self.calibpsf
if qa == 'Sky_Peaks':
qaopts[qa]['Peaks']=self.algorithms['Initialize']['PEAKS']
if self.singqa is not None:
qaopts[qa]['rawdir']=self.rawdata_dir
qaopts[qa]['specdir']=self.specprod_dir
if qa == 'Sky_Residual':
skyfile = findfile('sky',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
qaopts[qa]['SkyFile']=skyfile
if self.reference != None:
refkey=qaopts[qa]['refKey']
for padict in range(len(self.reference)):
pa_metrics=self.reference[padict].keys()
if refkey in pa_metrics:
qaopts[qa]['ReferenceMetrics']={'{}'.format(refkey): self.reference[padict][refkey]}
return qaopts
def read_fiberflat(filename):
"""Read fiberflat from filename
Args:
filename (str): Name of fiberflat file, or (night, expid, camera) tuple
Returns:
FiberFlat object with attributes
fiberflat, ivar, mask, meanspec, wave, header
Notes:
fiberflat, ivar, mask are 2D [nspec, nwave]
meanspec and wave are 1D [nwave]
"""
#- check if outfile is (night, expid, camera) tuple instead
if isinstance(filename, (tuple, list)) and len(filename) == 3:
night, expid, camera = filename
filename = findfile('fiberflat', night, expid, camera)
header = fits.getheader(filename, 0)
fiberflat = native_endian(fits.getdata(filename, 0)).astype('f8')
ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8'))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8'))
wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8'))
return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def tearDown(self):
for expid in [self.expid, 100, 101, 102]:
fibermap = desispec.io.findfile('fibermap', self.night, expid)
if os.path.exists(fibermap):
os.remove(fibermap)
simspecfile = io.findfile('simspec', self.night, expid)
if os.path.exists(simspecfile):
os.remove(simspecfile)
for camera in ('b0', 'r0', 'z0'):
framefile = desispec.io.findfile('frame',
self.night,
expid,
camera=camera)
if os.path.exists(framefile):
os.remove(framefile)
cframefile = desispec.io.findfile('cframe',
self.night,
expid,
camera=camera)
if os.path.exists(cframefile):
os.remove(cframefile)
skyfile = desispec.io.findfile('sky',
self.night,
expid,
camera=camera)
if os.path.exists(skyfile):
os.remove(skyfile)
fluxcalibfile = desispec.io.findfile('calib',
self.night,
expid,
camera=camera)
if os.path.exists(fluxcalibfile):
os.remove(fluxcalibfile)
def read_fiberflat(filename):
"""Read fiberflat from filename
Args:
filename (str): Name of fiberflat file, or (night, expid, camera) tuple
Returns:
FiberFlat object with attributes
fiberflat, ivar, mask, meanspec, wave, header
Notes:
fiberflat, ivar, mask are 2D [nspec, nwave]
meanspec and wave are 1D [nwave]
"""
#- check if outfile is (night, expid, camera) tuple instead
if isinstance(filename, (tuple, list)) and len(filename) == 3:
night, expid, camera = filename
filename = findfile('fiberflat', night, expid, camera)
header = fits.getheader(filename, 0)
fiberflat = native_endian(fits.getdata(filename, 0))
ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8'))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8'))
wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8'))
return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def qafile_from_framefile(frame_file, qaprod_dir=None, output_dir=None):
""" Derive the QA filename from an input frame file
Args:
frame_file: str
output_dir: str, optional Over-ride default output path
qa_dir: str, optional Over-ride default QA
Returns:
"""
frame_meta = read_meta_frame(frame_file)
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = int(frame_meta['EXPID'])
if frame_meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
# Name
qafile = findfile(qatype,
night=night,
camera=camera,
expid=expid,
outdir=output_dir,
qaprod_dir=qaprod_dir)
# Return
return qafile, qatype
def dump_pa(self, paname):
"""
dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case.
"""
pafilemap = {
'Preproc': 'pix',
'BoxcarExtract': 'frame',
'ApplyFiberFlat_QL': 'fframe',
'SkySub_QL': 'sframe'
}
if paname in pafilemap:
filetype = pafilemap[paname]
else:
raise IOError(
"PA name does not match any file type. Check PA name in config"
)
pafile = findfile(filetype,
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir,
outdir=self.outdir)
return pafile
def test_quickgen_simspec(self):
night = self.night
expid = self.expid
camera = 'r0'
# flavors = ['flat','dark','gray','bright','bgs','mws','elg','lrg','qso','arc']
flavors = ['flat', 'dark', 'bright']
for i in range(len(flavors)):
flavor = flavors[i]
obs.new_exposure(flavor, night=night, expid=expid, nspec=2)
#- output to same directory as input
os.environ['DESI_SPECTRO_REDUX'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
#- run quickgen
simspec = io.findfile('simspec', night, expid)
fibermap = io.findfile('simfibermap', night, expid)
self.assertTrue(os.path.exists(simspec))
self.assertTrue(os.path.exists(fibermap))
if flavor == 'flat':
try:
cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap)
quickgen.main(quickgen.parse(cmd.split()[1:]))
except SystemExit:
pass
#- verify flat outputs
fiberflatfile = desispec.io.findfile('fiberflat', night, expid, camera)
self.assertTrue(os.path.exists(fiberflatfile))
else:
cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap)
quickgen.main(quickgen.parse(cmd.split()[1:]))
#- verify outputs
framefile = desispec.io.findfile('frame', night, expid, camera)
self.assertTrue(os.path.exists(framefile))
cframefile = desispec.io.findfile('cframe', night, expid, camera)
self.assertTrue(os.path.exists(cframefile))
skyfile = desispec.io.findfile('sky', night, expid, camera)
self.assertTrue(os.path.exists(skyfile))
fluxcalibfile = desispec.io.findfile('calib', night, expid, camera)
self.assertTrue(os.path.exists(fluxcalibfile))
os.remove(simspec)
os.remove(fibermap)
def io_qa_pa(self,paname):
"""
Specify the filenames: yaml and png of the pa level qa files"
"""
if self.conf["Flavor"] == 'arcs':
filemap={'Initialize': 'ql_initial_arc',
'Preproc': 'ql_preproc_arc',
'BootCalibration': 'ql_bootcalib',
'BoxcarExtract': 'ql_boxextract_arc',
'ResolutionFit': 'ql_resfit_arc'
}
elif self.conf["Flavor"] == 'flat':
filemap={'Initialize': 'ql_initial',
'Preproc': 'ql_preproc',
'BoxcarExtract': 'ql_boxextract',
'ComputeFiberflat_QL': 'ql_computeflat',
}
elif self.conf["Flavor"] == 'bias':
filemap={'Initialize': 'ql_initial_bias',
'Preproc': 'ql_preproc_bias'
}
elif self.conf["Flavor"] == 'dark':
filemap={'Initialize': 'ql_initial_dark',
'Preproc': 'ql_preproc_dark'
}
else:
filemap={'Initialize': 'ql_initial',
'Preproc': 'ql_preproc',
'BootCalibration': 'ql_bootcalib',
'BoxcarExtract': 'ql_boxextract',
'ResolutionFit': 'ql_resfit',
'ApplyFiberFlat_QL': 'ql_fiberflat',
'SkySub_QL': 'ql_skysub'
}
if paname in filemap:
filetype=filemap[paname]+'_file'
figtype=filemap[paname]+'_fig'
else:
raise IOError("PA name does not match any file type. Check PA name in config for {}".format(paname))
outfile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfig=findfile(figtype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
return (outfile,outfig)
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0, 1, 2], ['flat', 'arc', 'dark']):
cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap),
expid)
inputs = []
outputs = [fibermap, simspec]
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs,
clobber=clobber) != 0:
raise RuntimeError(
'pixsim newexp failed for {} exposure {}'.format(
flavor, expid))
cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
#outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs,
clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(
flavor, expid))
return
def io_qa(self,qaname):
"""
Specify the filenames: yaml and png for the given qa output
"""
if self.conf["Flavor"] == 'arcs':
filemap={'Bias_From_Overscan': 'ql_getbias_arc',
'Get_RMS' : 'ql_getrms_arc',
'Count_Pixels': 'ql_countpix_arc',
'Calc_XWSigma': 'ql_xwsigma_arc',
'CountSpectralBins': 'ql_countbins_arc'
}
elif self.conf["Flavor"] == 'bias':
filemap={'Bias_From_Overscan': 'ql_getbias_bias',
'Get_RMS' : 'ql_getrms_bias',
'Count_Pixels': 'ql_countpix_bias'
}
elif self.conf["Flavor"] == 'dark':
filemap={'Bias_From_Overscan': 'ql_getbias_dark',
'Get_RMS' : 'ql_getrms_dark',
'Count_Pixels': 'ql_countpix_dark'
}
else:
filemap={'Bias_From_Overscan': 'ql_getbias',
'Get_RMS' : 'ql_getrms',
'Count_Pixels': 'ql_countpix',
'Calc_XWSigma': 'ql_xwsigma',
'CountSpectralBins': 'ql_countbins',
'Sky_Continuum': 'ql_skycont',
'Sky_Peaks': 'ql_skypeak',
'Sky_Residual': 'ql_skyresid',
'Integrate_Spec': 'ql_integ',
'Calculate_SNR': 'ql_snr'
}
if qaname in filemap:
filetype=filemap[qaname]+'_file'
figtype=filemap[qaname]+'_fig'
else:
raise IOError("QA name does not match any file type. Check QA name in config for {}".format(qaname))
outfile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
outfig=findfile(figtype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
return (outfile,outfig)
def dump_pa(self,filetype):
"""
dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case.
"""
if filetype in ["fframe","sframe"]: #- fiberflat fielded or sky subtracted intermediate files
pafile=os.path.join(self.specprod_dir,'exposures',self.night,"%08d"%self.expid,"%s-%s-%08d.fits"%(filetype,self.camera,self.expid))
else:
pafile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
return pafile
def _write_flat_file(self, camera='b0', night=None, expid=None):
# Init
if night is None:
night = self.nights[0]
if expid is None:
expid = self.expids[0]
# Filename
frame_file = findfile('frame', night=night, expid=expid, specprod_dir=self.testDir, camera=camera)
fflat_file = findfile('fiberflat', night=night, expid=expid, specprod_dir=self.testDir, camera=camera)
# Frames
fb = self._make_frame(camera=camera, flavor='flat', nspec=10)
_ = write_frame(frame_file, fb)
self.files_written.append(frame_file)
# Fiberflats
ff = get_fiberflat_from_frame(fb)
write_fiberflat(fflat_file, ff)
self.files_written.append(fflat_file)
# Return
return frame_file, fflat_file
def read_frame(filename, nspec=None):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file, or (night, expid, camera) tuple where
night = string YEARMMDD
expid = integer exposure ID
camera = b0, r1, .. z9
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('frame', night, expid, camera)
if not os.path.isfile(filename) :
raise IOError("cannot open"+filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8'))
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
else:
fibermap = None
if 'CHI2PIX' in fx:
chi2pix = native_endian(fx['CHI2PIX'].data.astype('f8'))
else:
chi2pix = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
resolution_data = resolution_data[0:nspec]
if chi2pix is not None:
chi2pix = chi2pix[0:nspec]
if mask is not None:
mask = mask[0:nspec]
# return flux,ivar,wave,resolution_data, hdr
return Frame(wave, flux, ivar, mask, resolution_data, meta=hdr, fibermap=fibermap, chi2pix=chi2pix)
def test_expand_args(self):
night = self.night
expid = self.expid
obs.new_exposure('arc', night=night, expid=expid, nspec=4)
simspec = io.findfile('simspec', self.night, self.expid)
fibermap = desispec.io.findfile('fibermap', self.night, self.expid)
opts = ['--simspec', simspec, '--fibermap', fibermap]
args = quickgen.parse(opts)
self.assertEqual(args.simspec, simspec)
self.assertEqual(args.fibermap, fibermap)
def test_expand_args(self):
night = self.night
expid = self.expid
obs.new_exposure('arc', night=night, expid=expid, nspec=4)
simspec = io.findfile('simspec', self.night, self.expid)
fibermap = desispec.io.findfile('fibermap', self.night, self.expid)
opts = ['--simspec', simspec, '--fibermap', fibermap]
args = quickgen.parse(opts)
self.assertEqual(args.simspec, simspec)
self.assertEqual(args.fibermap, fibermap)
def test_qa_frame_write_load_data(self):
# Write
frm0 = self._make_frame()
qafrm0 = QA_Frame(frm0)
# Write
outfile = findfile('qa_data', night=self.nights[0], expid=self.expids[0],
specprod_dir=self.testDir, camera='b0')
write_qa_frame(outfile, qafrm0)
self.files_written.append(outfile)
# Load
qafrm2 = load_qa_frame(outfile, frame_meta=frm0.meta)
assert qafrm2.night == qafrm0.night
def test_qa_frame_write_load_data(self):
# Write
frm0 = self._make_frame()
qafrm0 = QA_Frame(frm0)
# Write
outfile = findfile('qa_data', night=self.nights[0], expid=self.expids[0],
specprod_dir=self.testDir, camera='b0')
write_qa_frame(outfile, qafrm0)
self.files_written.append(outfile)
# Load
qafrm2 = load_qa_frame(outfile, frm0)
assert qafrm2.night == qafrm0.night
def __init__(self, night,flavor,expid,camera,palist,debuglevel=20,period=5.,psfboot=None,wavelength=None, dumpintermediates=True,amps=True,rawdata_dir=None,specprod_dir=None, outdir=None,timeout=120., fiberflat=None,outputfile=None,qlf=False):
"""
psfboot- does not seem to have a desispec.io.findfile entry, so passing this in argument.
May be this will be useful even so.
palist: Palist object. See class Palist below
Note: fiberflat will have a different expid. Passing the file directly in the path
"""
self.night=night
self.expid=expid
self.flavor=flavor
self.camera=camera
self.psfboot=psfboot
self.fiberflat=fiberflat
self.outputfile=outputfile #- final outputfile.
self.wavelength=wavelength
self.debuglevel=debuglevel
self.period=period
self.dumpintermediates=dumpintermediates
self.amps=amps
if rawdata_dir is None:
rawdata_dir=os.getenv('DESI_SPECTRO_DATA')
self.rawdata_dir=rawdata_dir
if specprod_dir is None:
specprod_dir=os.path.join(os.getenv('DESI_SPECTRO_REDUX'), os.getenv('SPECPROD'))
self.specprod_dir=specprod_dir
self.outdir=outdir
self.timeout=timeout
self._palist=palist
self.pamodule=palist.pamodule
self.qamodule=palist.qamodule
self._qlf=qlf
#- some global variables
self.rawfile=findfile("raw",night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir)
self.fibermap=findfile("fibermap", night=self.night,expid=self.expid,camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir)
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap),
expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError(
'newexp-random failed for {} exposure {}'.format(
program, expid))
cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
outputs.append(io.findfile('raw', night, expid))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(
program, expid))
return
def read_frame(filename, nspec=None):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file, or (night, expid, camera) tuple where
night = string YEARMMDD
expid = integer exposure ID
camera = b0, r1, .. z9
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('frame', night, expid, camera)
if not os.path.isfile(filename):
raise IOError("cannot open" + filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8'))
if 'FIBERMAP' in fx:
fibermap = fx['FIBERMAP'].data
else:
fibermap = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
resolution_data = resolution_data[0:nspec]
# return flux,ivar,wave,resolution_data, hdr
return Frame(wave,
flux,
ivar,
mask,
resolution_data,
meta=hdr,
fibermap=fibermap)
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))
cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
#outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
return
def test_quickgen_airmass_simspec(self):
night = self.night
camera = 'r0'
expid0 = 100
expid1 = 101
# generate exposures of varying airmass
obscond = simexp.reference_conditions['DARK']
obscond['AIRMASS'] = 1.5
obs.new_exposure('dark',
night=night,
expid=expid0,
nspec=1,
seed=1,
obsconditions=obscond)
simspec0 = io.findfile('simspec', night, expid0)
fibermap0 = desispec.io.findfile('fibermap', night, expid0)
obscond['AIRMASS'] = 1.0
obs.new_exposure('dark',
night=night,
expid=expid1,
nspec=1,
seed=1,
obsconditions=obscond)
simspec1 = io.findfile('simspec', night, expid1)
fibermap1 = desispec.io.findfile('fibermap', night, expid1)
# generate quickgen output for each airmass
cmd = "quickgen --simspec {} --fibermap {}".format(simspec0, fibermap0)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cmd = "quickgen --simspec {} --fibermap {}".format(simspec1, fibermap1)
quickgen.main(quickgen.parse(cmd.split()[1:]))
cframe0 = desispec.io.findfile("cframe", night, expid0, camera)
cframe1 = desispec.io.findfile("cframe", night, expid1, camera)
cf0 = desispec.io.read_frame(cframe0)
cf1 = desispec.io.read_frame(cframe1)
self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def batch_run(infile, outdir, cameras, queue, batchtime, batchopts):
"""Submits batch job to `nightwatch run infile outdir ...`
Args:
infile (str): input DESI raw data file
outdir (str): base output directory
cameras (list or None): None, or list of cameras to include
queue (str): slurm queue name
batchtime (int): batch job time limit [minutes]
batchopts (str): additional batch options
Returns error code from sbatch submission
Note: this is a non-blocking call and will return before the batch
processing is finished
"""
night, expid = io.get_night_expid(infile)
expdir = io.findfile('expdir', night=night, expid=expid, basedir=outdir)
infile = os.path.abspath(infile)
expdir = os.path.abspath(expdir)
outdir = os.path.abspath(outdir)
if cameras is None:
camera_options = ""
elif isinstance(cameras, (list, tuple)):
camera_options = "--cameras {}".format(','.join(cameras))
elif isinstance(cameras, str):
camera_options = f"--cameras {cameras}"
else:
raise ValueError('Unable to parse cameras {}'.format(cameras))
jobname = f'nightwatch-{expid:08d}'
batchfile = f'{expdir}/{jobname}.slurm'
with open(batchfile, 'w') as fx:
fx.write(f"""#!/bin/bash -l
#SBATCH {batchopts}
#SBATCH --qos {queue}
#SBATCH --time {batchtime}
#SBATCH --job-name {jobname}
#SBATCH --output {expdir}/{jobname}-%j.joblog
#SBATCH --exclusive
nightwatch run --infile {infile} --outdir {outdir} {camera_options}
""")
err = subprocess.call(["sbatch", batchfile])
return err
def io_qa_pa(self, paname):
"""
Specify the filenames: yaml and png of the pa level qa files"
"""
filemap = {
'Initialize': 'ql_initial',
'Preproc': 'ql_preproc',
'BoxcarExtract': 'ql_boxextract',
'ApplyFiberFlat_QL': 'ql_fiberflat',
'SkySub_QL': 'ql_skysub'
}
if paname in filemap:
filetype = filemap[paname] + '_file'
figtype = filemap[paname] + '_fig'
else:
raise IOError(
"PA name does not match any file type. Check PA name in config for {}"
.format(paname))
outfile = findfile(filetype,
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir,
outdir=self.outdir)
outfig = findfile(figtype,
night=self.night,
expid=self.expid,
camera=self.camera,
rawdata_dir=self.rawdata_dir,
specprod_dir=self.specprod_dir,
outdir=self.outdir)
return (outfile, outfig)
def read_qa_frame(filename):
"""Generate a QA_Frame object from a data file
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, basestring):
night, expid, camera = filename
filename = findfile('qa', night, expid, camera)
# Read
qa_data = read_qa_data(filename)
# Instantiate
qaframe = QA_Frame(qa_data)
return qaframe
def dump_pa(self,paname):
"""
dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case.
"""
pafilemap={'Preproc': 'preproc', 'Flexure': None, 'BoxcarExtract': 'frame','ResolutionFit': None, 'Extract_QP': 'qframe', 'ComputeFiberflat_QL': 'fiberflat', 'ComputeFiberflat_QP': 'fiberflat', 'ApplyFiberFlat_QL': 'fframe', 'ApplyFiberFlat_QP': 'fframe', 'SkySub_QL': 'sframe', 'SkySub_QP': 'sframe', 'ApplyFluxCalibration': 'cframe'}
if paname in pafilemap:
filetype=pafilemap[paname]
else:
raise IOError("PA name does not match any file type. Check PA name in config")
pafile=None
if filetype is not None:
pafile=findfile(filetype,night=self.night,expid=self.expid,camera=self.camera,rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir)
return pafile
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c + str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask
is not None):
if len(args.cameras) > 1:
raise ValueError(
'must use only one camera with --bias, --pixflat, --mask options'
)
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile,
camera,
bias=args.bias,
pixflat=args.pixflat,
mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix',
night=night,
expid=expid,
camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def test_parse(self):
night = self.night
expid = self.expid
obs.new_exposure('dark', night=night, expid=expid, nspec=4)
simspec = io.findfile('simspec', self.night, self.expid)
fibermap = desispec.io.findfile('fibermap', self.night, self.expid)
opts = ['--simspec', simspec, '--fibermap', fibermap]
opts += ['--spectrograph', '3', '--config', 'desi']
args = quickgen.parse(opts)
self.assertEqual(args.simspec, simspec)
self.assertEqual(args.fibermap, fibermap)
self.assertEqual(args.spectrograph, 3)
self.assertEqual(args.config, 'desi')
with self.assertRaises(SystemExit):
quickgen.parse([])
def read_sky(filename) :
"""Read sky model and return SkyModel object with attributes
wave, flux, ivar, mask, header.
skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave]
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('sky', night, expid, camera)
hdr = fits.getheader(filename, 0)
wave = native_endian(fits.getdata(filename, "WAVELENGTH"))
skyflux = native_endian(fits.getdata(filename, "SKY"))
ivar = native_endian(fits.getdata(filename, "IVAR"))
mask = native_endian(fits.getdata(filename, "MASK", uint=True))
skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr)
return skymodel
def tearDown(self):
for expid in [self.expid, 100, 101, 102]:
fibermap = desispec.io.findfile('fibermap', self.night, expid)
if os.path.exists(fibermap):
os.remove(fibermap)
simspecfile = io.findfile('simspec', self.night, expid)
if os.path.exists(simspecfile):
os.remove(simspecfile)
for camera in ('b0', 'r0', 'z0'):
framefile = desispec.io.findfile('frame', self.night, expid, camera=camera)
if os.path.exists(framefile):
os.remove(framefile)
cframefile = desispec.io.findfile('cframe', self.night, expid, camera=camera)
if os.path.exists(cframefile):
os.remove(cframefile)
skyfile = desispec.io.findfile('sky', self.night, expid, camera=camera)
if os.path.exists(skyfile):
os.remove(skyfile)
fluxcalibfile = desispec.io.findfile('calib', self.night, expid, camera=camera)
if os.path.exists(fluxcalibfile):
os.remove(fluxcalibfile)
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --pixflat, --mask options')
if (args.pixfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --pixfile option')
if args.outdir is None:
args.outdir = os.getcwd()
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=args.bias, pixflat=args.pixflat, mask=args.mask)
except IOError:
log.error('Camera {} not in {}'.format(camera, args.infile))
continue
if args.pixfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
pixfile = io.findfile('pix', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
pixfile = args.pixfile
io.write_image(pixfile, img)
def read_sky(filename) :
"""Read sky model and return SkyModel object with attributes
wave, flux, ivar, mask, header.
skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave]
"""
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, (str, unicode)):
night, expid, camera = filename
filename = findfile('sky', night, expid, camera)
fx = fits.open(filename, memmap=False, uint=True)
hdr = fx[0].header
wave = native_endian(fx["WAVELENGTH"].data.astype('f8'))
skyflux = native_endian(fx["SKY"].data.astype('f8'))
ivar = native_endian(fx["IVAR"].data.astype('f8'))
mask = native_endian(fx["MASK"].data)
fx.close()
skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr)
return skymodel
def qafile_from_framefile(frame_file, qaprod_dir=None, output_dir=None):
""" Derive the QA filename from an input frame file
Args:
frame_file: str
output_dir: str, optional Over-ride default output path
qa_dir: str, optional Over-ride default QA
Returns:
"""
frame_meta = read_meta_frame(frame_file)
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
if frame_meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
# Name
qafile = findfile(qatype, night=night, camera=camera, expid=expid,
outdir=output_dir, qaprod_dir=qaprod_dir)
# Return
return qafile, qatype
def runpipeline(pl,convdict,conf):
"""
Runs the quicklook pipeline as configured
Args:
pl: is a list of [pa,qas] where pa is a pipeline step and qas the corresponding
qas for that pa
convdict: converted dictionary e.g : conf["IMAGE"] is the real psf file
but convdict["IMAGE"] is like desispec.image.Image object and so on.
details in setup_pipeline method below for examples.
conf: a configured dictionary, read from the configuration yaml file.
e.g: conf=configdict=yaml.load(open('configfile.yaml','rb'))
"""
qlog=qllogger.QLLogger()
log=qlog.getlog()
hb=QLHB.QLHeartbeat(log,conf["Period"],conf["Timeout"])
inp=convdict["rawimage"]
singqa=conf["singleqa"]
paconf=conf["PipeLine"]
qlog=qllogger.QLLogger()
log=qlog.getlog()
passqadict=None #- pass this dict to QAs downstream
schemaMerger=QL_QAMerger(conf['Night'],conf['Expid'],conf['Flavor'],conf['Camera'],conf['Program'],convdict)
QAresults=[]
if singqa is None:
for s,step in enumerate(pl):
log.info("Starting to run step {}".format(paconf[s]["StepName"]))
pa=step[0]
pargs=mapkeywords(step[0].config["kwargs"],convdict)
schemaStep=schemaMerger.addPipelineStep(paconf[s]["StepName"])
try:
hb.start("Running {}".format(step[0].name))
oldinp=inp #- copy for QAs that need to see earlier input
inp=pa(inp,**pargs)
if step[0].name == 'Initialize':
schemaStep.addMetrics(inp[1])
except Exception as e:
log.critical("Failed to run PA {} error was {}".format(step[0].name,e),exc_info=True)
sys.exit("Failed to run PA {}".format(step[0].name))
qaresult={}
for qa in step[1]:
try:
qargs=mapkeywords(qa.config["kwargs"],convdict)
hb.start("Running {}".format(qa.name))
qargs["dict_countbins"]=passqadict #- pass this to all QA downstream
if qa.name=="RESIDUAL" or qa.name=="Sky_Residual":
res=qa(inp[0],inp[1],**qargs)
else:
if isinstance(inp,tuple):
res=qa(inp[0],**qargs)
else:
res=qa(inp,**qargs)
if qa.name=="COUNTBINS" or qa.name=="CountSpectralBins":
passqadict=res
if "qafile" in qargs:
qawriter.write_qa_ql(qargs["qafile"],res)
log.debug("{} {}".format(qa.name,inp))
qaresult[qa.name]=res
schemaStep.addParams(res['PARAMS'])
schemaStep.addMetrics(res['METRICS'])
except Exception as e:
log.warning("Failed to run QA {}. Got Exception {}".format(qa.name,e),exc_info=True)
hb.stop("Step {} finished.".format(paconf[s]["StepName"]))
QAresults.append([pa.name,qaresult])
hb.stop("Pipeline processing finished. Serializing result")
else:
import numpy as np
qa=None
qas=[[],['Bias_From_Overscan','Get_RMS','Count_Pixels','Calc_XWSigma'],'Trace_Shifts','CountSpectralBins',['Sky_Continuum','Sky_Peaks'],['Calculate_SNR'],['Sky_Rband','Integrate_Spec']]
singleqaperpa=['Bias_From_Overscan','Check_HDUs','Trace_Shifts','CountSpectralBins']
for palg in range(len(qas)):
if singqa in qas[palg]:
pa=pl[palg][0]
pac=paconf[palg]
if singqa in singleqaperpa:
qa = pl[palg][1][0]
else:
for qalg in range(len(qas[palg])):
if qas[palg][qalg] == singqa:
qa=pl[palg][1][qalg]
if qa is None:
log.critical("Unknown input QA... Valid QAs are: {}".format(qas))
sys.exit()
log.info("Starting to run step {}".format(pac["StepName"]))
pargs=mapkeywords(pa.config["kwargs"],convdict)
schemaStep=schemaMerger.addPipelineStep(pac["StepName"])
qaresult={}
try:
qargs=mapkeywords(qa.config["kwargs"],convdict)
hb.start("Running {}".format(qa.name))
if singqa=="Sky_Residual":
res=qa(inp[0],inp[1],**qargs)
else:
if isinstance(inp,tuple):
res=qa(inp[0],**qargs)
else:
res=qa(inp,**qargs)
if singqa=="CountSpectralBins":
passqadict=res
if "qafile" in qargs:
qawriter.write_qa_ql(qargs["qafile"],res)
log.debug("{} {}".format(qa.name,inp))
schemaStep.addMetrics(res['METRICS'])
except Exception as e:
log.warning("Failed to run QA {}. Got Exception {}".format(qa.name,e),exc_info=True)
if len(qaresult):
if conf["DumpIntermediates"]:
f = open(pac["OutputFile"],"w")
f.write(yaml.dump(yamlify(qaresult)))
log.info("{} finished".format(qa.name))
#- merge QAs for this pipeline execution
#- RS: don't write merged file if running single QA
if singqa is None:
log.debug("Dumping mergedQAs")
from desispec.io import findfile
specprod_dir=os.environ['QL_SPEC_REDUX'] if 'QL_SPEC_REDUX' in os.environ else ""
destFile=findfile('ql_mergedQA_file',night=conf['Night'],
expid=conf['Expid'],
camera=conf['Camera'],
specprod_dir=specprod_dir)
schemaMerger.writeTojsonFile(destFile)
log.info("Wrote merged QA file {}".format(destFile))
if isinstance(inp,tuple):
return inp[0]
else:
return inp
parser = optparse.OptionParser(usage = "%prog [options]")
parser.add_option("-b", "--brick", type=str, help="input brickname")
parser.add_option("-n", "--nspec", type=int, help="number of spectra to fit [default: all]")
parser.add_option("-o", "--outfile", type=str, help="output file name")
parser.add_option("--zspec", help="also include spectra in output file", action="store_true")
opts, args = parser.parse_args()
log = get_logger()
#- Read brick files for each channel
log.info("Reading bricks")
brick = dict()
for channel in ('b', 'r', 'z'):
filename = io.findfile('brick', band=channel, brickid=opts.brick)
brick[channel] = io.Brick(filename)
#- Assume all channels have the same number of targets
#- TODO: generalize this to allow missing channels
if opts.nspec is None:
opts.nspec = brick['b'].get_num_targets()
log.info("Fitting {} targets".format(opts.nspec))
else:
log.info("Fitting {} of {} targets".format(opts.nspec, brick['b'].get_num_targets()))
nspec = opts.nspec
#- Coadd individual exposures and combine channels
#- Full coadd code is a bit slow, so try something quick and dirty for
#- now to get something going for redshifting
def main() :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID')
parser.add_argument('--fibermap', type = str, help = 'path of fibermap file')
parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits')
parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9')
parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux')
args = parser.parse_args()
log = get_logger()
# Call necessary environment variables. No need if add argument to give full file path.
if 'DESI_SPECTRO_REDUX' not in os.environ:
raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles')
DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX']
PRODNAME=os.environ['PRODNAME']
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration')
DESISIM=os.environ['DESISIM'] # to read the filter transmission files
if args.fibermap is None or args.models is None or \
args.spectrograph is None or args.outfile is None or \
args.fiberflatexpid is None:
log.critical('Missing a required argument')
parser.print_help()
sys.exit(12)
# read Standard Stars from the fibermap file
# returns the Fiber id, filter names and mags for the standard stars
fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True)
#- Trim to just fibers on this spectrograph
ii = (500*args.spectrograph <= fiber_tbdata["FIBER"])
ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1))
fiber_tbdata = fiber_tbdata[ii]
#- Get info for the standard stars
refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD")
refFibers=fiber_tbdata["FIBER"][refStarIdx]
refFilters=fiber_tbdata["FILTER"][refStarIdx]
refMags=fiber_tbdata["MAG"]
fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags}
NIGHT=fiber_header['NIGHT']
EXPID=fiber_header['EXPID']
filters=fibers["FILTER"]
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. Can not find filter response files')
basepath=DESISIM+"/data/"
#now load all the skyfiles, framefiles, fiberflatfiles etc
# all three channels files are simultaneously treated for model fitting
skyfile={}
framefile={}
fiberflatfile={}
for i in ["b","r","z"]:
camera = i+str(args.spectrograph)
skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera)
framefile[i] = io.findfile('frame', NIGHT, EXPID, camera)
fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera)
#Read Frames, Flats and Sky files
frameFlux={}
frameIvar={}
frameWave={}
frameResolution={}
framehdr={}
fiberFlat={}
ivarFlat={}
maskFlat={}
meanspecFlat={}
waveFlat={}
headerFlat={}
sky={}
skyivar={}
skymask={}
skywave={}
skyhdr={}
for i in ["b","r","z"]:
#arg=(night,expid,'%s%s'%(i,spectrograph))
#- minimal code change for refactored I/O, while not taking advantage of simplified structure
frame = io.read_frame(framefile[i])
frameFlux[i] = frame.flux
frameIvar[i] = frame.ivar
frameWave[i] = frame.wave
frameResolution[i] = frame.resolution_data
framehdr[i] = frame.header
ff = io.read_fiberflat(fiberflatfile[i])
fiberFlat[i] = ff.fiberflat
ivarFlat[i] = ff.ivar
maskFlat[i] = ff.mask
meanspecFlat[i] = ff.meanspec
waveFlat[i] = ff.wave
headerFlat[i] = ff.header
skymodel = io.read_sky(skyfile[i])
sky[i] = skymodel.flux
skyivar[i] = skymodel.ivar
skymask[i] = skymodel.mask
skywave[i] = skymodel.wave
skyhdr[i] = skymodel.header
# Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract
convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]}
# Read the standard Star data and divide by flat and subtract sky
stars=[]
ivars=[]
for i in fibers["FIBER"]:
#flat and sky should have same wavelength binning as data, otherwise should be rebinned.
stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]],
"r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]],
"z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i]))
ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]}))
stdwave,stdflux,templateid=io.read_stdstar_templates(args.models)
#- Trim standard star wavelengths to just the range we need
minwave = min([min(w) for w in frameWave.values()])
maxwave = max([max(w) for w in frameWave.values()])
ii = (minwave-10 < stdwave) & (stdwave < maxwave+10)
stdwave = stdwave[ii]
stdflux = stdflux[:, ii]
log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars)))
if len(stars) == 0:
log.critical("No standard stars! Exiting")
sys.exit(1)
# Now for each star, find the best model and normalize.
normflux=[]
bestModelIndex=np.arange(len(stars))
templateID=np.arange(len(stars))
chi2dof=np.zeros(len(stars))
#- TODO: don't use 'l' as a variable name. Can look like a '1'
for k,l in enumerate(stars):
log.info("checking best model for star {0}".format(l[0]))
starindex=l[0]
mags=l[2]
filters=fibers["FILTER"][k]
rflux=stars[k][1]["r"][0]
bflux=stars[k][1]["b"][0]
zflux=stars[k][1]["z"][0]
flux={"b":bflux,"r":rflux,"z":zflux}
#print ivars
rivar=ivars[k][1]["r"][0]
bivar=ivars[k][1]["b"][0]
zivar=ivars[k][1]["z"][0]
ivar={"b":bivar,"r":rivar,"z":zivar}
resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]}
# Now find the best Model
bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux)
log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k]))
# Normalize the best model using reported magnitude
modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath)
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
stdfibers=fibers["FIBER"]
data={}
data['BESTMODEL']=bestModelIndex
data['CHI2DOF']=chi2dof
data['TEMPLATEID']=templateid[bestModelIndex]
norm_model_file=args.outfile
io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = get_logger()
#- YEARMMDD string, rolls over at noon not midnight
if night is None:
night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
#- check for required environment variables
check_env()
#- parameter dictionary that will later be used for formatting commands
params = dict(night=night, nspec=nspec)
#-----
#- Input fibermaps, spectra, and pixel-level raw data
for expid, flavor in zip([0,1,2], ['flat', 'arc', 'science']):
cmd = "pixsim-desi --newexp {flavor} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, flavor=flavor, **params)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid))
cmd = "pixsim-desi --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params)
inputs = [fibermap, simspec]
outputs = list()
for camera in ['b0', 'r0', 'z0']:
pixfile = io.findfile('pix', night, expid, camera)
outputs.append(pixfile)
outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-')))
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid))
#-----
#- Extract
waverange = dict(
b = "3570,5940,1.0",
r = "5630,7740,1.0",
z = "7440,9830,1.0",
)
for expid in [0,1,2]:
for channel in ['b', 'r', 'z']:
camera = channel+'0'
pixfile = io.findfile('pix', night, expid, camera)
psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel)
framefile = io.findfile('frame', night, expid, camera)
cmd = "exspec -i {pix} -p {psf} --specrange 0,{nspec} -w {wave} -o {frame}".format(
pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params)
inputs = [pixfile, psffile]
outputs = [framefile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid))
#-----
#- Fiber flat
expid = 0
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fiberflat = io.findfile('fiberflat', night, expid, camera)
cmd = "desi_compute_fiberflat.py --infile {frame} --outfile {fiberflat}".format(
frame=framefile, fiberflat=fiberflat, **params)
inputs = [framefile,]
outputs = [fiberflat,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('fiberflat failed for '+camera)
#-----
#- Sky model
flat_expid = 0
expid = 2
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
cmd="desi_compute_sky.py --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --outfile {sky}".format(
frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, **params)
inputs = [framefile, fibermap, fiberflat]
outputs = [skyfile, ]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('sky model failed for '+camera)
#-----
#- Fit standard stars
if 'STD_TEMPLATES' in os.environ:
std_templates = os.getenv('STD_TEMPLATES')
else:
std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/stellar_templates/v1.0/stdstar_templates_v1.0.fits'
stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0)
cmd = """desi_fit_stdstars.py --spectrograph 0 \
--fibermap {fibermap} \
--fiberflatexpid {flat_expid} \
--models {std_templates} --outfile {stdstars}""".format(
flat_expid=flat_expid, fibermap=fibermap, std_templates=std_templates,
stdstars=stdstarfile)
inputs = [fibermap, std_templates]
outputs = [stdstarfile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('fitting stdstars failed')
#-----
#- Flux calibration
for channel in ['b', 'r', 'z']:
camera = channel+"0"
framefile = io.findfile('frame', night, expid, camera)
fibermap = io.findfile('fibermap', night, expid)
fiberflat = io.findfile('fiberflat', night, flat_expid, camera)
skyfile = io.findfile('sky', night, expid, camera)
calibfile = io.findfile('calib', night, expid, camera)
#- Compute flux calibration vector
cmd = """desi_compute_fluxcalibration.py \
--infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --sky {sky} \
--models {stdstars} --outfile {calib}""".format(
frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile,
stdstars=stdstarfile, calib=calibfile,
)
inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile]
outputs = [calibfile,]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('flux calibration failed for '+camera)
#- Apply the flux calibration to write a cframe file
cframefile = io.findfile('cframe', night, expid, camera)
cmd = """desi_process_exposure.py \
--infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \
--outfile {cframe}""".format(frame=framefile, fibermap=fibermap,
fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile)
inputs = [framefile, fiberflat, skyfile, calibfile]
outputs = [cframefile, ]
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('combining calibration steps failed for '+camera)
#-----
#- Bricks
inputs = list()
for camera in ['b0', 'r0', 'z0']:
inputs.append( io.findfile('cframe', night, expid, camera) )
outputs = list()
fibermap = io.read_fibermap(io.findfile('fibermap', night, expid))
bricks = set(fibermap['BRICKNAME'])
for b in bricks:
for channel in ['b', 'r', 'z']:
outputs.append( io.findfile('brick', brickid=b, band=channel))
cmd = "desi_make_bricks.py --night "+night
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('brick generation failed')
#-----
#- Redshifts!
for b in bricks:
inputs = [io.findfile('brick', brickid=b, band=channel) for channel in ['b', 'r', 'z']]
zbestfile = io.findfile('zbest', brickid=b)
outputs = [zbestfile, ]
cmd = "desi_zfind.py --brick {} -o {}".format(b, zbestfile)
if runcmd(cmd, inputs, outputs, clobber) != 0:
raise RuntimeError('redshifts failed for brick '+b)
#-----
#- Did it work?
#- (this combination of fibermap, simspec, and zbest is a pain)
simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid))
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickid=b))
for i in range(len(zbest.z)):
if zbest.type[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.type[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.type[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5*(z-truez)/(1+truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype == 'STD' and objtype == 'STAR':
status = 'ok'
else:
status = 'oops'
else:
status = 'oops'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
import argparse
parser = argparse.ArgumentParser(usage = "{prog} [options]")
# parser.add_argument("-i", "--input", type=str, help="input data")
# parser.add_argument("-o", "--output", type=str, help="output data")
parser.add_argument("--skip-psf", action="store_true", help="Skip PSF fitting step")
args = parser.parse_args()
log = logging.get_logger()
# YEARMMDD string, rolls over at noon not midnight
if night is None:
night = "20160726"
# check for required environment variables
check_env()
# simulate inputs
sim(night, nspec=nspec, clobber=clobber)
# raw and production locations
rawdir = os.path.abspath(io.rawdata_root())
proddir = os.path.abspath(io.specprod_root())
# create production
if clobber and os.path.isdir(proddir):
shutil.rmtree(proddir)
dbfile = io.get_pipe_database()
if not os.path.exists(dbfile):
com = "desi_pipe create --db-sqlite"
log.info('Running {}'.format(com))
sp.check_call(com, shell=True)
else:
log.info("Using pre-existing production database {}".format(dbfile))
# Modify options file to restrict the spectral range
optpath = os.path.join(proddir, "run", "options.yaml")
opts = pipe.prod.yaml_read(optpath)
opts['extract']['specmin'] = 0
opts['extract']['nspec'] = nspec
opts['psf']['specmin'] = 0
opts['psf']['nspec'] = nspec
opts['traceshift']['nfibers'] = nspec
pipe.prod.yaml_write(optpath, opts)
if args.skip_psf:
#- Copy desimodel psf into this production instead of fitting psf
import shutil
for channel in ['b', 'r', 'z']:
refpsf = '{}/data/specpsf/psf-{}.fits'.format(
os.getenv('DESIMODEL'), channel)
nightpsf = io.findfile('psfnight', night, camera=channel+'0')
shutil.copy(refpsf, nightpsf)
for expid in [0,1,2]:
exppsf = io.findfile('psf', night, expid, camera=channel+'0')
shutil.copy(refpsf, exppsf)
#- Resync database to current state
dbpath = io.get_pipe_database()
db = pipe.load_db(dbpath, mode="w")
db.sync(night)
# Run the pipeline tasks in order
from desispec.pipeline.tasks.base import default_task_chain
for tasktype in default_task_chain:
#- if we skip psf/psfnight/traceshift, update state prior to extractions
if tasktype == 'traceshift' and args.skip_psf:
db.getready()
run_pipeline_step(tasktype)
# #-----
# #- Did it work?
# #- (this combination of fibermap, simspec, and zbest is a pain)
expid = 2
fmfile = io.findfile('fibermap', night=night, expid=expid)
fibermap = io.read_fibermap(fmfile)
simdir = os.path.dirname(fmfile)
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'TRUTH')
try:
elginfo = fits.getdata(simspec, 'TRUTH_ELG')
except:
elginfo = None
from desimodel.footprint import radec2pix
nside=64
pixels = np.unique(radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC']))
num_missing = 0
for pix in pixels:
zfile = io.findfile('zbest', groupname=pix)
if not os.path.exists(zfile):
log.error('Missing {}'.format(zfile))
num_missing += 1
if num_missing > 0:
log.critical('{} zbest files missing'.format(num_missing))
sys.exit(1)
print()
print("--------------------------------------------------")
print("Pixel True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for pix in pixels:
zfile = io.findfile('zbest', groupname=pix)
if not os.path.exists(zfile):
log.error('Missing {}'.format(zfile))
continue
zfx = fits.open(zfile, memmap=False)
zbest = zfx['ZBEST'].data
for i in range(len(zbest['Z'])):
objtype = zbest['SPECTYPE'][i]
z, zwarn = zbest['Z'][i], zbest['ZWARN'][i]
j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0]
truetype = siminfo['OBJTYPE'][j]
oiiflux = 0.0
if truetype == 'ELG':
k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0]
oiiflux = elginfo['OIIFLUX'][k]
truez = siminfo['REDSHIFT'][j]
dv = C_LIGHT*(z-truez)/(1+truez)
status = None
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GALAXY':
if abs(dv) < 150:
status = 'ok'
elif oiiflux < 8e-17:
status = 'ok ([OII] flux {:.2g})'.format(oiiflux)
else:
status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux)
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype in ('STD', 'FSTD') and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
pix, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def main(args=None):
if args is None:
args = parse()
elif isinstance(args, (list, tuple)):
args = parse(args)
bias=True
if args.bias : bias=args.bias
if args.nobias : bias=False
dark=True
if args.dark : dark=args.dark
if args.nodark : dark=False
pixflat=True
if args.pixflat : pixflat=args.pixflat
if args.nopixflat : pixflat=False
mask=True
if args.mask : mask=args.mask
if args.nomask : mask=False
if args.cameras is None:
args.cameras = [c+str(i) for c in 'brz' for i in range(10)]
else:
args.cameras = args.cameras.split(',')
if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None):
if len(args.cameras) > 1:
raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options')
if (args.pixfile is not None):
log.warning('--pixfile is deprecated; please use --outfile instead')
if args.outfile is None:
args.outfile = args.pixfile
else:
log.critical("Set --outfile not --pixfile and certainly not both")
sys.exit(1)
if (args.outfile is not None) and len(args.cameras) > 1:
raise ValueError('must use only one camera with --outfile option')
if args.outdir is None:
args.outdir = os.getcwd()
log.warning('--outdir not specified; using {}'.format(args.outdir))
ccd_calibration_filename = None
if args.no_ccd_calib_filename :
ccd_calibration_filename = False
elif args.ccd_calib_filename is not None :
ccd_calibration_filename = args.ccd_calib_filename
for camera in args.cameras:
try:
img = io.read_raw(args.infile, camera,
bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub,
nocosmic=args.nocosmic,
cosmics_nsig=args.cosmics_nsig,
cosmics_cfudge=args.cosmics_cfudge,
cosmics_c2fudge=args.cosmics_c2fudge,
ccd_calibration_filename=ccd_calibration_filename,
nocrosstalk=args.nocrosstalk,
nogain=args.nogain,
fill_header=args.fill_header
)
except IOError:
log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile))
continue
if(args.zero_masked) :
img.pix *= (img.mask==0)
if args.outfile is None:
night = img.meta['NIGHT']
expid = img.meta['EXPID']
outfile = io.findfile('preproc', night=night, expid=expid, camera=camera,
outdir=args.outdir)
else:
outfile = args.outfile
io.write_image(outfile, img)
def main(args, comm=None) :
log = get_logger()
if args.npoly < 0 :
log.warning("Need npoly>=0, changing this %d -> 1"%args.npoly)
args.npoly=0
if args.nproc < 1 :
log.warning("Need nproc>=1, changing this %d -> 1"%args.nproc)
args.nproc=1
if comm is not None:
if args.nproc != 1:
if comm.rank == 0:
log.warning("Using MPI, forcing multiprocessing nproc -> 1")
args.nproc = 1
if args.objtype is not None:
args.objtype = args.objtype.split(',')
#- Read brick files for each channel
if (comm is None) or (comm.rank == 0):
log.info("Reading bricks")
brick = dict()
if args.brick is not None:
if len(args.brickfiles) != 0:
raise RuntimeError('Give -b/--brick or input brickfiles but not both')
for channel in ('b', 'r', 'z'):
filename = None
if (comm is None) or (comm.rank == 0):
filename = io.findfile('brick', band=channel, brickname=args.brick,
specprod_dir=args.specprod_dir)
if comm is not None:
filename = comm.bcast(filename, root=0)
brick[channel] = io.Brick(filename)
else:
for filename in args.brickfiles:
bx = io.Brick(filename)
if bx.channel not in brick:
brick[bx.channel] = bx
else:
if (comm is None) or (comm.rank == 0):
log.error('Channel {} in multiple input files'.format(bx.channel))
sys.exit(2)
filters=brick.keys()
for fil in filters:
if (comm is None) or (comm.rank == 0):
log.info("Filter found: "+fil)
#- Assume all channels have the same number of targets
#- TODO: generalize this to allow missing channels
#if args.nspec is None:
# args.nspec = brick['b'].get_num_targets()
# log.info("Fitting {} targets".format(args.nspec))
#else:
# log.info("Fitting {} of {} targets".format(args.nspec, brick['b'].get_num_targets()))
#- Coadd individual exposures and combine channels
#- Full coadd code is a bit slow, so try something quick and dirty for
#- now to get something going for redshifting
if (comm is None) or (comm.rank == 0):
log.info("Combining individual channels and exposures")
wave=[]
for fil in filters:
wave=np.concatenate([wave,brick[fil].get_wavelength_grid()])
np.ndarray.sort(wave)
nwave = len(wave)
#- flux and ivar arrays to fill for all targets
#flux = np.zeros((nspec, nwave))
#ivar = np.zeros((nspec, nwave))
flux = []
ivar = []
good_targetids=[]
targetids = brick['b'].get_target_ids()
fpinfo = None
if args.print_info is not None:
if (comm is None) or (comm.rank == 0):
fpinfo = open(args.print_info,"w")
for i, targetid in enumerate(targetids):
#- wave, flux, and ivar for this target; concatenate
xwave = list()
xflux = list()
xivar = list()
good=True
for channel in filters:
exp_flux, exp_ivar, resolution, info = brick[channel].get_target(targetid)
weights = np.sum(exp_ivar, axis=0)
ii, = np.where(weights > 0)
if len(ii)==0:
good=False
break
xwave.extend(brick[channel].get_wavelength_grid()[ii])
#- Average multiple exposures on the same wavelength grid for each channel
xflux.extend(np.average(exp_flux[:,ii], weights=exp_ivar[:,ii], axis=0))
xivar.extend(weights[ii])
if not good:
continue
xwave = np.array(xwave)
xivar = np.array(xivar)
xflux = np.array(xflux)
ii = np.argsort(xwave)
#flux[i], ivar[i] = resample_flux(wave, xwave[ii], xflux[ii], xivar[ii])
fl, iv = resample_flux(wave, xwave[ii], xflux[ii], xivar[ii])
flux.append(fl)
ivar.append(iv)
good_targetids.append(targetid)
if not args.print_info is None:
s2n = np.median(fl[:-1]*np.sqrt(iv[:-1])/np.sqrt(wave[1:]-wave[:-1]))
if (comm is None) or (comm.rank == 0):
print targetid,s2n
fpinfo.write(str(targetid)+" "+str(s2n)+"\n")
if not args.print_info is None:
if (comm is None) or (comm.rank == 0):
fpinfo.close()
sys.exit()
good_targetids=good_targetids[args.first_spec:]
flux=np.array(flux[args.first_spec:])
ivar=np.array(ivar[args.first_spec:])
nspec=len(good_targetids)
if (comm is None) or (comm.rank == 0):
log.info("number of good targets = %d"%nspec)
if (args.nspec is not None) and (args.nspec < nspec):
if (comm is None) or (comm.rank == 0):
log.info("Fitting {} of {} targets".format(args.nspec, nspec))
nspec=args.nspec
good_targetids=good_targetids[:nspec]
flux=flux[:nspec]
ivar=ivar[:nspec]
else :
if (comm is None) or (comm.rank == 0):
log.info("Fitting {} targets".format(nspec))
if (comm is None) or (comm.rank == 0):
log.debug("flux.shape={}".format(flux.shape))
zf = None
if comm is None:
# Use multiprocessing built in to RedMonster.
zf = RedMonsterZfind(wave= wave,flux= flux,ivar=ivar,
objtype=args.objtype,zrange_galaxy= args.zrange_galaxy,
zrange_qso=args.zrange_qso,zrange_star=args.zrange_star,
nproc=args.nproc,npoly=args.npoly)
else:
# Use MPI
# distribute the spectra among processes
my_firstspec, my_nspec = dist_uniform(nspec, comm.size, comm.rank)
my_specs = slice(my_firstspec, my_firstspec + my_nspec)
for p in range(comm.size):
if p == comm.rank:
if my_nspec > 0:
log.info("process {} fitting spectra {} - {}".format(p, my_firstspec, my_firstspec+my_nspec-1))
else:
log.info("process {} idle".format(p))
sys.stdout.flush()
comm.barrier()
# do redshift fitting on each process
myzf = None
if my_nspec > 0:
savelevel = os.environ["DESI_LOGLEVEL"]
os.environ["DESI_LOGLEVEL"] = "WARNING"
myzf = RedMonsterZfind(wave=wave, flux=flux[my_specs,:], ivar=ivar[my_specs,:],
objtype=args.objtype,zrange_galaxy= args.zrange_galaxy,
zrange_qso=args.zrange_qso,zrange_star=args.zrange_star,
nproc=args.nproc,npoly=args.npoly)
os.environ["DESI_LOGLEVEL"] = savelevel
# Combine results into a single ZFindBase object on the root process.
# We could do this with a gather, but we are using a small number of
# processes, and point-to-point communication is easier for people to
# understand.
if comm.rank == 0:
zf = ZfindBase(myzf.wave, np.zeros((nspec, myzf.nwave)), np.zeros((nspec, myzf.nwave)), R=None, results=None)
for p in range(comm.size):
if comm.rank == 0:
if p == 0:
# root process copies its own data into output
zf.flux[my_specs] = myzf.flux
zf.ivar[my_specs] = myzf.ivar
zf.model[my_specs] = myzf.model
zf.z[my_specs] = myzf.z
zf.zerr[my_specs] = myzf.zerr
zf.zwarn[my_specs] = myzf.zwarn
zf.spectype[my_specs] = myzf.spectype
zf.subtype[my_specs] = myzf.subtype
else:
# root process receives from process p and copies
# it into the output.
p_nspec = comm.recv(source=p, tag=0)
# only proceed if the sending process actually
# has some spectra assigned to it.
if p_nspec > 0:
p_firstspec = comm.recv(source=p, tag=1)
p_slice = slice(p_firstspec, p_firstspec+p_nspec)
p_flux = comm.recv(source=p, tag=2)
zf.flux[p_slice] = p_flux
p_ivar = comm.recv(source=p, tag=3)
zf.ivar[p_slice] = p_ivar
p_model = comm.recv(source=p, tag=4)
zf.model[p_slice] = p_model
p_z = comm.recv(source=p, tag=5)
zf.z[p_slice] = p_z
p_zerr = comm.recv(source=p, tag=6)
zf.zerr[p_slice] = p_zerr
p_zwarn = comm.recv(source=p, tag=7)
zf.zwarn[p_slice] = p_zwarn
p_type = comm.recv(source=p, tag=8)
zf.spectype[p_slice] = p_type
p_subtype = comm.recv(source=p, tag=9)
zf.subtype[p_slice] = p_subtype
else:
if p == comm.rank:
# process p sends to root
comm.send(my_nspec, dest=0, tag=0)
if my_nspec > 0:
comm.send(my_firstspec, dest=0, tag=1)
comm.send(myzf.flux, dest=0, tag=2)
comm.send(myzf.ivar, dest=0, tag=3)
comm.send(myzf.model, dest=0, tag=4)
comm.send(myzf.z, dest=0, tag=5)
comm.send(myzf.zerr, dest=0, tag=6)
comm.send(myzf.zwarn, dest=0, tag=7)
comm.send(myzf.spectype, dest=0, tag=8)
comm.send(myzf.subtype, dest=0, tag=9)
comm.barrier()
if (comm is None) or (comm.rank == 0):
# The full results exist only on the rank zero process.
# reformat results
dtype = list()
dtype = [
('Z', zf.z.dtype),
('ZERR', zf.zerr.dtype),
('ZWARN', zf.zwarn.dtype),
('SPECTYPE', zf.spectype.dtype),
('SUBTYPE', zf.subtype.dtype),
]
formatted_data = np.empty(nspec, dtype=dtype)
formatted_data['Z'] = zf.z
formatted_data['ZERR'] = zf.zerr
formatted_data['ZWARN'] = zf.zwarn
formatted_data['SPECTYPE'] = zf.spectype
formatted_data['SUBTYPE'] = zf.subtype
# Create a ZfindBase object with formatted results
zfi = ZfindBase(None, None, None, results=formatted_data)
zfi.nspec = nspec
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qabrick = load_qa_brick(args.qafile)
# Run
qabrick.run_qa('ZBEST', (zfi,brick))
# Write
if args.qafile is not None:
write_qa_brick(args.qafile, qabrick)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
raise IOError("Not yet implemented")
qa_plots.brick_zbest(args.qafig, zfi, qabrick)
#- Write some output
if args.outfile is None:
args.outfile = io.findfile('zbest', brickname=args.brick)
log.info("Writing "+args.outfile)
#io.write_zbest(args.outfile, args.brick, targetids, zfi, zspec=args.zspec)
io.write_zbest(args.outfile, args.brick, good_targetids, zfi, zspec=args.zspec)
return
def integration_test(night=None, nspec=5, clobber=False):
"""Run an integration test from raw data simulations through redshifts
Args:
night (str, optional): YEARMMDD, defaults to current night
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
log.setLevel(logging.DEBUG)
# YEARMMDD string, rolls over at noon not midnight
# TODO: fix usage of night to be something other than today
if night is None:
#night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600))
night = "20160726"
# check for required environment variables
check_env()
# simulate inputs
sim(night, nspec=nspec, clobber=clobber)
# create production
# FIXME: someday run PSF estimation too...
### com = "desi_pipe --env env.txt --spectrographs 0 --fakeboot --fakepsf"
rawdir = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf --raw {}".format(rawdir)
sp.check_call(com, shell=True)
# raw and production locations
rawdir = os.path.abspath(io.rawdata_root())
proddir = os.path.abspath(io.specprod_root())
# Modify options file to restrict the spectral range
optpath = os.path.join(proddir, "run", "options.yaml")
opts = pipe.read_options(optpath)
opts['extract']['specmin'] = 0
opts['extract']['nspec'] = nspec
pipe.write_options(optpath, opts)
# run the generated shell scripts
# FIXME: someday run PSF estimation too...
# print("Running bootcalib script...")
# com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh")
# sp.check_call(["bash", com])
# print("Running specex script...")
# com = os.path.join(proddir, "run", "scripts", "specex_all.sh")
# sp.check_call(["bash", com])
# print("Running psfcombine script...")
# com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh")
# sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "extract_all.sh")
print("Running extraction script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "fiberflat-procexp_all.sh")
print("Running calibration script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "bricks.sh")
print("Running makebricks script "+com)
sp.check_call(["bash", com])
com = os.path.join(proddir, "run", "scripts", "zfind_all.sh")
print("Running zfind script "+com)
sp.check_call(["bash", com])
# #-----
# #- Did it work?
# #- (this combination of fibermap, simspec, and zbest is a pain)
expid = 2
fmfile = io.findfile('fibermap', night=night, expid=expid)
fibermap = io.read_fibermap(fmfile)
simdir = os.path.dirname(fmfile)
simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid)
siminfo = fits.getdata(simspec, 'METADATA')
brickdirs = glob.glob(os.path.join(proddir, "bricks", "*"))
bricks = [ os.path.basename(x) for x in brickdirs ]
print()
print("--------------------------------------------------")
print("Brick True z -> Class z zwarn")
# print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok")
for b in bricks:
zbest = io.read_zbest(io.findfile('zbest', brickname=b))
for i in range(len(zbest.z)):
if zbest.spectype[i] == 'ssp_em_galaxy':
objtype = 'GAL'
elif zbest.spectype[i] == 'spEigenStar':
objtype = 'STAR'
else:
objtype = zbest.spectype[i]
z, zwarn = zbest.z[i], zbest.zwarn[i]
j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0]
truetype = siminfo['OBJTYPE'][j]
truez = siminfo['REDSHIFT'][j]
dv = 3e5*(z-truez)/(1+truez)
if truetype == 'SKY' and zwarn > 0:
status = 'ok'
elif zwarn == 0:
if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150:
status = 'ok'
elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750:
status = 'ok'
elif truetype == 'STD' and objtype == 'STAR':
status = 'ok'
else:
status = 'OOPS'
else:
status = 'OOPS'
print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format(
b, truetype, truez, objtype, z, zwarn, status))
print("--------------------------------------------------")
def load_all_s2n_values(nights, channel, sub_exposures=None):
"""
Calculate S/N values for a set of spectra from an input list of nights
Args:
nights: list
channel: str ('b','r','z')
sub_exposures:
Returns:
fdict: dict
Contains all the S/N info for all nights in the given channel
"""
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night)#, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
log.debug('Getting truth from {}'.format(simspec_file))
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data,masked=True)
#- Get OIIFLUX from separate HDU and join
if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu):
elg_truth = Table(sps_hdu['TRUTH_ELG'].data)
sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'],
keys='TARGETID', join_type='left')
else:
sps_tab['OIIFLUX'] = 0.0
sps_hdu.close()
#objs = sps_tab['TEMPLATETYPE'] == objtype
#if np.sum(objs) == 0:
# continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel+str(ii)
cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera)
try:
log.debug('Reading from {}'.format(cframe_path))
cframe = read_frame(cframe_path)
except (IOError, OSError):
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave,1)
dwave[0] = dwave[1]
# Calculate
s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave)
#s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave)
# Save
fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers])
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers])
fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
def main(args) :
# imports
import glob
from desispec.io import findfile, makepath
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import get_reduced_frames
from desispec.io import specprod_root
from desispec.io import qaprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Init
specprod_dir = specprod_root()
# Log
log=get_logger()
log.info("starting")
# Path
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = qaprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b','r','z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Exposure plot?
if args.expid is not None:
# Nights
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=args.expid, specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=args.expid, specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(channel_dict[channel], 'wave',
outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel))
skysub_resid_series(channel_dict[channel], 'flux',
outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel))
return
# Skyline
if args.skyline:
from desispec.qa.qa_plots import skyline_resid
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
if len(cframes) == 1:
log.error('len(cframes)==1; starting debugging')
pdb.set_trace() # Need to call differently
else:
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes, flatten=False)
# Plot
outfile=args.outdir+'/skyline_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel)
makepath(outfile)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Test sky noise for Gaussianity
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes)
# Plot
log.info("Plotting..")
outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel)
makepath(outfile)
skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
def main(args) :
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.qa.qa_plots import skysub_resid
import copy
import pdb
# Log
log=get_logger()
log.info("starting")
# Exposures?
if args.expids is not None:
expids = [int(iarg) for iarg in args.expids.split(',')]
else:
expids = 'all'
# Nights?
if args.nights is not None:
gdnights = [iarg for iarg in args.nights.split(',')]
else:
gdnights = 'all'
# Channels?
if args.channels is not None:
gdchannels = [iarg for iarg in args.channels.split(',')]
else:
gdchannels = 'all'
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Loop on nights
path_nights = glob.glob(args.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
if gdnights == 'all':
pass
else:
if night not in gdnights:
continue
# Get em
for exposure in get_exposures(night, specprod_dir = args.specprod_dir):
# Check against input expids
if expids == 'all':
pass
else:
if exposure not in expids:
continue
# Get em
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=exposure, specprod_dir=args.specprod_dir)
for camera, cframe_fil in frames_dict.items():
channel = camera[0]
# Check against input
if gdchannels == 'all':
pass
else:
if channel not in gdchannels:
continue
# Load frame
log.info('Loading {:s}'.format(cframe_fil))
cframe = read_frame(cframe_fil)
if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen
continue
# Sky
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=exposure, specprod_dir=args.specprod_dir)
skymodel = read_sky(sky_file)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
tmp = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Append
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
channel_dict[channel]['wave'].append(tmp.flatten())
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux.flatten(),1e-1)))
channel_dict[channel]['res'].append(res.flatten())
channel_dict[channel]['count'] += 1
# Figure
for channel in ['b', 'r', 'z']:
if channel_dict[channel]['count'] > 0:
sky_wave = np.concatenate(channel_dict[channel]['wave'])
sky_flux = np.concatenate(channel_dict[channel]['skyflux'])
sky_res = np.concatenate(channel_dict[channel]['res'])
# Plot
skysub_resid(sky_wave, sky_flux, sky_res,
outfile='tmp{:s}.png'.format(channel))
