def __init__(self, night, **kwargs):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
**kwargs are passed to QA_MultiExp
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
# Init
self.night = night
# Instantiate
QA_MultiExp.__init__(self, **kwargs)
# Load up exposures for the full production
nights = get_nights(specprod_dir=self.specprod_dir)
# Check the night exists
if self.night not in nights:
raise IOError("Bad input night for this production")
# Load up
self.mexp_dict[self.night] = {}
for exposure in get_exposures(self.night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=self.night,
expid=exposure,
specprod_dir=self.specprod_dir)
self.mexp_dict[self.night][exposure] = frames_dict
# Output file names
self.qaexp_outroot = self.qaprod_dir + '/' + self.night + '_qa'
def __init__(self, specprod_dir=None, **kwargs):
""" Class to organize and execute QA for a DESI production
Args:
specprod_dir(str): Path containing the exposures/ directory to use. If the value
is None, then the value of :func:`specprod_root` is used instead.
Notes:
Attributes:
qa_exps : list
List of QA_Exposure classes, one per exposure in production
data : dict
"""
if specprod_dir is None:
specprod_dir = specprod_root()
self.specprod_dir = specprod_dir
# Init
QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs)
# Load up exposures for the full production
nights = get_nights(specprod_dir=self.specprod_dir)
for night in nights:
self.mexp_dict[night] = {}
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
self.mexp_dict[night][exposure] = frames_dict
# Output file names
self.qaexp_outroot = self.qaprod_dir + '/' + self.prod_name + '_qa'
# Nights list
self.qa_nights = []
def load_s2n_values(objtype, nights, channel, sub_exposures=None):
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night) #, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap',
night=night,
expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(
flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data, masked=True)
sps_hdu.close()
objs = sps_tab['TEMPLATETYPE'] == objtype
if np.sum(objs) == 0:
continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel + str(ii)
cframe_path = findfile(filetype='cframe',
night=night,
expid=exposure,
camera=camera)
try:
cframe = read_frame(cframe_path)
except:
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave, 1)
dwave[0] = dwave[1]
#
iobjs = objs[cframe.fibers]
if np.sum(iobjs) == 0:
continue
s2n = cframe.flux[iobjs, :] * np.sqrt(
cframe.ivar[iobjs, :]) / np.sqrt(dwave)
# Save
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]])
if objtype == 'ELG':
fdict['OII'].append(
sps_tab['OIIFLUX'][cframe.fibers[iobjs]])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files into one master QA file
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove
Returns:
"""
from desispec.io import meta
from desispec.qa import QA_Exposure
from desispec.io import write_qa_prod
import pdb
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
# Reset
log.info("Resetting qa_exps in qa_prod")
self.qa_exps = []
# Loop
for night in nights:
# Loop on exposures
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
if len(frames_dict) == 0:
continue
# Load any frame (for the type)
key = list(frames_dict.keys())[0]
frame_fil = frames_dict[key]
frame = read_frame(frame_fil)
qa_exp = QA_Exposure(exposure,
night,
frame.meta['FLAVOR'],
specprod_dir=self.specprod_dir,
remove=remove)
# Append
self.qa_exps.append(qa_exp)
# Write
outroot = self.specprod_dir + '/' + self.prod_name + '_qa'
write_qa_prod(outroot, self)
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files into one master QA file
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove
Returns:
"""
from desispec.io import meta
from desispec.qa import QA_Exposure
from desispec.io import write_qa_prod
import pdb
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
# Reset
log.info("Resetting qa_exps in qa_prod")
self.qa_exps = []
# Loop
for night in nights:
# Loop on exposures
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
if len(frames_dict.keys()) == 0:
continue
# Load any frame (for the type)
key = frames_dict.keys()[0]
frame_fil = frames_dict[key]
frame = read_frame(frame_fil)
qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'],
specprod_dir=self.specprod_dir, remove=remove)
# Append
self.qa_exps.append(qa_exp)
# Write
outroot = self.specprod_dir+'/'+self.prod_name+'_qa'
write_qa_prod(outroot, self)
def make_exposures():
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html')
html_path, _ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path + '/' + night +
'/{:08d}'.format(expid)):
continue
# Link
f.write(
'<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'
.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid)
f.write('</ul></h3>\n')
# Finish
finish(f, body)
def make_exposures(qaprod_dir=None):
""" Generate HTML to organize exposure HTML
Parameters
----------
Returns
-------
links : str
body : str
"""
# Organized HTML
html_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir)
html_path,_ = os.path.split(html_file)
f = open(html_file, 'w')
init(f, 'Exposures QA')
# Loop on Nights
nights = get_nights()
nights.sort()
links = ''
body = ''
for night in nights:
# HTML
f.write('<h2> Night -- {:s} </h2>\n'.format(night))
f.write('<h3><ul>\n')
# Loop on expsoures
for expid in get_exposures(night):
if not os.path.exists(html_path+'/'+night+'/{:08d}'.format(expid)):
continue
# Link
f.write('<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid, expid))
# Generate Exposure html
make_exposure(night, expid, qaprod_dir=qaprod_dir)
f.write('</ul></h3>\n')
# Finish
finish(f,body)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.qa.qa_frame import qaframe_from_frame
from desispec.io.fluxcalibration import read_stdstar_models
log = get_logger()
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
qaframe_from_frame(frame_fil, make_plots=make_plots)
'''
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
for camera,frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat','arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple=read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def main(args):
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.qa.qa_plots import skysub_resid
import copy
import pdb
# Log
log = get_logger()
log.info("starting")
# Exposures?
if args.expids is not None:
expids = [int(iarg) for iarg in args.expids.split(',')]
else:
expids = 'all'
# Nights?
if args.nights is not None:
gdnights = [iarg for iarg in args.nights.split(',')]
else:
gdnights = 'all'
# Channels?
if args.channels is not None:
gdchannels = [iarg for iarg in args.channels.split(',')]
else:
gdchannels = 'all'
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(
b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Loop on nights
path_nights = glob.glob(args.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
if gdnights == 'all':
pass
else:
if night not in gdnights:
continue
# Get em
for exposure in get_exposures(night, specprod_dir=args.specprod_dir):
# Check against input expids
if expids == 'all':
pass
else:
if exposure not in expids:
continue
# Get em
frames_dict = get_files(filetype=str('cframe'),
night=night,
expid=exposure,
specprod_dir=args.specprod_dir)
for camera, cframe_fil in frames_dict.items():
channel = camera[0]
# Check against input
if gdchannels == 'all':
pass
else:
if channel not in gdchannels:
continue
# Load frame
log.info('Loading {:s}'.format(cframe_fil))
cframe = read_frame(cframe_fil)
if cframe.meta['FLAVOR'] in ['flat',
'arc']: # Probably can't happen
continue
# Sky
sky_file = findfile(str('sky'),
night=night,
camera=camera,
expid=exposure,
specprod_dir=args.specprod_dir)
skymodel = read_sky(sky_file)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
tmp = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Append
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
channel_dict[channel]['wave'].append(tmp.flatten())
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux.flatten(), 1e-1)))
channel_dict[channel]['res'].append(res.flatten())
channel_dict[channel]['count'] += 1
# Figure
for channel in ['b', 'r', 'z']:
if channel_dict[channel]['count'] > 0:
sky_wave = np.concatenate(channel_dict[channel]['wave'])
sky_flux = np.concatenate(channel_dict[channel]['skyflux'])
sky_res = np.concatenate(channel_dict[channel]['res'])
# Plot
skysub_resid(sky_wave,
sky_flux,
sky_res,
outfile='tmp{:s}.png'.format(channel))
def graph_night(rawdir, rawnight):
grph = {}
node = {}
node['type'] = 'night'
node['in'] = []
node['out'] = []
grph[rawnight] = node
allbricks = {}
expcount = {}
expcount['flat'] = 0
expcount['arc'] = 0
expcount['science'] = 0
# First, insert raw data into the graph. We use the existence of the raw data
# as a filter over spectrographs. Spectrographs whose raw data do not exist
# are excluded from the graph.
expid = io.get_exposures(rawnight, raw=True, rawdata_dir=rawdir)
campat = re.compile(r'([brz])([0-9])')
keepspec = set()
for ex in sorted(expid):
# get the fibermap for this exposure
fibermap = io.get_raw_files("fibermap", rawnight, ex, rawdata_dir=rawdir)
# read the fibermap to get the exposure type, and while we are at it,
# also accumulate the total list of bricks
fmdata, fmheader = io.read_fibermap(fibermap, header=True)
flavor = fmheader['flavor']
fmbricks = {}
for fmb in fmdata['BRICKNAME']:
if len(fmb) > 0:
if fmb in fmbricks.keys():
fmbricks[fmb] += 1
else:
fmbricks[fmb] = 1
for fmb in fmbricks.keys():
if fmb in allbricks.keys():
allbricks[fmb] += fmbricks[fmb]
else:
allbricks[fmb] = fmbricks[fmb]
if flavor == 'arc':
expcount['arc'] += 1
elif flavor == 'flat':
expcount['flat'] += 1
else:
expcount['science'] += 1
node = {}
node['type'] = 'fibermap'
node['id'] = ex
node['flavor'] = flavor
node['bricks'] = fmbricks
node['in'] = [rawnight]
node['out'] = []
name = graph_name(rawnight, "fibermap-{:08d}".format(ex))
grph[name] = node
grph[rawnight]['out'].append(name)
# get the raw exposures
raw = io.get_raw_files("pix", rawnight, ex, rawdata_dir=rawdir)
for cam in sorted(raw.keys()):
cammat = campat.match(cam)
if cammat is None:
raise RuntimeError("invalid camera string {}".format(cam))
band = cammat.group(1)
spec = cammat.group(2)
keepspec.update(spec)
node = {}
node['type'] = 'pix'
node['id'] = ex
node['band'] = band
node['spec'] = spec
node['flavor'] = flavor
node['in'] = [rawnight]
node['out'] = []
name = graph_name(rawnight, "pix-{}{}-{:08d}".format(band, spec, ex))
grph[name] = node
grph[rawnight]['out'].append(name)
keep = sorted(list(keepspec))
# Now that we have added all the raw data to the graph, we work our way
# through the processing steps.
# This step is a placeholder, in case we want to combine information from
# multiple flats or arcs before running bootcalib. We mark these bootcalib
# outputs as depending on all arcs and flats, but in reality we may just
# use the first or last set.
# Since each psfboot file takes multiple exposures as input, we first
# create those nodes.
for band in ['b', 'r', 'z']:
for spec in keep:
name = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
node = {}
node['type'] = 'psfboot'
node['band'] = band
node['spec'] = spec
node['in'] = []
node['out'] = []
grph[name] = node
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if (nd['flavor'] != 'flat') and (nd['flavor'] != 'arc'):
continue
band = nd['band']
spec = nd['spec']
bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
grph[bootname]['in'].append(name)
nd['out'].append(bootname)
# Next is full PSF estimation. Inputs are the arc image and the bootcalib
# output file. We also add nodes for the combined psfs.
for band in ['b', 'r', 'z']:
for spec in keep:
name = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
node = {}
node['type'] = 'psfnight'
node['band'] = band
node['spec'] = spec
node['in'] = []
node['out'] = []
grph[name] = node
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if nd['flavor'] != 'arc':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
psfname = graph_name(rawnight, "psf-{}{}-{:08d}".format(band, spec, id))
psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
node = {}
node['type'] = 'psf'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, bootname]
node['out'] = [psfnightname]
grph[psfname] = node
grph[bootname]['out'].append(psfname)
grph[psfnightname]['in'].append(psfname)
nd['out'].append(psfname)
# Now we extract the flats and science frames using the nightly psf
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if nd['flavor'] == 'arc':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
flavor = nd['flavor']
framename = graph_name(rawnight, "frame-{}{}-{:08d}".format(band, spec, id))
psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
fmname = graph_name(rawnight, "fibermap-{:08d}".format(id))
node = {}
node['type'] = 'frame'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['flavor'] = flavor
node['in'] = [name, fmname, psfnightname]
node['out'] = []
grph[framename] = node
grph[psfnightname]['out'].append(framename)
grph[fmname]['out'].append(framename)
nd['out'].append(framename)
# Now build the fiberflats for each flat exposure. We keep a list of all
# available fiberflats while we are looping over them, since we'll need
# that in the next step to select the "most recent" fiberflat.
flatexpid = {}
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] != 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'fiberflat'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name]
node['out'] = []
grph[flatname] = node
nd['out'].append(flatname)
cam = "{}{}".format(band, spec)
if cam not in flatexpid.keys():
flatexpid[cam] = []
flatexpid[cam].append(id)
# To compute the sky file, we use the "most recent fiberflat" that came
# before the current exposure.
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id))
flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid))
node = {}
node['type'] = 'sky'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname]
node['out'] = []
grph[skyname] = node
nd['out'].append(skyname)
grph[flatname]['out'].append(skyname)
# Construct the standard star files. These are one per spectrograph,
# and depend on the frames and the corresponding flats and sky files.
stdgrph = {}
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id))
# does this spectrograph exist yet in the graph?
if starname not in stdgrph.keys():
fmname = graph_name(rawnight, "fibermap-{:08d}".format(id))
grph[fmname]['out'].append(starname)
node = {}
node['type'] = 'stdstars'
node['spec'] = spec
node['id'] = id
node['in'] = [fmname]
node['out'] = []
stdgrph[starname] = node
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid))
skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id))
stdgrph[starname]['in'].extend([skyname, name, flatname])
nd['out'].append(starname)
grph[flatname]['out'].append(starname)
grph[skyname]['out'].append(starname)
grph.update(stdgrph)
# Construct calibration files
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id))
starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id))
flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid))
calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'calib'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname, skyname, starname]
node['out'] = []
grph[calname] = node
grph[flatname]['out'].append(calname)
grph[skyname]['out'].append(calname)
grph[starname]['out'].append(calname)
nd['out'].append(calname)
# Build cframe files
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id))
flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid))
calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id))
cfname = graph_name(rawnight, "cframe-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'cframe'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname, skyname, calname]
node['out'] = []
grph[cfname] = node
grph[flatname]['out'].append(cfname)
grph[skyname]['out'].append(cfname)
grph[calname]['out'].append(cfname)
nd['out'].append(cfname)
# Brick / Zbest dependencies
for b in allbricks.keys():
zbname = "zbest-{}".format(b)
inb = []
for band in ['b', 'r', 'z']:
node = {}
node['type'] = 'brick'
node['brick'] = b
node['band'] = band
node['in'] = []
node['out'] = [zbname]
bname = "brick-{}-{}".format(band, b)
inb.append(bname)
grph[bname] = node
node = {}
node['type'] = 'zbest'
node['brick'] = b
node['ntarget'] = allbricks[b]
node['in'] = inb
node['out'] = []
grph[zbname] = node
for name, nd in grph.items():
if nd['type'] != 'fibermap':
continue
if nd['flavor'] == 'arc':
continue
if nd['flavor'] == 'flat':
continue
id = nd['id']
bricks = nd['bricks']
for band in ['b', 'r', 'z']:
for spec in keep:
cfname = graph_name(rawnight, "cframe-{}{}-{:08d}".format(band, spec, id))
for b in bricks:
bname = "brick-{}-{}".format(band, b)
grph[bname]['in'].append(cfname)
grph[cfname]['out'].append(bname)
return (grph, expcount, allbricks)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype,
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info(
"qafile={:s} exists. Not over-writing. Consider clobber=True"
.format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat),
clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile(
'qa_flat_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame,
fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile(
'qa_sky_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel,
qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile(
'stdstars',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple = read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile(
'calib',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
if make_plots:
qafig = meta.findfile(
'qa_flux_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def graph_night(rawdir, rawnight):
grph = {}
node = {}
node['type'] = 'night'
node['in'] = []
node['out'] = []
grph[rawnight] = node
allbricks = {}
expcount = {}
expcount['flat'] = 0
expcount['arc'] = 0
expcount['science'] = 0
# First, insert raw data into the graph. We use the existence of the raw data
# as a filter over spectrographs. Spectrographs whose raw data do not exist
# are excluded from the graph.
expid = io.get_exposures(rawnight, raw=True, rawdata_dir=rawdir)
campat = re.compile(r'([brz])([0-9])')
keepspec = set()
for ex in sorted(expid):
# get the fibermap for this exposure
fibermap = io.get_raw_files("fibermap",
rawnight,
ex,
rawdata_dir=rawdir)
# read the fibermap to get the exposure type, and while we are at it,
# also accumulate the total list of bricks
fmdata, fmheader = io.read_fibermap(fibermap, header=True)
flavor = fmheader['flavor']
fmbricks = {}
for fmb in fmdata['BRICKNAME']:
if len(fmb) > 0:
if fmb in fmbricks.keys():
fmbricks[fmb] += 1
else:
fmbricks[fmb] = 1
for fmb in fmbricks.keys():
if fmb in allbricks.keys():
allbricks[fmb] += fmbricks[fmb]
else:
allbricks[fmb] = fmbricks[fmb]
if flavor == 'arc':
expcount['arc'] += 1
elif flavor == 'flat':
expcount['flat'] += 1
else:
expcount['science'] += 1
node = {}
node['type'] = 'fibermap'
node['id'] = ex
node['flavor'] = flavor
node['bricks'] = fmbricks
node['in'] = [rawnight]
node['out'] = []
name = graph_name(rawnight, "fibermap-{:08d}".format(ex))
grph[name] = node
grph[rawnight]['out'].append(name)
# get the raw exposures
raw = io.get_raw_files("pix", rawnight, ex, rawdata_dir=rawdir)
for cam in sorted(raw.keys()):
cammat = campat.match(cam)
if cammat is None:
raise RuntimeError("invalid camera string {}".format(cam))
band = cammat.group(1)
spec = cammat.group(2)
keepspec.update(spec)
node = {}
node['type'] = 'pix'
node['id'] = ex
node['band'] = band
node['spec'] = spec
node['flavor'] = flavor
node['in'] = [rawnight]
node['out'] = []
name = graph_name(rawnight,
"pix-{}{}-{:08d}".format(band, spec, ex))
grph[name] = node
grph[rawnight]['out'].append(name)
keep = sorted(list(keepspec))
# Now that we have added all the raw data to the graph, we work our way
# through the processing steps.
# This step is a placeholder, in case we want to combine information from
# multiple flats or arcs before running bootcalib. We mark these bootcalib
# outputs as depending on all arcs and flats, but in reality we may just
# use the first or last set.
# Since each psfboot file takes multiple exposures as input, we first
# create those nodes.
for band in ['b', 'r', 'z']:
for spec in keep:
name = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
node = {}
node['type'] = 'psfboot'
node['band'] = band
node['spec'] = spec
node['in'] = []
node['out'] = []
grph[name] = node
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if (nd['flavor'] != 'flat') and (nd['flavor'] != 'arc'):
continue
band = nd['band']
spec = nd['spec']
bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
grph[bootname]['in'].append(name)
nd['out'].append(bootname)
# Next is full PSF estimation. Inputs are the arc image and the bootcalib
# output file. We also add nodes for the combined psfs.
for band in ['b', 'r', 'z']:
for spec in keep:
name = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
node = {}
node['type'] = 'psfnight'
node['band'] = band
node['spec'] = spec
node['in'] = []
node['out'] = []
grph[name] = node
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if nd['flavor'] != 'arc':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec))
psfname = graph_name(rawnight,
"psf-{}{}-{:08d}".format(band, spec, id))
psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
node = {}
node['type'] = 'psf'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, bootname]
node['out'] = [psfnightname]
grph[psfname] = node
grph[bootname]['out'].append(psfname)
grph[psfnightname]['in'].append(psfname)
nd['out'].append(psfname)
# Now we extract the flats and science frames using the nightly psf
for name, nd in grph.items():
if nd['type'] != 'pix':
continue
if nd['flavor'] == 'arc':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
flavor = nd['flavor']
framename = graph_name(rawnight,
"frame-{}{}-{:08d}".format(band, spec, id))
psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec))
fmname = graph_name(rawnight, "fibermap-{:08d}".format(id))
node = {}
node['type'] = 'frame'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['flavor'] = flavor
node['in'] = [name, fmname, psfnightname]
node['out'] = []
grph[framename] = node
grph[psfnightname]['out'].append(framename)
grph[fmname]['out'].append(framename)
nd['out'].append(framename)
# Now build the fiberflats for each flat exposure. We keep a list of all
# available fiberflats while we are looping over them, since we'll need
# that in the next step to select the "most recent" fiberflat.
flatexpid = {}
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] != 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
flatname = graph_name(rawnight,
"fiberflat-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'fiberflat'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name]
node['out'] = []
grph[flatname] = node
nd['out'].append(flatname)
cam = "{}{}".format(band, spec)
if cam not in flatexpid.keys():
flatexpid[cam] = []
flatexpid[cam].append(id)
# To compute the sky file, we use the "most recent fiberflat" that came
# before the current exposure.
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight,
"sky-{}{}-{:08d}".format(band, spec, id))
flatname = graph_name(rawnight,
"fiberflat-{}{}-{:08d}".format(band, spec, fid))
node = {}
node['type'] = 'sky'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname]
node['out'] = []
grph[skyname] = node
nd['out'].append(skyname)
grph[flatname]['out'].append(skyname)
# Construct the standard star files. These are one per spectrograph,
# and depend on the frames and the corresponding flats and sky files.
stdgrph = {}
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id))
# does this spectrograph exist yet in the graph?
if starname not in stdgrph.keys():
fmname = graph_name(rawnight, "fibermap-{:08d}".format(id))
grph[fmname]['out'].append(starname)
node = {}
node['type'] = 'stdstars'
node['spec'] = spec
node['id'] = id
node['in'] = [fmname]
node['out'] = []
stdgrph[starname] = node
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
flatname = graph_name(rawnight,
"fiberflat-{}{}-{:08d}".format(band, spec, fid))
skyname = graph_name(rawnight,
"sky-{}{}-{:08d}".format(band, spec, id))
stdgrph[starname]['in'].extend([skyname, name, flatname])
nd['out'].append(starname)
grph[flatname]['out'].append(starname)
grph[skyname]['out'].append(starname)
grph.update(stdgrph)
# Construct calibration files
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight,
"sky-{}{}-{:08d}".format(band, spec, id))
starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id))
flatname = graph_name(rawnight,
"fiberflat-{}{}-{:08d}".format(band, spec, fid))
calname = graph_name(rawnight,
"calib-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'calib'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname, skyname, starname]
node['out'] = []
grph[calname] = node
grph[flatname]['out'].append(calname)
grph[skyname]['out'].append(calname)
grph[starname]['out'].append(calname)
nd['out'].append(calname)
# Build cframe files
for name, nd in grph.items():
if nd['type'] != 'frame':
continue
if nd['flavor'] == 'flat':
continue
band = nd['band']
spec = nd['spec']
id = nd['id']
cam = "{}{}".format(band, spec)
flatid = None
for fid in sorted(flatexpid[cam]):
if (flatid is None):
flatid = fid
elif (fid > flatid) and (fid < id):
flatid = fid
skyname = graph_name(rawnight,
"sky-{}{}-{:08d}".format(band, spec, id))
flatname = graph_name(rawnight,
"fiberflat-{}{}-{:08d}".format(band, spec, fid))
calname = graph_name(rawnight,
"calib-{}{}-{:08d}".format(band, spec, id))
cfname = graph_name(rawnight,
"cframe-{}{}-{:08d}".format(band, spec, id))
node = {}
node['type'] = 'cframe'
node['band'] = band
node['spec'] = spec
node['id'] = id
node['in'] = [name, flatname, skyname, calname]
node['out'] = []
grph[cfname] = node
grph[flatname]['out'].append(cfname)
grph[skyname]['out'].append(cfname)
grph[calname]['out'].append(cfname)
nd['out'].append(cfname)
# Brick / Zbest dependencies
for b in allbricks.keys():
zbname = "zbest-{}".format(b)
inb = []
for band in ['b', 'r', 'z']:
node = {}
node['type'] = 'brick'
node['brick'] = b
node['band'] = band
node['in'] = []
node['out'] = [zbname]
bname = "brick-{}-{}".format(band, b)
inb.append(bname)
grph[bname] = node
node = {}
node['type'] = 'zbest'
node['brick'] = b
node['ntarget'] = allbricks[b]
node['in'] = inb
node['out'] = []
grph[zbname] = node
for name, nd in grph.items():
if nd['type'] != 'fibermap':
continue
if nd['flavor'] == 'arc':
continue
if nd['flavor'] == 'flat':
continue
id = nd['id']
bricks = nd['bricks']
for band in ['b', 'r', 'z']:
for spec in keep:
cfname = graph_name(
rawnight, "cframe-{}{}-{:08d}".format(band, spec, id))
for b in bricks:
bname = "brick-{}-{}".format(band, b)
grph[bname]['in'].append(cfname)
grph[cfname]['out'].append(bname)
return (grph, expcount, allbricks)
def main(args) :
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.qa.qa_plots import skysub_resid
import copy
import pdb
# Log
log=get_logger()
log.info("starting")
# Exposures?
if args.expids is not None:
expids = [int(iarg) for iarg in args.expids.split(',')]
else:
expids = 'all'
# Nights?
if args.nights is not None:
gdnights = [iarg for iarg in args.nights.split(',')]
else:
gdnights = 'all'
# Channels?
if args.channels is not None:
gdchannels = [iarg for iarg in args.channels.split(',')]
else:
gdchannels = 'all'
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Loop on nights
path_nights = glob.glob(args.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
if gdnights == 'all':
pass
else:
if night not in gdnights:
continue
# Get em
for exposure in get_exposures(night, specprod_dir = args.specprod_dir):
# Check against input expids
if expids == 'all':
pass
else:
if exposure not in expids:
continue
# Get em
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=exposure, specprod_dir=args.specprod_dir)
for camera, cframe_fil in frames_dict.items():
channel = camera[0]
# Check against input
if gdchannels == 'all':
pass
else:
if channel not in gdchannels:
continue
# Load frame
log.info('Loading {:s}'.format(cframe_fil))
cframe = read_frame(cframe_fil)
if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen
continue
# Sky
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=exposure, specprod_dir=args.specprod_dir)
skymodel = read_sky(sky_file)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
tmp = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Append
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
channel_dict[channel]['wave'].append(tmp.flatten())
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux.flatten(),1e-1)))
channel_dict[channel]['res'].append(res.flatten())
channel_dict[channel]['count'] += 1
# Figure
for channel in ['b', 'r', 'z']:
if channel_dict[channel]['count'] > 0:
sky_wave = np.concatenate(channel_dict[channel]['wave'])
sky_flux = np.concatenate(channel_dict[channel]['skyflux'])
sky_res = np.concatenate(channel_dict[channel]['res'])
# Plot
skysub_resid(sky_wave, sky_flux, sky_res,
outfile='tmp{:s}.png'.format(channel))
def main(args):
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import read_frame
from desispec.io import get_reduced_frames
from desispec.io.sky import read_sky
from desispec.io import specprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Log
log = get_logger()
log.info("starting")
# Path
if args.reduxdir is not None:
specprod_dir = args.reduxdir
else:
specprod_dir = specprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b', 'r', 'z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(
b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Exposure plot?
if args.expid is not None:
# Nights
path_nights = glob.glob(specprod_dir + '/exposures/*')
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'),
night=night,
expid=args.expid,
specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'),
night=night,
camera=camera,
expid=args.expid,
specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(
cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux, 1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(
channel_dict[channel],
'wave',
outfile='QA_skyresid_wave_expid_{:d}{:s}.png'.
format(args.expid, channel))
skysub_resid_series(
channel_dict[channel],
'flux',
outfile='QA_skyresid_flux_expid_{:d}{:s}.png'.
format(args.expid, channel))
return
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(
len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile = 'QA/skyresid_prod_dual_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Full Prod Plot?
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(
len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes)
# Plot
log.info("Plotting..")
skysub_gauss(
sky_wave,
sky_flux,
sky_res,
sky_ivar,
outfile='skyresid_prod_gauss_{:s}.png'.format(channel))
return
def main(args, comm=None):
rank = 0
nproc = 1
if comm is not None:
rank = comm.rank
nproc = comm.size
# Determine which nights we are using
nights = None
if args.nights is not None:
nights = args.nights.split(",")
else:
if rank == 0:
rawdir = os.path.abspath(specio.rawdata_root())
nights = []
nightpat = re.compile(r"\d{8}")
for root, dirs, files in os.walk(rawdir, topdown=True):
for d in dirs:
nightmat = nightpat.match(d)
if nightmat is not None:
nights.append(d)
break
if comm is not None:
nights = comm.bcast(nights, root=0)
# Get the list of exposures for each night
night_expid = {}
all_expid = []
exp_to_night = {}
if rank == 0:
for nt in nights:
night_expid[nt] = specio.get_exposures(nt, raw=True)
all_expid.extend(night_expid[nt])
for ex in night_expid[nt]:
exp_to_night[ex] = nt
if comm is not None:
night_expid = comm.bcast(night_expid, root=0)
all_expid = comm.bcast(all_expid, root=0)
exp_to_night = comm.bcast(exp_to_night, root=0)
expids = np.array(all_expid, dtype=np.int32)
nexp = len(expids)
# Get the list of cameras
cams = None
if args.cameras is not None:
cams = args.cameras.split(",")
else:
cams = []
for band in ['b', 'r', 'z']:
for spec in range(10):
cams.append('{}{}'.format(band, spec))
# number of cameras
ncamera = len(cams)
# check that our communicator is an appropriate size
if comm is not None:
if ncamera * args.camera_procs > comm.size:
if comm.rank == 0:
print("Communicator size ({}) too small for {} cameras each with {} procs".format(comm.size, ncamera, args.camera_procs), flush=True)
comm.Abort()
# create a set of reproducible seeds for each exposure
np.random.seed(args.seed)
maxexp = np.max(expids)
allseeds = np.random.randint(2**32, size=(maxexp+1))
seeds = allseeds[-nexp:]
taskproc = ncamera * args.camera_procs
comm_group = comm
comm_rank = None
group = comm.rank
ngroup = comm.size
group_rank = 0
if comm is not None:
from mpi4py import MPI
if taskproc > 1:
ngroup = int(comm.size / taskproc)
group = int(comm.rank / taskproc)
group_rank = comm.rank % taskproc
comm_group = comm.Split(color=group, key=group_rank)
comm_rank = comm.Split(color=group_rank, key=group)
else:
comm_group = MPI.COMM_SELF
comm_rank = comm
myexpids = np.array_split(expids, ngroup)[group]
for ex in myexpids:
nt = exp_to_night[ex]
# path to raw file
simspecfile = simio.findfile('simspec', nt, ex)
rawfile = specio.findfile('raw', nt, ex)
rawfile = os.path.join(os.path.dirname(simspecfile), rawfile)
# Is this exposure already finished?
done = True
if group_rank == 0:
if not os.path.isfile(rawfile):
done = False
if args.preproc:
for c in cams:
pixfile = specio.findfile('pix', night=nt,
expid=ex, camera=c)
if not os.path.isfile(pixfile):
done = False
if comm_group is not None:
done = comm_group.bcast(done, root=0)
if done and not args.overwrite:
if group_rank == 0:
print("Skipping completed exposure {:08d} on night {}".format(ex, nt))
continue
# Write per-process logs to a separate directory,
# since there are so many of them.
logdir = "{}_logs".format(rawfile)
if group_rank == 0:
if not os.path.isdir(logdir):
os.makedirs(logdir)
if comm_group is not None:
comm_group.barrier()
tasklog = os.path.join(logdir, "pixsim")
with stdouterr_redirected(to=tasklog, comm=comm_group):
try:
options = {}
options["night"] = nt
options["expid"] = int(ex)
options["cosmics"] = args.cosmics
options["seed"] = seeds[ex]
options["cameras"] = ",".join(cams)
options["mpi_camera"] = args.camera_procs
options["verbose"] = args.verbose
options["preproc"] = args.preproc
optarray = option_list(options)
pixargs = pixsim.parse(optarray)
pixsim.main(pixargs, comm_group)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print("".join(lines), flush=True)
def load_all_s2n_values(nights, channel, sub_exposures=None):
"""
Calculate S/N values for a set of spectra from an input list of nights
Args:
nights: list
channel: str ('b','r','z')
sub_exposures:
Returns:
fdict: dict
Contains all the S/N info for all nights in the given channel
"""
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night)#, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
log.debug('Getting truth from {}'.format(simspec_file))
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data,masked=True)
#- Get OIIFLUX from separate HDU and join
if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu):
elg_truth = Table(sps_hdu['TRUTH_ELG'].data)
sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'],
keys='TARGETID', join_type='left')
else:
sps_tab['OIIFLUX'] = 0.0
sps_hdu.close()
#objs = sps_tab['TEMPLATETYPE'] == objtype
#if np.sum(objs) == 0:
# continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel+str(ii)
cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera)
try:
log.debug('Reading from {}'.format(cframe_path))
cframe = read_frame(cframe_path)
except (IOError, OSError):
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave,1)
dwave[0] = dwave[1]
# Calculate
s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave)
#s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave)
# Save
fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers])
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers])
fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
def main(args, comm=None):
if args.verbose:
import logging
log.setLevel(logging.DEBUG)
#we do this so we can use operator.itemgetter
import operator
#we do this so our print statements can have timestamps
import time
rank = 0
nproc = 1
if comm is not None:
import mpi4py
rank = comm.rank
nproc = comm.size
if rank == 0:
log.info('Starting pixsim at {}'.format(asctime()))
#no preflight check here, too complicated.
#we'll assume the user knows what he or she is doing...
# Determine which nights we are using
nights = None
if args.nights is not None:
nights = args.nights.split(",")
else:
rawdir = os.path.abspath(specio.rawdata_root())
nights = []
nightpat = re.compile(r"\d{8}")
for root, dirs, files in os.walk(rawdir, topdown=True):
for d in dirs:
nightmat = nightpat.match(d)
if nightmat is not None:
nights.append(d)
# Get the list of exposures for each night
night_expid = {}
all_expid = []
exp_to_night = {}
for nt in nights:
night_expid[nt] = specio.get_exposures(nt, raw=True)
#get a list of tuples of (night,expid) that we can evenly divide between communicators
night_exposure_list=list()
if comm is None or comm.rank == 0:
for nt in nights:
for exp in night_expid[nt]:
rawfile = desispec.io.findfile('raw', nt, exp)
if not os.path.exists(rawfile):
night_exposure_list.append([nt,exp])
elif args.overwrite:
log.warning('Overwriting pre-existing {}'.format(os.path.basename(rawfile)))
os.remove(rawfile)
night_exposure_list.append([nt,exp])
else:
log.info('Skipping pre-existing {}'.format(os.path.basename(rawfile)))
if args.nexp is not None:
night_exposure_list = night_exposure_list[0:args.nexp]
if comm is not None:
night_exposure_list = comm.bcast(night_exposure_list, root=0)
if len(night_exposure_list) == 0:
if comm is None or comm.rank == 0:
log.error('No exposures to process')
sys.exit(1)
# Get the list of cameras and make sure it's in the right format
cams = []
if args.cameras is not None:
entry = args.cameras.split(',')
for i in entry:
cams.append(i)
else:
#do this with band first so we can avoid re-broadcasting cosmics
for band in ['b', 'r', 'z']:
for spec in range(10):
cams.append('{}{}'.format(band, spec))
#are we using cosmics?
if args.cosmics is not None:
addcosmics = True
else:
addcosmics = False
ncameras=len(cams)
nexposures=len(night_exposure_list)
#call ultity function to figure out how many nodes we have
nnodes=mpi_count_nodes(comm)
#call utility functions to divide our workload
if args.nodes_per_exp is not None:
user_specified_nodes=args.nodes_per_exp
else:
user_specified_nodes=None
nodes_per_comm_exp=get_nodes_per_exp(nnodes,nexposures,ncameras,user_specified_nodes)
#also figure out how many exposure communicators we have
num_exp_comm = nnodes // nodes_per_comm_exp
#split the communicator into exposure communicators
comm_exp, node_index_exp, num_nodes_exp = mpi_split_by_node(comm, nodes_per_comm_exp)
#further splitting will happen automatically in simulate_exposure
#based on this, figure out which simspecfiles and rawfiles are assigned to each communicator
#find all specfiles
#find all rawfiles
rawfile_list=[]
simspecfile_list=[]
night_list=[]
expid_list=[]
for i in range(len(night_exposure_list)):
night_list.append(night_exposure_list[i][0])
expid_list.append(night_exposure_list[i][1])
rawfile_list.append(desispec.io.findfile('raw', night_list[i], expid_list[i]))
simspecfile_list.append(io.findfile('simspec', night_list[i], expid_list[i]))
#now divy the rawfiles and specfiles between node communicators
#there is onerawfile and one specfile for each exposure
rawfile_comm_exp=[]
simspecfile_comm_exp=[]
for i in range(num_exp_comm):
if node_index_exp == i: #assign rawfile, simspec file to one communicator at a time
rawfile_comm_exp=rawfile_list[i::num_exp_comm]
simspecfile_comm_exp=simspecfile_list[i::num_exp_comm]
night_comm_exp=night_list[i::num_exp_comm]
expid_comm_exp=expid_list[i::num_exp_comm]
comm.Barrier()
#now wrap pixsim.simulate_exposure for each exposure (in desisim.pixsim)
if comm_exp.rank == 0:
log.info("Starting simulate_exposure for night {} expid {}".format(night_comm_exp, expid_comm_exp))
for i in range(len(rawfile_comm_exp)):
simulate_exposure(simspecfile_comm_exp[i], rawfile_comm_exp[i], cameras=cams,
ccdshape=None, simpixfile=None, addcosmics=addcosmics, comm=comm_exp)
comm.Barrier()
if rank == 0:
log.info('Finished pixsim nights {}'.format(args.nights, asctime()))
def main(args) :
# imports
import glob
from desispec.io import findfile, makepath
from desispec.io import get_exposures
from desispec.io import get_files, get_nights
from desispec.io import get_reduced_frames
from desispec.io import specprod_root
from desispec.io import qaprod_root
from desispec.qa import utils as qa_utils
import copy
import pdb
# Init
specprod_dir = specprod_root()
# Log
log=get_logger()
log.info("starting")
# Path
if args.qaprod_dir is not None:
qaprod_dir = args.qaprod_dir
else:
qaprod_dir = qaprod_root()
# Channels
if args.channels is not None:
channels = [iarg for iarg in args.channels.split(',')]
else:
channels = ['b','r','z']
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Nights
if args.nights is not None:
nights = [iarg for iarg in args.nights.split(',')]
else:
nights = None
# Exposure plot?
if args.expid is not None:
# Nights
if nights is None:
nights = get_nights()
nights.sort()
# Find the exposure
for night in nights:
if args.expid in get_exposures(night, specprod_dir=specprod_dir):
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=args.expid, specprod_dir=specprod_dir)
# Loop on channel
#for channel in ['b','r','z']:
for channel in ['z']:
channel_dict[channel]['cameras'] = []
for camera, cframe_fil in frames_dict.items():
if channel in camera:
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=args.expid, specprod_dir=specprod_dir)
wave, flux, res, _ = qa_utils.get_skyres(cframe_fil)
# Append
channel_dict[channel]['wave'].append(wave)
channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1)))
channel_dict[channel]['res'].append(res)
channel_dict[channel]['cameras'].append(camera)
channel_dict[channel]['count'] += 1
if channel_dict[channel]['count'] > 0:
from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging
skysub_resid_series(channel_dict[channel], 'wave',
outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel))
skysub_resid_series(channel_dict[channel], 'flux',
outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel))
return
# Skyline
if args.skyline:
from desispec.qa.qa_plots import skyline_resid
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
if len(cframes) == 1:
log.error('len(cframes)==1; starting debugging')
pdb.set_trace() # Need to call differently
else:
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(
cframes, flatten=False)
# Plot
outfile=args.outdir+'/skyline_{:s}.png'.format(channel)
log.info("Plotting to {:s}".format(outfile))
skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
# Full Prod Plot?
if args.prod:
from desispec.qa.qa_plots import skysub_resid_dual
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes)
# Plot
outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel)
makepath(outfile)
log.info("Plotting to {:s}".format(outfile))
skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile)
return
# Test sky noise for Gaussianity
if args.gauss:
from desispec.qa.qa_plots import skysub_gauss
# Loop on channel
for channel in channels:
cframes = get_reduced_frames(nights=nights, channels=[channel])
if len(cframes) > 0:
# Cut down for debugging
#cframes = [cframes[ii] for ii in range(15)]
#
log.info("Loading sky residuals for {:d} cframes".format(len(cframes)))
sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes)
# Plot
log.info("Plotting..")
outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel)
makepath(outfile)
skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar,
outfile=outfile)
return
