def exposure_fiberflat(channel, expid, metric, outfile=None): """ Generate an Exposure level plot of a FiberFlat metric Args: channel: str, e.g. 'b', 'r', 'z' expid: int metric: str, allowed entires are: ['meanflux'] Returns: """ from desispec.io.meta import find_exposure_night, findfile from desispec.io.frame import read_meta_frame, read_frame from desispec.io.fiberflat import read_fiberflat from desimodel.focalplane import fiber_area_arcsec2 log = get_logger() # Find exposure night = find_exposure_night(expid) # Search for frames with the input channel frame0 = findfile('frame', camera=channel+'0', night=night, expid=expid) if not os.path.exists(frame0): log.fatal("No Frame 0 for channel={:s} and expid={:d}".format(channel, expid)) # Confirm frame is a Flat fmeta = read_meta_frame(frame0) assert fmeta['FLAVOR'].strip() == 'flat' # Load up all the frames x,y,metrics = [],[],[] for wedge in range(10): # Load frame_file = findfile('frame', camera=channel+'{:d}'.format(wedge), night=night, expid=expid) fiber_file = findfile('fiberflat', camera=channel+'{:d}'.format(wedge), night=night, expid=expid) try: frame = read_frame(frame_file) except: continue else: fiberflat = read_fiberflat(fiber_file) fibermap = frame.fibermap gdp = fiberflat.mask == 0 # X,Y x.append([fibermap['X_TARGET']]) y.append([fibermap['Y_TARGET']]) area = fiber_area_arcsec2(x[-1], y[-1]) mean_area = np.mean(area) # Metric if metric == 'meanflux': mean_norm = np.mean(fiberflat.fiberflat*gdp,axis=1) / (area / mean_area) metrics.append([mean_norm]) # Cocatenate x = np.concatenate(x) y = np.concatenate(y) metrics = np.concatenate(metrics) # Plot if outfile is None: outfile='qa_{:08d}_{:s}_fiberflat.png'.format(expid, channel) exposure_map(x,y,metrics, mlbl='Mean Flux', title='Mean Flux for Exposure {:08d}, Channel {:s}'.format(expid, channel), outfile=outfile)
def make_exposure(night, expid, qaprod_dir=None): """ Generate HTML for exposure PNGs Parameters ---------- setup : str cbset : str det : int Returns ------- links : str body : str """ # File name html_file = meta.findfile('qa_exposure_html', night=night, expid=expid, qaprod_dir=qaprod_dir) html_path, _ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposure QA') links = '' body = '' # Loop on Nights for ctype in ['sky', 'flux']: # all_png = glob.glob(html_path + '/qa-{:s}-*-{:08d}.png'.format(ctype, expid)) all_png.sort() if len(all_png) == 0: continue # Type links += '<h2> {:s} Calib</h2>\n'.format(ctype) for png in all_png: _, png_file = os.path.split(png) # Image href = "{:s}".format(png_file[:-4]) links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format( href, href) body += '<div class="section" id="{:s}">\n'.format(href) body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format( png_file) #f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid)) f.write('<ul>\n') f.write(links) f.write('</ul>\n') f.write(body) # Finish finish(f, '') # Return return links, body
def make_exposure(night, expid, qaprod_dir=None): """ Generate HTML for exposure PNGs Parameters ---------- setup : str cbset : str det : int Returns ------- links : str body : str """ # File name html_file = meta.findfile('qa_exposure_html', night=night, expid=expid, qaprod_dir=qaprod_dir) html_path,_ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposure QA') links = '' body = '' # Loop on Nights for ctype in ['sky', 'flux']: # all_png = glob.glob(html_path+'/qa-{:s}-*-{:08d}.png'.format(ctype,expid)) all_png.sort() if len(all_png) == 0: continue # Type links += '<h2> {:s} Calib</h2>\n'.format(ctype) for png in all_png: _,png_file = os.path.split(png) # Image href="{:s}".format(png_file[:-4]) links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(href, href) body += '<div class="section" id="{:s}">\n'.format(href) body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(png_file) #f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid)) f.write('<ul>\n') f.write(links) f.write('</ul>\n') f.write(body) # Finish finish(f,'') # Return return links, body
def calib_exp(night, expid): """ Geneate HTML for calib exposure PNGs Args: night: expid: Returns: """ # File name html_file = meta.findfile('qa_calib_exp_html', night=night, expid=expid) html_path, _ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Calibration Exposure QA') # Loop on Nights for ctype in ['flat']: links = '' body = '' # all_png = glob.glob(html_path + '/qa-{:s}-*-{:08d}.png'.format(ctype, expid)) all_png.sort() if len(all_png) == 0: continue # Type links += '<h2> {:s} Calib</h2>\n'.format(ctype) for png in all_png: _, png_file = os.path.split(png) # Image href = "{:s}".format(png_file[:-4]) links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format( href, href) body += '<div class="section" id="{:s}">\n'.format(href) body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format( png_file) #f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid)) f.write('<ul>\n') f.write(links) f.write('</ul>\n') f.write(body) # Finish finish(f, '') # Return return links, body
def make_exposures(): """ Generate HTML to organize exposure HTML Parameters ---------- Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_exposures_html') html_path, _ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposures QA') # Loop on Nights nights = get_nights() nights.sort() links = '' body = '' for night in nights: # HTML f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') # Loop on expsoures for expid in get_exposures(night): if not os.path.exists(html_path + '/' + night + '/{:08d}'.format(expid)): continue # Link f.write( '<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n' .format(night, expid, expid, expid)) # Generate Exposure html make_exposure(night, expid) f.write('</ul></h3>\n') # Finish finish(f, body)
def calib(qaprod_dir=None, specprod_dir=None): """ Generate HTML to orgainze calib HTML """ # Organized HTML html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir) html_path, _ = os.path.split(html_file) makepath(html_file) # Open f = open(html_file, 'w') init(f, 'Calibration QA') # Loop on Nights nights = get_nights(sub_folder='calibnight', specprod_dir=specprod_dir) nights.sort() links = '' body = '' for night in nights: all_png = glob.glob(html_path + '/' + night + '/qa*.png') if len(all_png) == 0: continue # Find expid expids = [] for png in all_png: expids.append(int(png[-12:-4])) # A bit risky expids = np.unique(expids) expids.sort() f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') for expid in expids: # Link f.write( '<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'. format(night, expid, expid)) # Generate Exposure html calib_exp(night, expid, qaprod_dir=qaprod_dir) f.write('</ul></h3>\n') # Finish finish(f, body) # Return return links, body
def make_exposures(qaprod_dir=None): """ Generate HTML to organize exposure HTML Parameters ---------- Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir) html_path,_ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposures QA') # Loop on Nights nights = get_nights() nights.sort() links = '' body = '' for night in nights: # HTML f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') # Loop on expsoures for expid in get_exposures(night): if not os.path.exists(html_path+'/'+night+'/{:08d}'.format(expid)): continue # Link f.write('<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid, expid)) # Generate Exposure html make_exposure(night, expid, qaprod_dir=qaprod_dir) f.write('</ul></h3>\n') # Finish finish(f,body)
def calib(qaprod_dir=None): """ Generate HTML to orgainze calib HTML """ # Organized HTML html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir) html_path,_ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Calibration QA') # Loop on Nights nights = get_nights(sub_folder='calibnight') nights.sort() links = '' body = '' for night in nights: all_png = glob.glob(html_path+'/'+night+'/qa*.png') if len(all_png) == 0: continue # Find expid expids = [] for png in all_png: expids.append(int(png[-12:-4])) # A bit risky expids = np.unique(expids) expids.sort() f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') for expid in expids: # Link f.write('<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid)) # Generate Exposure html calib_exp(night, expid, qaprod_dir=qaprod_dir) f.write('</ul></h3>\n') # Finish finish(f,body) # Return return links, body
def assemble_fibermap(night, expid, badamps=None, force=False): """ Create a fibermap for a given night and expid Args: night (int): YEARMMDD night of sunset expid (int): exposure ID Options: badamps (str): comma separated list of "{camera}{petal}{amp}", i.e. "[brz][0-9][ABCD]". Example: 'b7D,z8A' force (bool): create fibermap even if missing coordinates/guide files """ log = get_logger() #- raw data file for header rawfile = findfile('raw', night, expid) try: rawheader = fits.getheader(rawfile, 'SPEC') except KeyError: rawheader = fits.getheader(rawfile, 'SPS') #- Find fiberassign file fafile = find_fiberassign_file(night, expid) #- Find coordinates file in same directory dirname, filename = os.path.split(fafile) globfiles = glob.glob(dirname + '/coordinates-*.fits') if len(globfiles) == 1: coordfile = globfiles[0] elif len(globfiles) == 0: message = f'No coordinates*.fits file in fiberassign dir {dirname}' if force: log.error(message + '; continuing anyway') coordfile = None else: raise FileNotFoundError(message) elif len(globfiles) > 1: raise RuntimeError( f'Multiple coordinates*.fits files in fiberassign dir {dirname}') #- And guide file dirname, filename = os.path.split(fafile) globfiles = glob.glob(dirname + '/guide-????????.fits.fz') if len(globfiles) == 0: #- try falling back to acquisition image globfiles = glob.glob(dirname + '/guide-????????-0000.fits.fz') if len(globfiles) == 1: guidefile = globfiles[0] elif len(globfiles) == 0: message = f'No guide-*.fits.fz file in fiberassign dir {dirname}' if force: log.error(message + '; continuing anyway') guidefile = None else: raise FileNotFoundError(message) elif len(globfiles) > 1: raise RuntimeError( f'Multiple guide-*.fits.fz files in fiberassign dir {dirname}') #- Preflight announcements log.info(f'Night {night} spectro expid {expid}') log.info(f'Raw data file {rawfile}') log.info(f'Fiberassign file {fafile}') log.info(f'Platemaker coordinates file {coordfile}') log.info(f'Guider file {guidefile}') #---- #- Read and assemble fa = Table.read(fafile, 'FIBERASSIGN') fa.sort('LOCATION') #- also read extra keywords from HDU 0 fa_hdr0 = fits.getheader(fafile, 0) if 'OUTDIR' in fa_hdr0: fa_hdr0.rename_keyword('OUTDIR', 'FAOUTDIR') skipkeys = ['SIMPLE', 'EXTEND', 'COMMENT', 'EXTNAME', 'BITPIX', 'NAXIS'] addkeys(fa.meta, fa_hdr0, skipkeys=skipkeys) #- Read platemaker (pm) coordinates file; 3 formats to support: # 1. has FLAGS_CNT/EXP_n and DX_n, DX_n (e.g. 20201214/00067678) # 2. has FLAGS_CNT/EXP_n but not DX_n, DY_n (e.g. 20210402/00083144) # 3. doesn't have any of these (e.g. 20201220/00069029) # Notes: # * don't use FIBER_DX/DY because some files are missing those # (e.g. 20210224/00077902) # * don't use FLAGS_COR_n because some files are missing that # (e.g. 20210402/00083144) pm = None numiter = 0 if coordfile is None: log.error('No coordinates file, thus no info on fiber positioning') else: pm = Table.read(coordfile, 'DATA') #- PM = PlateMaker #- If missing columns *and* not the first in a (split) sequence, #- try again with the first expid in the sequence #- (e.g. 202010404/00083419 -> 83418) if 'DX_0' not in pm.colnames: log.error(f'Missing DX_0 in {coordfile}') if 'VISITIDS' in rawheader: firstexp = int(rawheader['VISITIDS'].split(',')[0]) if firstexp != rawheader['EXPID']: origcorrdfile = coordfile coordfile = findfile('coordinates', night, firstexp) log.info(f'trying again with {coordfile}') pm = Table.read(coordfile, 'DATA') else: log.error(f'no earlier coordinates file for this tile') else: log.error( 'Missing VISITIDS header keywords to find earlier coordinates file' ) if 'FLAGS_CNT_0' not in pm.colnames: log.error( f'Missing spotmatch FLAGS_CNT_0 in {coordfile}; no positioner offset info' ) pm = None numiter = 0 else: #- Count number of iterations in file numiter = len( [col for col in pm.colnames if col.startswith('FLAGS_CNT_')]) log.info(f'Using FLAGS_CNT_{numiter-1} in {coordfile}') #- Now let's merge that platemaker coordinates table (pm) with fiberassign if pm is not None: pm['LOCATION'] = 1000 * pm['PETAL_LOC'] + pm['DEVICE_LOC'] keep = np.in1d(pm['LOCATION'], fa['LOCATION']) pm = pm[keep] pm.sort('LOCATION') log.info('{}/{} fibers in coordinates file'.format(len(pm), len(fa))) #- Create fibermap table to merge with fiberassign file fibermap = Table() fibermap['LOCATION'] = pm['LOCATION'] fibermap['NUM_ITER'] = numiter #- Sometimes these columns are missing in the coordinates files, maybe #- only when numiter=1, i.e. only a blind move but not corrections? if f'FPA_X_{numiter-1}' in pm.colnames: fibermap['FIBER_X'] = pm[f'FPA_X_{numiter-1}'] fibermap['FIBER_Y'] = pm[f'FPA_Y_{numiter-1}'] fibermap['DELTA_X'] = pm[f'DX_{numiter-1}'] fibermap['DELTA_Y'] = pm[f'DY_{numiter-1}'] else: log.warning( 'No FIBER_X/Y or DELTA_X/Y information from platemaker') fibermap['FIBER_X'] = np.zeros(len(pm)) fibermap['FIBER_Y'] = np.zeros(len(pm)) fibermap['DELTA_X'] = np.zeros(len(pm)) fibermap['DELTA_Y'] = np.zeros(len(pm)) #- Bit definitions at https://desi.lbl.gov/trac/wiki/FPS/PositionerFlags #- FLAGS_EXP bit 2 is for positioners (not FIF, GIF, ...) #- These should match what is in fiberassign expflags = pm[f'FLAGS_EXP_{numiter-1}'] good = ((expflags & 4) == 4) if np.any(~good): badloc = list(pm['LOCATION'][~good]) log.error( f'Flagging {len(badloc)} locations without POS_POS bit set: {badloc}' ) #- Keep only matched positioners (FLAGS_CNT_n bit 0) cntflags = pm[f'FLAGS_CNT_{numiter-1}'] spotmatched = ((cntflags & 1) == 1) num_nomatch = np.sum(good & ~spotmatched) if num_nomatch > 0: badloc = list(pm['LOCATION'][good & ~spotmatched]) log.error( f'Flagging {num_nomatch} unmatched fiber locations: {badloc}') good &= spotmatched #- Add our own requirement on good positioning if ((f'DX_{numiter-1}' in pm.colnames) and (f'DY_{numiter-1}' in pm.colnames)): #- offset in cm -> um dr = np.sqrt(pm[f'DX_{numiter-1}']**2 + pm[f'DY_{numiter-1}']**2) * 1000 goodpos = (dr < 100) #- HARDCODE num_badpos = np.sum(good & ~goodpos) if num_badpos > 0: log.error( f'Flagging {num_badpos} positioners >100 um off target') good &= goodpos bad = ~good fibermap['_BADPOS'] = np.zeros(len(fibermap), dtype=bool) fibermap['_BADPOS'][bad] = True #- Missing columns from coordinates file... if ('FIBER_RA' in pm.colnames) and ('FIBER_DEC' in pm.colnames): fibermap['FIBER_RA'] = pm['FIBER_RA'] fibermap['FIBER_DEC'] = pm['FIBER_DEC'] else: log.warning('No FIBER_RA or FIBER_DEC from platemaker') fibermap['FIBER_RA'] = np.zeros(len(pm)) fibermap['FIBER_DEC'] = np.zeros(len(pm)) fibermap = join(fa, fibermap, join_type='left') #- Set fiber status bits missing = np.in1d(fibermap['LOCATION'], pm['LOCATION'], invert=True) fibermap['FIBERSTATUS'][missing] |= fibermask.MISSINGPOSITION badpos = fibermap['_BADPOS'] fibermap['FIBERSTATUS'][badpos] |= fibermask.BADPOSITION fibermap.remove_column('_BADPOS') else: #- No coordinates file or no positioning iterations; #- just use fiberassign + dummy columns log.error( 'Unable to find useful coordinates file; proceeding with fiberassign + dummy columns' ) fibermap = fa fibermap['NUM_ITER'] = 0 fibermap['FIBER_X'] = 0.0 fibermap['FIBER_Y'] = 0.0 fibermap['DELTA_X'] = 0.0 fibermap['DELTA_Y'] = 0.0 fibermap['FIBER_RA'] = 0.0 fibermap['FIBER_DEC'] = 0.0 # Update data types to be consistent with updated value if coord file was used. for val in ['FIBER_X', 'FIBER_Y', 'DELTA_X', 'DELTA_Y']: old_col = fibermap[val] fibermap.replace_column( val, Table.Column(name=val, data=old_col.data, dtype='>f8')) for val in ['LOCATION', 'NUM_ITER']: old_col = fibermap[val] fibermap.replace_column( val, Table.Column(name=val, data=old_col.data, dtype=np.int64)) #- Update SKY and STD target bits to be in both CMX_TARGET and DESI_TARGET #- i.e. if they are set in one, also set in the other. Ditto for SV* for targetcol in ['CMX_TARGET', 'SV0_TARGET', 'SV1_TARGET', 'SV2_TARGET']: if targetcol in fibermap.colnames: for mask in [ desi_mask.SKY, desi_mask.STD_FAINT, desi_mask.STD_BRIGHT ]: ii = (fibermap[targetcol] & mask) != 0 iidesi = (fibermap['DESI_TARGET'] & mask) != 0 fibermap[targetcol][iidesi] |= mask fibermap['DESI_TARGET'][ii] |= mask #- Add header information from rawfile log.debug(f'Adding header keywords from {rawfile}') skipkeys = [ 'EXPTIME', ] addkeys(fibermap.meta, rawheader, skipkeys=skipkeys) fibermap['EXPTIME'] = rawheader['EXPTIME'] #- Add header info from guide file #- sometimes full header is in HDU 0, other times HDU 1... if guidefile is not None: log.debug(f'Adding header keywords from {guidefile}') guideheader = fits.getheader(guidefile, 0) if 'TILEID' not in guideheader: guideheader = fits.getheader(guidefile, 1) if fibermap.meta['TILEID'] != guideheader['TILEID']: raise RuntimeError('fiberassign tile {} != guider tile {}'.format( fibermap.meta['TILEID'], guideheader['TILEID'])) addkeys(fibermap.meta, guideheader, skipkeys=skipkeys) fibermap.meta['EXTNAME'] = 'FIBERMAP' #- Early data raw headers had bad >8 char 'FIBERASSIGN' keyword if 'FIBERASSIGN' in fibermap.meta: log.warning('Renaming header keyword FIBERASSIGN -> FIBASSGN') fibermap.meta['FIBASSGN'] = fibermap.meta['FIBERASSIGN'] del fibermap.meta['FIBERASSIGN'] #- similarly for early splits in raw data file if 'USESPLITS' in fibermap.meta: log.warning('Renaming header keyword USESPLITS -> USESPLIT') fibermap.meta['USESPLIT'] = fibermap.meta['USESPLITS'] del fibermap.meta['USESPLITS'] #- Record input guide and coordinates files if guidefile is not None: fibermap.meta['GUIDEFIL'] = os.path.basename(guidefile) else: fibermap.meta['GUIDEFIL'] = 'MISSING' if coordfile is not None: fibermap.meta['COORDFIL'] = os.path.basename(coordfile) else: fibermap.meta['COORDFIL'] = 'MISSING' #- Lastly, mask the fibers defined by badamps if badamps is not None: maskbits = { 'b': fibermask.BADAMPB, 'r': fibermask.BADAMPR, 'z': fibermask.BADAMPZ } ampoffsets = {'A': 0, 'B': 250, 'C': 0, 'D': 250} for (camera, petal, amplifier) in parse_badamps(badamps): maskbit = maskbits[camera] ampoffset = ampoffsets[amplifier] fibermin = int(petal) * 500 + ampoffset fibermax = fibermin + 250 ampfibs = np.arange(fibermin, fibermax) truefmax = fibermax - 1 log.info(f'Masking fibers from {fibermin} to {truefmax} for camera {camera} because of badamp entry '+\ f'{camera}{petal}{amplifier}') ampfiblocs = np.in1d(fibermap['FIBER'], ampfibs) fibermap['FIBERSTATUS'][ampfiblocs] |= maskbit #- Some code incorrectly relies upon the fibermap being sorted by #- fiber number, so accomodate that before returning the table fibermap.sort('FIBER') return fibermap
def toplevel(qaprod_dir=None): """ Generate HTML to top level QA Mainly generates the highest level HTML file which has links to the Exposure and Calib QA. This also slurps any .png files in the top-level Parameters ---------- setup : str cbset : str det : int Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_toplevel_html', qaprod_dir=qaprod_dir) html_path,_ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Top Level QA') # Calib? calib2d_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir) if os.path.exists(calib2d_file): # Truncate the path c2d_path, fname = os.path.split(calib2d_file) last_slash = c2d_path.rfind('/') f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(c2d_path[last_slash+1:]+'/'+fname)) # Full path #f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(calib2d_file)) # Exposures? exposures_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir) if os.path.exists(exposures_file): # Truncated path exp_path, fname = os.path.split(exposures_file) last_slash = exp_path.rfind('/') f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exp_path[last_slash+1:]+'/'+fname)) # Full path #f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exposures_file)) # Existing PNGs f.write('<hr>\n') f.write('<h2>PNGs</h2>\n') all_png = glob.glob(html_path+'/*.png') all_png.sort() # Type links = '' body = '' for png in all_png: _, png_file = os.path.split(png) # Image href="{:s}".format(png_file[:-4]) links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format(href, href) body += '<div class="section" id="{:s}">\n'.format(href) body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format(png_file) f.write('<h3><ul>\n') f.write(links) f.write('</ul></h3>\n') f.write(body) # Finish finish(f,'') # Return return
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat','arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple=read_stdstar_models(stdstar_fil) flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def ql_main(args=None): qlog = qllogger.QLLogger("QuickLook", 20) log = qlog.getlog() if args is None: args = parse() if args.config is not None: if args.rawdata_dir: rawdata_dir = args.rawdata_dir else: if 'QL_SPEC_DATA' not in os.environ: sys.exit( "must set ${} environment variable or provide rawdata_dir". format('QL_SPEC_DATA')) rawdata_dir = os.getenv('QL_SPEC_DATA') if args.specprod_dir: specprod_dir = args.specprod_dir else: if 'QL_SPEC_REDUX' not in os.environ: sys.exit( "must set ${} environment variable or provide specprod_dir" .format('QL_SPEC_REDUX')) specprod_dir = os.getenv('QL_SPEC_REDUX') log.info("Running Quicklook using configuration file {}".format( args.config)) if os.path.exists(args.config): if "yaml" in args.config: config = qlconfig.Config(args.config, args.night, args.camera, args.expid, rawdata_dir=rawdata_dir, specprod_dir=specprod_dir) configdict = config.expand_config() else: log.critical("Can't open config file {}".format(args.config)) sys.exit("Can't open config file") else: sys.exit("File does not exist: {}".format(args.config)) elif args.fullconfig is not None: #- This is mostly for development/debugging purpose log.info("Running Quicklook using full configuration file {}".format( args.fullconfig)) if os.path.exists(args.fullconfig): if "yaml" in args.fullconfig: configdict = yaml.load(open(args.fullconfig, "r")) else: log.critical("Can't open config file {}".format(args.config)) sys.exit("Can't open config file") else: sys.exit("File does not exist: {}".format(args.config)) else: sys.exit( "Must provide a valid config file. See desispec/data/quicklook for an example" ) #- save the expanded config to a file if args.save: if "yaml" in args.save: f = open(args.save, "w") yaml.dump(configdict, f) log.info("Output saved for this configuration to {}".format( args.save)) f.close() else: log.info( "Can save config to only yaml output. Put a yaml in the argument" ) pipeline, convdict = quicklook.setup_pipeline(configdict) res = quicklook.runpipeline(pipeline, convdict, configdict, mergeQA=args.mergeQA) inpname = configdict["RawImage"] night = configdict["Night"] camera = configdict["Camera"] expid = configdict["Expid"] if isinstance(res, image.Image): if configdict["OutputFile"]: finalname = configdict["OutputFile"] else: finalname = "image-{}-{:08d}.fits".format(camera, expid) log.critical( "No final outputname given. Writing to a image file {}".format( finalname)) imIO.write_image(finalname, res, meta=None) elif isinstance(res, frame.Frame): if configdict["Flavor"] == 'arcs': from desispec.io.meta import findfile finalname = "psfnight-{}.fits".format(camera) finalframe = findfile('frame', night=night, expid=expid, camera=camera) frIO.write_frame(finalframe, res, header=None) else: if configdict["OutputFile"]: finalname = configdict["OutputFile"] else: finalname = "frame-{}-{:08d}.fits".format(camera, expid) log.critical( "No final outputname given. Writing to a frame file {}". format(finalname)) frIO.write_frame(finalname, res, header=None) else: log.error( "Result of pipeline is an unknown type {}. Don't know how to write" .format(type(res))) sys.exit("Unknown pipeline result type {}.".format(type(res))) log.info("Pipeline completed. Final result is in {}".format(finalname))
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write = True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe, ), clobber=clobber): write = True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array( s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]] * len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
if flavor == 'science': parts = prog.split(' ') if parts[0] == 'SV1': if parts[1] == targets: parts = x.split('/') night = parts[9] expid = np.int(parts[10]) name = parts[-1] parts = name.split('-') camera = parts[1] calib = findfile('fluxcalib', night=night, expid=expid, camera=camera, specprod_dir=None) # findfile('cframe', night=night, expid=expid, camera=camera, specprod_dir=blanc) cframe = fits.open(x) hdr = cframe[0].header['FIBERFLT'] hdr = hdr.replace('SPECPROD', blanc) tileid = cframe[0].header['TILEID'] if tileid != tile: print('Skipping tile {}'.format(tileid)) continue iin = x.replace('cframe', 'frame') sky = x.replace('cframe', 'sky') psf = sky.replace('sky', 'psf')
def toplevel(): """ Generate HTML to top level QA Parameters ---------- setup : str cbset : str det : int Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_toplevel_html') html_path, _ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Top Level QA') # Calib? calib2d_file = meta.findfile('qa_calib_html') if os.path.exists(calib2d_file): # Truncate the path c2d_path, fname = os.path.split(calib2d_file) last_slash = c2d_path.rfind('/') f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format( c2d_path[last_slash + 1:] + '/' + fname)) # Full path #f.write('<h2><a href="{:s}">Calibration QA</a></h2>\n'.format(calib2d_file)) # Exposures? exposures_file = meta.findfile('qa_exposures_html') if os.path.exists(exposures_file): # Truncated path exp_path, fname = os.path.split(exposures_file) last_slash = exp_path.rfind('/') f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format( exp_path[last_slash + 1:] + '/' + fname)) # Full path #f.write('<h2><a href="{:s}">Exposures QA</a></h2>\n'.format(exposures_file)) # Existing PNGs f.write('<hr>\n') f.write('<h2>PNGs</h2>\n') all_png = glob.glob(html_path + '/*.png') all_png.sort() # Type links = '' body = '' for png in all_png: _, png_file = os.path.split(png) # Image href = "{:s}".format(png_file[:-4]) links += '<li><a class="reference internal" href="#{:s}">{:s}</a></li>\n'.format( href, href) body += '<div class="section" id="{:s}">\n'.format(href) body += '<img class ="research" src="{:s}" width="100%" height="auto"/>\n'.format( png_file) f.write('<h3><ul>\n') f.write(links) f.write('</ul></h3>\n') f.write(body) # Finish finish(f, '') # Return return
def SNRFit(frame, night, camera, expid, objlist, params, fidboundary=None): """ Signal vs. Noise With fitting Take flux and inverse variance arrays and calculate S/N for individual targets (ELG, LRG, QSO, STD) and for each amplifier of the camera. then fit the log(snr)=a+b*mag or log(snr)=poly(mag) see http://arXiv.org/abs/0706.1062v2 for proper fitting of power-law distributions it is not implemented here! Args: frame: desispec.Frame object params: parameters dictionary { "Func": "linear", # Fit function type one of ["linear","poly"] "FIDMAG": 22.0, # magnitude to evaluate the fit "Filter":"DECAM_R", #filter name } fidboundary : list of slices indicating where to select in fiber and wavelength directions for each amp (output of slice_fidboundary function) Returns a dictionary similar to SignalVsNoise """ #- Get imaging magnitudes and calculate S/N magnitudes = frame.fibermap['MAG'] fmag = 22.0 if "FIDMAG" in params: fmag = params["FIDMAG"] filters = frame.fibermap['FILTER'] mediansnr = SN_ratio(frame.flux, frame.ivar) qadict = {"MEDIAN_SNR": mediansnr} if camera[0] == 'b': thisfilter = 'DECAM_G' #- should probably come from param. Hard coding for now elif camera[0] == 'r': thisfilter = 'DECAM_R' else: thisfilter = 'DECAM_Z' if "Filter" in params: thisfilter = params["Filter"] #- This is a very basic (and temporary!!) flux calibration #- used to convert sky background and noise to proper flux units #- NOTE: the B+R**2 term in the S/N equation is being multipled #- by 1e-33 based on simple unit conversion analysis comparing #- imaging magnitude converted to flux and integrated spectral counts try: #- Get sky background and read noise from previous QAs cfile = findfile('ql_skycont_file', int(night), int(expid), camera, specprod_dir=os.environ['QL_SPEC_REDUX']) with open(cfile) as cf: contfile = yaml.load(cf) contval = contfile["METRICS"]["SKYCONT"] rfile = findfile('ql_getrms_file', int(night), int(expid), camera, specprod_dir=os.environ['QL_SPEC_REDUX']) with open(rfile) as rf: rmsfile = yaml.load(rf) rmsval = rmsfile["METRICS"]["NOISE_OVER"] #- NOTE: the 1e-3 used here is because scipy doesn't converge with #- such small flux values (thus multiplying the flux and conversion #- factor of 1e-33 by 1e30, which does not change any output) br2 = 1e-3 * (len(frame.wave) * (contval + rmsval**2)) #- Set up fit of SNR vs. Mag. #- Using astronomical S/N equation, fitting only 'a' funcMap = { "linear": lambda x, a, b: a + b * x, "poly": lambda x, a, b, c: a + b * x + c * x**2, "astro": lambda x, a: (a * x) / np.sqrt(a * x + br2) } except: br2 = None log.info( "Was not able to calculate B+R**2 from prior knowledge, fitting this too..." ) #- Fitting 'a' and 'B+R**2' because the necessary information wasn't available) funcMap = { "linear": lambda x, a, b: a + b * x, "poly": lambda x, a, b, c: a + b * x + c * x**2, "astro": lambda x, a, b: (a * x) / np.sqrt(a * x + b) } fitfunc = funcMap["astro"] if br2 is None: initialParams = [0.3, 600.] else: initialParams = [0.3] neg_snr_tot = [] #- neg_snr_tot counts the number of times a fiber has a negative median SNR. This should #- not happen for non-sky fibers with actual flux in them. However, it does happen rarely #- in sims. To avoid this, we omit such fibers in the fit, but keep count for diagnostic #- purposes. #- Loop over each target type, and associate SNR and image magnitudes for each type. ra = [] dec = [] resid_snr = [] fidsnr_tgt = [] fitcoeff = [] fitcovar = [] snrmag = [] badfibs = [] fitsnr = [] for T in objlist: fibers = np.where(frame.fibermap['OBJTYPE'] == T)[0] medsnr = mediansnr[fibers] mags = np.zeros(medsnr.shape) if T == "STD": # this should be fixed "STAR" or "STD" should be used consistently everywhere! T = "STAR" for ii, fib in enumerate(fibers): if thisfilter not in filters[fib]: print( "WARNING!!! {} magnitude is not available filter for fiber {}" .format(thisfilter, fib)) mags[ii] = None else: mags[ii] = magnitudes[fib][filters[fib] == thisfilter] try: #- Determine invalid SNR and mag values and remove m = mags s = medsnr neg_snr = len(np.where(medsnr <= 0.0)[0]) neg_snr_tot.append(neg_snr) neg_val = np.where(medsnr <= 0.0)[0] inf_mag = np.where(m == np.inf)[0] nan_mag = np.where(np.isnan(m))[0] none_mag = np.where(m == np.array(None))[0] cut = [] cuts = [neg_val, inf_mag, nan_mag, none_mag] for cc in range(len(cuts)): if len(cuts[cc]) > 0: for ci in range(len(cuts[cc])): cut.append(cuts[cc][ci]) if len(cut) > 0: m = list(m) s = list(s) makecut = sorted(list(set(cut))) for nn in range(len(makecut)): m.remove(m[makecut[nn]]) s.remove(s[makecut[nn]]) for ni in range(len(makecut)): makecut[ni] -= 1 m = np.array(m) s = np.array(s) log.warning( "In fit of {}, had to remove NANs from data for fitting!". format(T)) badfibs.append(fibers[sorted(list(set(cut)))]) xs = m.argsort() #- Using AB magnitude system for conversion to flux x = 1e30 * (10**(-0.4 * (m[xs] + 48.6))) med_snr = s[xs] y = med_snr #- Fit SNR vs. Mag. to fit function, evaluate at fiducial magnitude, #- and store results in METRICS out = optimize.curve_fit(fitfunc, x, y, p0=initialParams) vs = list(out[0]) cov = list(out[1]) fitcoeff.append(vs) fitcovar.append(cov) fidsnr_tgt.append(fitfunc(fmag, *out[0])) except RuntimeError: log.warning("In fit of {}, Fit minimization failed!".format(T)) vs = np.array(initialParams) vs.fill(np.nan) cov = np.empty((len(initialParams), len(initialParams))) cov.fill(np.nan) vs = list(vs) cov = list(cov) fitcoeff.append(vs) fitcovar.append(cov) fidsnr_tgt.append(np.nan) except scipy.optimize.OptimizeWarning: log.warning( "WARNING!!! {} Covariance estimation failed!".format(T)) vs = out[0] cov = np.empty((len(initialParams), len(initialParams))) cov.fill(np.nan) vs = list(vs) cov = list(cov) fitcoeff.append(vs) fitcovar.append(cov) fidsnr_tgt.append(np.nan) qadict["%s_FIBERID" % T] = fibers.tolist() snr_mag = [medsnr, mags] snrmag.append(snr_mag) #- Calculate residual S/N for focal plane plots fit_snr = [] for mm in range(len(x)): snr = fitfunc(x[mm], *vs) fit_snr.append(snr) for rr in range(len(fit_snr)): resid = (med_snr[rr] - fit_snr[rr]) / fit_snr[rr] resid_snr.append(resid) fitsnr.append(fit_snr) qadict["NUM_NEGATIVE_SNR"] = sum(neg_snr_tot) qadict["SNR_MAG_TGT"] = snrmag qadict["FITCOEFF_TGT"] = fitcoeff qadict["FITCOVAR_TGT"] = fitcovar qadict["SNR_RESID"] = resid_snr qadict["FIDSNR_TGT"] = fidsnr_tgt qadict["RA"] = frame.fibermap['RA_TARGET'] qadict["DEC"] = frame.fibermap['DEC_TARGET'] return qadict, badfibs, fitsnr
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path,_ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path,_ = os.path.split(path) # Remove night fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write=True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe,), clobber=clobber): write=True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
flux = pickle.load(handle)['ELG'] with open( os.environ['CSCRATCH'] + '/radlss/test/ensemble/template-elg-ensemble-objmeta.fits', 'rb') as handle: objmeta = pickle.load(handle)['ELG'] for band in ['b', 'r', 'z']: cam = band + petal # E.g. ~/andes/exposures/20200315/00055642/cframe-b0-00055642.fits cframes[cam] = read_frame( findfile('cframe', night=night, expid=expid, camera=cam, specprod_dir='/global/homes/m/mjwilson/andes/')) cframes[cam].flux[fiber, :] = flux[band][115] rrz = meta['REDSHIFT'][115] z = 1.0 v = 75.0 r = 0.7 lnA = 2.5 _, mod = doublet(z, twave, sigmav=v, r=r) postages = cframe_postage(cframes, fiber, rrz) postage = postages[3]
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info( "qafile={:s} exists. Not over-writing. Consider clobber=True" .format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile( 'qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile( 'qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile( 'stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple = read_stdstar_models(stdstar_fil) flux_fil = meta.findfile( 'calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) if make_plots: qafig = meta.findfile( 'qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib)) # , model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write write_qa_frame(qafile, qaframe, verbose=True) return qaframe