def __init__(self, night, **kwargs): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. Notes: **kwargs are passed to QA_MultiExp Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ # Init self.night = night # Instantiate QA_MultiExp.__init__(self, **kwargs) # Load up exposures for the full production nights = get_nights(specprod_dir=self.specprod_dir) # Check the night exists if self.night not in nights: raise IOError("Bad input night for this production") # Load up self.mexp_dict[self.night] = {} for exposure in get_exposures(self.night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=self.night, expid=exposure, specprod_dir=self.specprod_dir) self.mexp_dict[self.night][exposure] = frames_dict # Output file names self.qaexp_outroot = self.qaprod_dir + '/' + self.night + '_qa'
def __init__(self, specprod_dir=None, **kwargs): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. Notes: Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ if specprod_dir is None: specprod_dir = specprod_root() self.specprod_dir = specprod_dir # Init QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs) # Load up exposures for the full production nights = get_nights(specprod_dir=self.specprod_dir) for night in nights: self.mexp_dict[night] = {} for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) self.mexp_dict[night][exposure] = frames_dict # Output file names self.qaexp_outroot = self.qaprod_dir + '/' + self.prod_name + '_qa' # Nights list self.qa_nights = []
def load_s2n_values(objtype, nights, channel, sub_exposures=None): fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night) #, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format( flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data, masked=True) sps_hdu.close() objs = sps_tab['TEMPLATETYPE'] == objtype if np.sum(objs) == 0: continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel + str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: cframe = read_frame(cframe_path) except: log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave, 1) dwave[0] = dwave[1] # iobjs = objs[cframe.fibers] if np.sum(iobjs) == 0: continue s2n = cframe.flux[iobjs, :] * np.sqrt( cframe.ivar[iobjs, :]) / np.sqrt(dwave) # Save fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]]) if objtype == 'ELG': fdict['OII'].append( sps_tab['OIIFLUX'][cframe.fibers[iobjs]]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def slurp(self, make_frameqa=False, remove=True, **kwargs): """ Slurp all the individual QA files into one master QA file Args: make_frameqa: bool, optional Regenerate the individual QA files (at the frame level first) remove: bool, optional Remove Returns: """ from desispec.io import meta from desispec.qa import QA_Exposure from desispec.io import write_qa_prod import pdb log = get_logger() # Remake? if make_frameqa: self.make_frameqa(**kwargs) # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] # Reset log.info("Resetting qa_exps in qa_prod") self.qa_exps = [] # Loop for night in nights: # Loop on exposures for exposure in get_exposures(night, specprod_dir=self.specprod_dir): frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) if len(frames_dict) == 0: continue # Load any frame (for the type) key = list(frames_dict.keys())[0] frame_fil = frames_dict[key] frame = read_frame(frame_fil) qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'], specprod_dir=self.specprod_dir, remove=remove) # Append self.qa_exps.append(qa_exp) # Write outroot = self.specprod_dir + '/' + self.prod_name + '_qa' write_qa_prod(outroot, self)
def slurp(self, make_frameqa=False, remove=True, **kwargs): """ Slurp all the individual QA files into one master QA file Args: make_frameqa: bool, optional Regenerate the individual QA files (at the frame level first) remove: bool, optional Remove Returns: """ from desispec.io import meta from desispec.qa import QA_Exposure from desispec.io import write_qa_prod import pdb # Remake? if make_frameqa: self.make_frameqa(**kwargs) # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] # Reset log.info("Resetting qa_exps in qa_prod") self.qa_exps = [] # Loop for night in nights: # Loop on exposures for exposure in get_exposures(night, specprod_dir = self.specprod_dir): frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) if len(frames_dict.keys()) == 0: continue # Load any frame (for the type) key = frames_dict.keys()[0] frame_fil = frames_dict[key] frame = read_frame(frame_fil) qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'], specprod_dir=self.specprod_dir, remove=remove) # Append self.qa_exps.append(qa_exp) # Write outroot = self.specprod_dir+'/'+self.prod_name+'_qa' write_qa_prod(outroot, self)
def make_exposures(): """ Generate HTML to organize exposure HTML Parameters ---------- Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_exposures_html') html_path, _ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposures QA') # Loop on Nights nights = get_nights() nights.sort() links = '' body = '' for night in nights: # HTML f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') # Loop on expsoures for expid in get_exposures(night): if not os.path.exists(html_path + '/' + night + '/{:08d}'.format(expid)): continue # Link f.write( '<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n' .format(night, expid, expid, expid)) # Generate Exposure html make_exposure(night, expid) f.write('</ul></h3>\n') # Finish finish(f, body)
def make_exposures(qaprod_dir=None): """ Generate HTML to organize exposure HTML Parameters ---------- Returns ------- links : str body : str """ # Organized HTML html_file = meta.findfile('qa_exposures_html', qaprod_dir=qaprod_dir) html_path,_ = os.path.split(html_file) f = open(html_file, 'w') init(f, 'Exposures QA') # Loop on Nights nights = get_nights() nights.sort() links = '' body = '' for night in nights: # HTML f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') # Loop on expsoures for expid in get_exposures(night): if not os.path.exists(html_path+'/'+night+'/{:08d}'.format(expid)): continue # Link f.write('<li><a href="{:s}/{:08d}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'.format(night, expid, expid, expid)) # Generate Exposure html make_exposure(night, expid, qaprod_dir=qaprod_dir) f.write('</ul></h3>\n') # Finish finish(f,body)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.qa.qa_frame import qaframe_from_frame from desispec.io.fluxcalibration import read_stdstar_models log = get_logger() # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame qaframe_from_frame(frame_fil, make_plots=make_plots) '''
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat','arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple=read_stdstar_models(stdstar_fil) flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def main(args): # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log = get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir=args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat', 'arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(), 1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def graph_night(rawdir, rawnight): grph = {} node = {} node['type'] = 'night' node['in'] = [] node['out'] = [] grph[rawnight] = node allbricks = {} expcount = {} expcount['flat'] = 0 expcount['arc'] = 0 expcount['science'] = 0 # First, insert raw data into the graph. We use the existence of the raw data # as a filter over spectrographs. Spectrographs whose raw data do not exist # are excluded from the graph. expid = io.get_exposures(rawnight, raw=True, rawdata_dir=rawdir) campat = re.compile(r'([brz])([0-9])') keepspec = set() for ex in sorted(expid): # get the fibermap for this exposure fibermap = io.get_raw_files("fibermap", rawnight, ex, rawdata_dir=rawdir) # read the fibermap to get the exposure type, and while we are at it, # also accumulate the total list of bricks fmdata, fmheader = io.read_fibermap(fibermap, header=True) flavor = fmheader['flavor'] fmbricks = {} for fmb in fmdata['BRICKNAME']: if len(fmb) > 0: if fmb in fmbricks.keys(): fmbricks[fmb] += 1 else: fmbricks[fmb] = 1 for fmb in fmbricks.keys(): if fmb in allbricks.keys(): allbricks[fmb] += fmbricks[fmb] else: allbricks[fmb] = fmbricks[fmb] if flavor == 'arc': expcount['arc'] += 1 elif flavor == 'flat': expcount['flat'] += 1 else: expcount['science'] += 1 node = {} node['type'] = 'fibermap' node['id'] = ex node['flavor'] = flavor node['bricks'] = fmbricks node['in'] = [rawnight] node['out'] = [] name = graph_name(rawnight, "fibermap-{:08d}".format(ex)) grph[name] = node grph[rawnight]['out'].append(name) # get the raw exposures raw = io.get_raw_files("pix", rawnight, ex, rawdata_dir=rawdir) for cam in sorted(raw.keys()): cammat = campat.match(cam) if cammat is None: raise RuntimeError("invalid camera string {}".format(cam)) band = cammat.group(1) spec = cammat.group(2) keepspec.update(spec) node = {} node['type'] = 'pix' node['id'] = ex node['band'] = band node['spec'] = spec node['flavor'] = flavor node['in'] = [rawnight] node['out'] = [] name = graph_name(rawnight, "pix-{}{}-{:08d}".format(band, spec, ex)) grph[name] = node grph[rawnight]['out'].append(name) keep = sorted(list(keepspec)) # Now that we have added all the raw data to the graph, we work our way # through the processing steps. # This step is a placeholder, in case we want to combine information from # multiple flats or arcs before running bootcalib. We mark these bootcalib # outputs as depending on all arcs and flats, but in reality we may just # use the first or last set. # Since each psfboot file takes multiple exposures as input, we first # create those nodes. for band in ['b', 'r', 'z']: for spec in keep: name = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) node = {} node['type'] = 'psfboot' node['band'] = band node['spec'] = spec node['in'] = [] node['out'] = [] grph[name] = node for name, nd in grph.items(): if nd['type'] != 'pix': continue if (nd['flavor'] != 'flat') and (nd['flavor'] != 'arc'): continue band = nd['band'] spec = nd['spec'] bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) grph[bootname]['in'].append(name) nd['out'].append(bootname) # Next is full PSF estimation. Inputs are the arc image and the bootcalib # output file. We also add nodes for the combined psfs. for band in ['b', 'r', 'z']: for spec in keep: name = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) node = {} node['type'] = 'psfnight' node['band'] = band node['spec'] = spec node['in'] = [] node['out'] = [] grph[name] = node for name, nd in grph.items(): if nd['type'] != 'pix': continue if nd['flavor'] != 'arc': continue band = nd['band'] spec = nd['spec'] id = nd['id'] bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) psfname = graph_name(rawnight, "psf-{}{}-{:08d}".format(band, spec, id)) psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) node = {} node['type'] = 'psf' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, bootname] node['out'] = [psfnightname] grph[psfname] = node grph[bootname]['out'].append(psfname) grph[psfnightname]['in'].append(psfname) nd['out'].append(psfname) # Now we extract the flats and science frames using the nightly psf for name, nd in grph.items(): if nd['type'] != 'pix': continue if nd['flavor'] == 'arc': continue band = nd['band'] spec = nd['spec'] id = nd['id'] flavor = nd['flavor'] framename = graph_name(rawnight, "frame-{}{}-{:08d}".format(band, spec, id)) psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) fmname = graph_name(rawnight, "fibermap-{:08d}".format(id)) node = {} node['type'] = 'frame' node['band'] = band node['spec'] = spec node['id'] = id node['flavor'] = flavor node['in'] = [name, fmname, psfnightname] node['out'] = [] grph[framename] = node grph[psfnightname]['out'].append(framename) grph[fmname]['out'].append(framename) nd['out'].append(framename) # Now build the fiberflats for each flat exposure. We keep a list of all # available fiberflats while we are looping over them, since we'll need # that in the next step to select the "most recent" fiberflat. flatexpid = {} for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] != 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'fiberflat' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name] node['out'] = [] grph[flatname] = node nd['out'].append(flatname) cam = "{}{}".format(band, spec) if cam not in flatexpid.keys(): flatexpid[cam] = [] flatexpid[cam].append(id) # To compute the sky file, we use the "most recent fiberflat" that came # before the current exposure. for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) node = {} node['type'] = 'sky' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname] node['out'] = [] grph[skyname] = node nd['out'].append(skyname) grph[flatname]['out'].append(skyname) # Construct the standard star files. These are one per spectrograph, # and depend on the frames and the corresponding flats and sky files. stdgrph = {} for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id)) # does this spectrograph exist yet in the graph? if starname not in stdgrph.keys(): fmname = graph_name(rawnight, "fibermap-{:08d}".format(id)) grph[fmname]['out'].append(starname) node = {} node['type'] = 'stdstars' node['spec'] = spec node['id'] = id node['in'] = [fmname] node['out'] = [] stdgrph[starname] = node cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) stdgrph[starname]['in'].extend([skyname, name, flatname]) nd['out'].append(starname) grph[flatname]['out'].append(starname) grph[skyname]['out'].append(starname) grph.update(stdgrph) # Construct calibration files for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'calib' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname, skyname, starname] node['out'] = [] grph[calname] = node grph[flatname]['out'].append(calname) grph[skyname]['out'].append(calname) grph[starname]['out'].append(calname) nd['out'].append(calname) # Build cframe files for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id)) cfname = graph_name(rawnight, "cframe-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'cframe' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname, skyname, calname] node['out'] = [] grph[cfname] = node grph[flatname]['out'].append(cfname) grph[skyname]['out'].append(cfname) grph[calname]['out'].append(cfname) nd['out'].append(cfname) # Brick / Zbest dependencies for b in allbricks.keys(): zbname = "zbest-{}".format(b) inb = [] for band in ['b', 'r', 'z']: node = {} node['type'] = 'brick' node['brick'] = b node['band'] = band node['in'] = [] node['out'] = [zbname] bname = "brick-{}-{}".format(band, b) inb.append(bname) grph[bname] = node node = {} node['type'] = 'zbest' node['brick'] = b node['ntarget'] = allbricks[b] node['in'] = inb node['out'] = [] grph[zbname] = node for name, nd in grph.items(): if nd['type'] != 'fibermap': continue if nd['flavor'] == 'arc': continue if nd['flavor'] == 'flat': continue id = nd['id'] bricks = nd['bricks'] for band in ['b', 'r', 'z']: for spec in keep: cfname = graph_name(rawnight, "cframe-{}{}-{:08d}".format(band, spec, id)) for b in bricks: bname = "brick-{}-{}".format(band, b) grph[bname]['in'].append(cfname) grph[cfname]['out'].append(bname) return (grph, expcount, allbricks)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info( "qafile={:s} exists. Not over-writing. Consider clobber=True" .format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile( 'qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile( 'qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile( 'stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple = read_stdstar_models(stdstar_fil) flux_fil = meta.findfile( 'calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) if make_plots: qafig = meta.findfile( 'qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def graph_night(rawdir, rawnight): grph = {} node = {} node['type'] = 'night' node['in'] = [] node['out'] = [] grph[rawnight] = node allbricks = {} expcount = {} expcount['flat'] = 0 expcount['arc'] = 0 expcount['science'] = 0 # First, insert raw data into the graph. We use the existence of the raw data # as a filter over spectrographs. Spectrographs whose raw data do not exist # are excluded from the graph. expid = io.get_exposures(rawnight, raw=True, rawdata_dir=rawdir) campat = re.compile(r'([brz])([0-9])') keepspec = set() for ex in sorted(expid): # get the fibermap for this exposure fibermap = io.get_raw_files("fibermap", rawnight, ex, rawdata_dir=rawdir) # read the fibermap to get the exposure type, and while we are at it, # also accumulate the total list of bricks fmdata, fmheader = io.read_fibermap(fibermap, header=True) flavor = fmheader['flavor'] fmbricks = {} for fmb in fmdata['BRICKNAME']: if len(fmb) > 0: if fmb in fmbricks.keys(): fmbricks[fmb] += 1 else: fmbricks[fmb] = 1 for fmb in fmbricks.keys(): if fmb in allbricks.keys(): allbricks[fmb] += fmbricks[fmb] else: allbricks[fmb] = fmbricks[fmb] if flavor == 'arc': expcount['arc'] += 1 elif flavor == 'flat': expcount['flat'] += 1 else: expcount['science'] += 1 node = {} node['type'] = 'fibermap' node['id'] = ex node['flavor'] = flavor node['bricks'] = fmbricks node['in'] = [rawnight] node['out'] = [] name = graph_name(rawnight, "fibermap-{:08d}".format(ex)) grph[name] = node grph[rawnight]['out'].append(name) # get the raw exposures raw = io.get_raw_files("pix", rawnight, ex, rawdata_dir=rawdir) for cam in sorted(raw.keys()): cammat = campat.match(cam) if cammat is None: raise RuntimeError("invalid camera string {}".format(cam)) band = cammat.group(1) spec = cammat.group(2) keepspec.update(spec) node = {} node['type'] = 'pix' node['id'] = ex node['band'] = band node['spec'] = spec node['flavor'] = flavor node['in'] = [rawnight] node['out'] = [] name = graph_name(rawnight, "pix-{}{}-{:08d}".format(band, spec, ex)) grph[name] = node grph[rawnight]['out'].append(name) keep = sorted(list(keepspec)) # Now that we have added all the raw data to the graph, we work our way # through the processing steps. # This step is a placeholder, in case we want to combine information from # multiple flats or arcs before running bootcalib. We mark these bootcalib # outputs as depending on all arcs and flats, but in reality we may just # use the first or last set. # Since each psfboot file takes multiple exposures as input, we first # create those nodes. for band in ['b', 'r', 'z']: for spec in keep: name = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) node = {} node['type'] = 'psfboot' node['band'] = band node['spec'] = spec node['in'] = [] node['out'] = [] grph[name] = node for name, nd in grph.items(): if nd['type'] != 'pix': continue if (nd['flavor'] != 'flat') and (nd['flavor'] != 'arc'): continue band = nd['band'] spec = nd['spec'] bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) grph[bootname]['in'].append(name) nd['out'].append(bootname) # Next is full PSF estimation. Inputs are the arc image and the bootcalib # output file. We also add nodes for the combined psfs. for band in ['b', 'r', 'z']: for spec in keep: name = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) node = {} node['type'] = 'psfnight' node['band'] = band node['spec'] = spec node['in'] = [] node['out'] = [] grph[name] = node for name, nd in grph.items(): if nd['type'] != 'pix': continue if nd['flavor'] != 'arc': continue band = nd['band'] spec = nd['spec'] id = nd['id'] bootname = graph_name(rawnight, "psfboot-{}{}".format(band, spec)) psfname = graph_name(rawnight, "psf-{}{}-{:08d}".format(band, spec, id)) psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) node = {} node['type'] = 'psf' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, bootname] node['out'] = [psfnightname] grph[psfname] = node grph[bootname]['out'].append(psfname) grph[psfnightname]['in'].append(psfname) nd['out'].append(psfname) # Now we extract the flats and science frames using the nightly psf for name, nd in grph.items(): if nd['type'] != 'pix': continue if nd['flavor'] == 'arc': continue band = nd['band'] spec = nd['spec'] id = nd['id'] flavor = nd['flavor'] framename = graph_name(rawnight, "frame-{}{}-{:08d}".format(band, spec, id)) psfnightname = graph_name(rawnight, "psfnight-{}{}".format(band, spec)) fmname = graph_name(rawnight, "fibermap-{:08d}".format(id)) node = {} node['type'] = 'frame' node['band'] = band node['spec'] = spec node['id'] = id node['flavor'] = flavor node['in'] = [name, fmname, psfnightname] node['out'] = [] grph[framename] = node grph[psfnightname]['out'].append(framename) grph[fmname]['out'].append(framename) nd['out'].append(framename) # Now build the fiberflats for each flat exposure. We keep a list of all # available fiberflats while we are looping over them, since we'll need # that in the next step to select the "most recent" fiberflat. flatexpid = {} for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] != 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'fiberflat' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name] node['out'] = [] grph[flatname] = node nd['out'].append(flatname) cam = "{}{}".format(band, spec) if cam not in flatexpid.keys(): flatexpid[cam] = [] flatexpid[cam].append(id) # To compute the sky file, we use the "most recent fiberflat" that came # before the current exposure. for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) node = {} node['type'] = 'sky' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname] node['out'] = [] grph[skyname] = node nd['out'].append(skyname) grph[flatname]['out'].append(skyname) # Construct the standard star files. These are one per spectrograph, # and depend on the frames and the corresponding flats and sky files. stdgrph = {} for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id)) # does this spectrograph exist yet in the graph? if starname not in stdgrph.keys(): fmname = graph_name(rawnight, "fibermap-{:08d}".format(id)) grph[fmname]['out'].append(starname) node = {} node['type'] = 'stdstars' node['spec'] = spec node['id'] = id node['in'] = [fmname] node['out'] = [] stdgrph[starname] = node cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) stdgrph[starname]['in'].extend([skyname, name, flatname]) nd['out'].append(starname) grph[flatname]['out'].append(starname) grph[skyname]['out'].append(starname) grph.update(stdgrph) # Construct calibration files for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) starname = graph_name(rawnight, "stdstars-{}-{:08d}".format(spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'calib' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname, skyname, starname] node['out'] = [] grph[calname] = node grph[flatname]['out'].append(calname) grph[skyname]['out'].append(calname) grph[starname]['out'].append(calname) nd['out'].append(calname) # Build cframe files for name, nd in grph.items(): if nd['type'] != 'frame': continue if nd['flavor'] == 'flat': continue band = nd['band'] spec = nd['spec'] id = nd['id'] cam = "{}{}".format(band, spec) flatid = None for fid in sorted(flatexpid[cam]): if (flatid is None): flatid = fid elif (fid > flatid) and (fid < id): flatid = fid skyname = graph_name(rawnight, "sky-{}{}-{:08d}".format(band, spec, id)) flatname = graph_name(rawnight, "fiberflat-{}{}-{:08d}".format(band, spec, fid)) calname = graph_name(rawnight, "calib-{}{}-{:08d}".format(band, spec, id)) cfname = graph_name(rawnight, "cframe-{}{}-{:08d}".format(band, spec, id)) node = {} node['type'] = 'cframe' node['band'] = band node['spec'] = spec node['id'] = id node['in'] = [name, flatname, skyname, calname] node['out'] = [] grph[cfname] = node grph[flatname]['out'].append(cfname) grph[skyname]['out'].append(cfname) grph[calname]['out'].append(cfname) nd['out'].append(cfname) # Brick / Zbest dependencies for b in allbricks.keys(): zbname = "zbest-{}".format(b) inb = [] for band in ['b', 'r', 'z']: node = {} node['type'] = 'brick' node['brick'] = b node['band'] = band node['in'] = [] node['out'] = [zbname] bname = "brick-{}-{}".format(band, b) inb.append(bname) grph[bname] = node node = {} node['type'] = 'zbest' node['brick'] = b node['ntarget'] = allbricks[b] node['in'] = inb node['out'] = [] grph[zbname] = node for name, nd in grph.items(): if nd['type'] != 'fibermap': continue if nd['flavor'] == 'arc': continue if nd['flavor'] == 'flat': continue id = nd['id'] bricks = nd['bricks'] for band in ['b', 'r', 'z']: for spec in keep: cfname = graph_name( rawnight, "cframe-{}{}-{:08d}".format(band, spec, id)) for b in bricks: bname = "brick-{}-{}".format(band, b) grph[bname]['in'].append(cfname) grph[cfname]['out'].append(bname) return (grph, expcount, allbricks)
def main(args) : # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log=get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir = args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(),1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def main(args): # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files, get_nights from desispec.io import read_frame from desispec.io import get_reduced_frames from desispec.io.sky import read_sky from desispec.io import specprod_root from desispec.qa import utils as qa_utils import copy import pdb # Log log = get_logger() log.info("starting") # Path if args.reduxdir is not None: specprod_dir = args.reduxdir else: specprod_dir = specprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b', 'r', 'z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Exposure plot? if args.expid is not None: # Nights path_nights = glob.glob(specprod_dir + '/exposures/*') if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres( cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux, 1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series( channel_dict[channel], 'wave', outfile='QA_skyresid_wave_expid_{:d}{:s}.png'. format(args.expid, channel)) skysub_resid_series( channel_dict[channel], 'flux', outfile='QA_skyresid_flux_expid_{:d}{:s}.png'. format(args.expid, channel)) return # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile = 'QA/skyresid_prod_dual_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Full Prod Plot? if args.gauss: from desispec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes) # Plot log.info("Plotting..") skysub_gauss( sky_wave, sky_flux, sky_res, sky_ivar, outfile='skyresid_prod_gauss_{:s}.png'.format(channel)) return
def main(args, comm=None): rank = 0 nproc = 1 if comm is not None: rank = comm.rank nproc = comm.size # Determine which nights we are using nights = None if args.nights is not None: nights = args.nights.split(",") else: if rank == 0: rawdir = os.path.abspath(specio.rawdata_root()) nights = [] nightpat = re.compile(r"\d{8}") for root, dirs, files in os.walk(rawdir, topdown=True): for d in dirs: nightmat = nightpat.match(d) if nightmat is not None: nights.append(d) break if comm is not None: nights = comm.bcast(nights, root=0) # Get the list of exposures for each night night_expid = {} all_expid = [] exp_to_night = {} if rank == 0: for nt in nights: night_expid[nt] = specio.get_exposures(nt, raw=True) all_expid.extend(night_expid[nt]) for ex in night_expid[nt]: exp_to_night[ex] = nt if comm is not None: night_expid = comm.bcast(night_expid, root=0) all_expid = comm.bcast(all_expid, root=0) exp_to_night = comm.bcast(exp_to_night, root=0) expids = np.array(all_expid, dtype=np.int32) nexp = len(expids) # Get the list of cameras cams = None if args.cameras is not None: cams = args.cameras.split(",") else: cams = [] for band in ['b', 'r', 'z']: for spec in range(10): cams.append('{}{}'.format(band, spec)) # number of cameras ncamera = len(cams) # check that our communicator is an appropriate size if comm is not None: if ncamera * args.camera_procs > comm.size: if comm.rank == 0: print("Communicator size ({}) too small for {} cameras each with {} procs".format(comm.size, ncamera, args.camera_procs), flush=True) comm.Abort() # create a set of reproducible seeds for each exposure np.random.seed(args.seed) maxexp = np.max(expids) allseeds = np.random.randint(2**32, size=(maxexp+1)) seeds = allseeds[-nexp:] taskproc = ncamera * args.camera_procs comm_group = comm comm_rank = None group = comm.rank ngroup = comm.size group_rank = 0 if comm is not None: from mpi4py import MPI if taskproc > 1: ngroup = int(comm.size / taskproc) group = int(comm.rank / taskproc) group_rank = comm.rank % taskproc comm_group = comm.Split(color=group, key=group_rank) comm_rank = comm.Split(color=group_rank, key=group) else: comm_group = MPI.COMM_SELF comm_rank = comm myexpids = np.array_split(expids, ngroup)[group] for ex in myexpids: nt = exp_to_night[ex] # path to raw file simspecfile = simio.findfile('simspec', nt, ex) rawfile = specio.findfile('raw', nt, ex) rawfile = os.path.join(os.path.dirname(simspecfile), rawfile) # Is this exposure already finished? done = True if group_rank == 0: if not os.path.isfile(rawfile): done = False if args.preproc: for c in cams: pixfile = specio.findfile('pix', night=nt, expid=ex, camera=c) if not os.path.isfile(pixfile): done = False if comm_group is not None: done = comm_group.bcast(done, root=0) if done and not args.overwrite: if group_rank == 0: print("Skipping completed exposure {:08d} on night {}".format(ex, nt)) continue # Write per-process logs to a separate directory, # since there are so many of them. logdir = "{}_logs".format(rawfile) if group_rank == 0: if not os.path.isdir(logdir): os.makedirs(logdir) if comm_group is not None: comm_group.barrier() tasklog = os.path.join(logdir, "pixsim") with stdouterr_redirected(to=tasklog, comm=comm_group): try: options = {} options["night"] = nt options["expid"] = int(ex) options["cosmics"] = args.cosmics options["seed"] = seeds[ex] options["cameras"] = ",".join(cams) options["mpi_camera"] = args.camera_procs options["verbose"] = args.verbose options["preproc"] = args.preproc optarray = option_list(options) pixargs = pixsim.parse(optarray) pixsim.main(pixargs, comm_group) except: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) print("".join(lines), flush=True)
def load_all_s2n_values(nights, channel, sub_exposures=None): """ Calculate S/N values for a set of spectra from an input list of nights Args: nights: list channel: str ('b','r','z') sub_exposures: Returns: fdict: dict Contains all the S/N info for all nights in the given channel """ fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night)#, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') log.debug('Getting truth from {}'.format(simspec_file)) sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data,masked=True) #- Get OIIFLUX from separate HDU and join if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu): elg_truth = Table(sps_hdu['TRUTH_ELG'].data) sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'], keys='TARGETID', join_type='left') else: sps_tab['OIIFLUX'] = 0.0 sps_hdu.close() #objs = sps_tab['TEMPLATETYPE'] == objtype #if np.sum(objs) == 0: # continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel+str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: log.debug('Reading from {}'.format(cframe_path)) cframe = read_frame(cframe_path) except (IOError, OSError): log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave,1) dwave[0] = dwave[1] # Calculate s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave) #s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave) # Save fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers]) fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers]) fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def main(args, comm=None): if args.verbose: import logging log.setLevel(logging.DEBUG) #we do this so we can use operator.itemgetter import operator #we do this so our print statements can have timestamps import time rank = 0 nproc = 1 if comm is not None: import mpi4py rank = comm.rank nproc = comm.size if rank == 0: log.info('Starting pixsim at {}'.format(asctime())) #no preflight check here, too complicated. #we'll assume the user knows what he or she is doing... # Determine which nights we are using nights = None if args.nights is not None: nights = args.nights.split(",") else: rawdir = os.path.abspath(specio.rawdata_root()) nights = [] nightpat = re.compile(r"\d{8}") for root, dirs, files in os.walk(rawdir, topdown=True): for d in dirs: nightmat = nightpat.match(d) if nightmat is not None: nights.append(d) # Get the list of exposures for each night night_expid = {} all_expid = [] exp_to_night = {} for nt in nights: night_expid[nt] = specio.get_exposures(nt, raw=True) #get a list of tuples of (night,expid) that we can evenly divide between communicators night_exposure_list=list() if comm is None or comm.rank == 0: for nt in nights: for exp in night_expid[nt]: rawfile = desispec.io.findfile('raw', nt, exp) if not os.path.exists(rawfile): night_exposure_list.append([nt,exp]) elif args.overwrite: log.warning('Overwriting pre-existing {}'.format(os.path.basename(rawfile))) os.remove(rawfile) night_exposure_list.append([nt,exp]) else: log.info('Skipping pre-existing {}'.format(os.path.basename(rawfile))) if args.nexp is not None: night_exposure_list = night_exposure_list[0:args.nexp] if comm is not None: night_exposure_list = comm.bcast(night_exposure_list, root=0) if len(night_exposure_list) == 0: if comm is None or comm.rank == 0: log.error('No exposures to process') sys.exit(1) # Get the list of cameras and make sure it's in the right format cams = [] if args.cameras is not None: entry = args.cameras.split(',') for i in entry: cams.append(i) else: #do this with band first so we can avoid re-broadcasting cosmics for band in ['b', 'r', 'z']: for spec in range(10): cams.append('{}{}'.format(band, spec)) #are we using cosmics? if args.cosmics is not None: addcosmics = True else: addcosmics = False ncameras=len(cams) nexposures=len(night_exposure_list) #call ultity function to figure out how many nodes we have nnodes=mpi_count_nodes(comm) #call utility functions to divide our workload if args.nodes_per_exp is not None: user_specified_nodes=args.nodes_per_exp else: user_specified_nodes=None nodes_per_comm_exp=get_nodes_per_exp(nnodes,nexposures,ncameras,user_specified_nodes) #also figure out how many exposure communicators we have num_exp_comm = nnodes // nodes_per_comm_exp #split the communicator into exposure communicators comm_exp, node_index_exp, num_nodes_exp = mpi_split_by_node(comm, nodes_per_comm_exp) #further splitting will happen automatically in simulate_exposure #based on this, figure out which simspecfiles and rawfiles are assigned to each communicator #find all specfiles #find all rawfiles rawfile_list=[] simspecfile_list=[] night_list=[] expid_list=[] for i in range(len(night_exposure_list)): night_list.append(night_exposure_list[i][0]) expid_list.append(night_exposure_list[i][1]) rawfile_list.append(desispec.io.findfile('raw', night_list[i], expid_list[i])) simspecfile_list.append(io.findfile('simspec', night_list[i], expid_list[i])) #now divy the rawfiles and specfiles between node communicators #there is onerawfile and one specfile for each exposure rawfile_comm_exp=[] simspecfile_comm_exp=[] for i in range(num_exp_comm): if node_index_exp == i: #assign rawfile, simspec file to one communicator at a time rawfile_comm_exp=rawfile_list[i::num_exp_comm] simspecfile_comm_exp=simspecfile_list[i::num_exp_comm] night_comm_exp=night_list[i::num_exp_comm] expid_comm_exp=expid_list[i::num_exp_comm] comm.Barrier() #now wrap pixsim.simulate_exposure for each exposure (in desisim.pixsim) if comm_exp.rank == 0: log.info("Starting simulate_exposure for night {} expid {}".format(night_comm_exp, expid_comm_exp)) for i in range(len(rawfile_comm_exp)): simulate_exposure(simspecfile_comm_exp[i], rawfile_comm_exp[i], cameras=cams, ccdshape=None, simpixfile=None, addcosmics=addcosmics, comm=comm_exp) comm.Barrier() if rank == 0: log.info('Finished pixsim nights {}'.format(args.nights, asctime()))
def main(args) : # imports import glob from desispec.io import findfile, makepath from desispec.io import get_exposures from desispec.io import get_files, get_nights from desispec.io import get_reduced_frames from desispec.io import specprod_root from desispec.io import qaprod_root from desispec.qa import utils as qa_utils import copy import pdb # Init specprod_dir = specprod_root() # Log log=get_logger() log.info("starting") # Path if args.qaprod_dir is not None: qaprod_dir = args.qaprod_dir else: qaprod_dir = qaprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b','r','z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Exposure plot? if args.expid is not None: # Nights if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres(cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series(channel_dict[channel], 'wave', outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel)) skysub_resid_series(channel_dict[channel], 'flux', outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel)) return # Skyline if args.skyline: from desispec.qa.qa_plots import skyline_resid # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) if len(cframes) == 1: log.error('len(cframes)==1; starting debugging') pdb.set_trace() # Need to call differently else: sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes, flatten=False) # Plot outfile=args.outdir+'/skyline_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel) makepath(outfile) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Test sky noise for Gaussianity if args.gauss: from desispec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes) # Plot log.info("Plotting..") outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel) makepath(outfile) skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return