def test_quickgen_moonzenith_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures obs.new_exposure('bgs', night=night, expid=expid0, nspec=1, seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = lvmspec.io.findfile('fibermap', night, expid0) obs.new_exposure('bgs', night=night, expid=expid1, nspec=1, seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = lvmspec.io.findfile('fibermap', night, expid1) # generate quickgen output for each moon phase cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 0".format( simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 90".format( simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = lvmspec.io.findfile("cframe", night, expid0, camera) cframe1 = lvmspec.io.findfile("cframe", night, expid1, camera) cf0 = lvmspec.io.read_frame(cframe0) cf1 = lvmspec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def load_s2n_values(objtype, nights, channel, sub_exposures=None): fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night) #, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format( flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data, masked=True) sps_hdu.close() objs = sps_tab['TEMPLATETYPE'] == objtype if np.sum(objs) == 0: continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel + str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: cframe = read_frame(cframe_path) except: log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave, 1) dwave[0] = dwave[1] # iobjs = objs[cframe.fibers] if np.sum(iobjs) == 0: continue s2n = cframe.flux[iobjs, :] * np.sqrt( cframe.ivar[iobjs, :]) / np.sqrt(dwave) # Save fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]]) if objtype == 'ELG': fdict['OII'].append( sps_tab['OIIFLUX'][cframe.fibers[iobjs]]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def test_quickgen_seed_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 expid2 = 102 # generate exposures seed 1 & 2 obs.new_exposure('dark', night=night, expid=expid0, nspec=1, seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = lvmspec.io.findfile('fibermap', night, expid0) obs.new_exposure('dark', night=night, expid=expid1, nspec=1, seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = lvmspec.io.findfile('fibermap', night, expid1) obs.new_exposure('dark', night=night, expid=expid2, nspec=1, seed=2) simspec2 = io.findfile('simspec', night, expid2) fibermap2 = lvmspec.io.findfile('fibermap', night, expid2) # generate quickgen output for each exposure cmd = "quickgen --simspec {} --fibermap {} --seed 1".format( simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --seed 1".format( simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --seed 2".format( simspec2, fibermap2) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = lvmspec.io.findfile("cframe", night, expid0, camera) cframe1 = lvmspec.io.findfile("cframe", night, expid1, camera) cframe2 = lvmspec.io.findfile("cframe", night, expid2, camera) cf0 = lvmspec.io.read_frame(cframe0) cf1 = lvmspec.io.read_frame(cframe1) cf2 = lvmspec.io.read_frame(cframe2) self.assertTrue(np.all(cf0.flux == cf1.flux)) #- same seed self.assertTrue(np.all(cf0.ivar == cf1.ivar)) self.assertTrue(np.any(cf0.flux != cf2.flux)) #- different seed s0 = fits.open(simspec0) s1 = fits.open(simspec1) s2 = fits.open(simspec2) self.assertEqual(s0['TRUTH'].data['OBJTYPE'][0], s1['TRUTH'].data['OBJTYPE'][0]) self.assertEqual(s0['TRUTH'].data['REDSHIFT'][0], s1['TRUTH'].data['REDSHIFT'][0]) self.assertNotEqual(s0['TRUTH'].data['REDSHIFT'][0], s2['TRUTH'].data['REDSHIFT'][0])
def tearDown(self): for expid in [self.expid, 100, 101, 102]: fibermap = lvmspec.io.findfile('fibermap', self.night, expid) if os.path.exists(fibermap): os.remove(fibermap) simspecfile = io.findfile('simspec', self.night, expid) if os.path.exists(simspecfile): os.remove(simspecfile) for camera in ('b0', 'r0', 'z0'): framefile = lvmspec.io.findfile('frame', self.night, expid, camera=camera) if os.path.exists(framefile): os.remove(framefile) cframefile = lvmspec.io.findfile('cframe', self.night, expid, camera=camera) if os.path.exists(cframefile): os.remove(cframefile) skyfile = lvmspec.io.findfile('sky', self.night, expid, camera=camera) if os.path.exists(skyfile): os.remove(skyfile) fluxcalibfile = lvmspec.io.findfile('calib', self.night, expid, camera=camera) if os.path.exists(fluxcalibfile): os.remove(fluxcalibfile)
def dump_pa(self, paname): """ dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case. """ pafilemap = { 'Preproc': 'pix', 'BootCalibration': 'psfboot', 'BoxcarExtract': 'frame', 'ResolutionFit': None, 'ComputeFiberflat_QL': 'fiberflat', 'ApplyFiberFlat_QL': 'fframe', 'SkySub_QL': 'sframe' } if paname in pafilemap: filetype = pafilemap[paname] else: raise IOError( "PA name does not match any file type. Check PA name in config" ) pafile = None if filetype is not None: pafile = findfile(filetype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) return pafile
def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError( 'newexp-random failed for {} exposure {}'.format( program, expid)) cmd = "pixsim --preproc --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) for camera in ['b0', 'r0', 'z0']: pixfile = io.findfile('pix', night, expid, camera) outputs.append(pixfile) #outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-'))) if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format( program, expid)) return
def test_expand_args(self): night = self.night expid = self.expid obs.new_exposure('arc', night=night, expid=expid, nspec=4) simspec = io.findfile('simspec', self.night, self.expid) fibermap = lvmspec.io.findfile('fibermap', self.night, self.expid) opts = ['--simspec', simspec, '--fibermap', fibermap] args = quickgen.parse(opts) self.assertEqual(args.simspec, simspec) self.assertEqual(args.fibermap, fibermap)
def test_quickgen_airmass_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures of varying airmass obscond = simexp.reference_conditions['DARK'] obscond['AIRMASS'] = 1.5 obs.new_exposure('dark', night=night, expid=expid0, nspec=1, seed=1, obsconditions=obscond) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = lvmspec.io.findfile('fibermap', night, expid0) obscond['AIRMASS'] = 1.0 obs.new_exposure('dark', night=night, expid=expid1, nspec=1, seed=1, obsconditions=obscond) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = lvmspec.io.findfile('fibermap', night, expid1) # generate quickgen output for each airmass cmd = "quickgen --simspec {} --fibermap {}".format(simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {}".format(simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = lvmspec.io.findfile("cframe", night, expid0, camera) cframe1 = lvmspec.io.findfile("cframe", night, expid1, camera) cf0 = lvmspec.io.read_frame(cframe0) cf1 = lvmspec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def read_qa_frame(filename): """Generate a QA_Frame object from a data file """ from lvmspec.qa.qa_frame import QA_Frame #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, str): night, expid, camera = filename filename = findfile('qa', night, expid, camera) # Read qa_data = read_qa_data(filename) # Instantiate qaframe = QA_Frame(qa_data) return qaframe
def test_parse(self): night = self.night expid = self.expid obs.new_exposure('dark', night=night, expid=expid, nspec=4) simspec = io.findfile('simspec', self.night, self.expid) fibermap = lvmspec.io.findfile('fibermap', self.night, self.expid) opts = ['--simspec', simspec, '--fibermap', fibermap] opts += ['--spectrograph', '3', '--config', 'desi'] args = quickgen.parse(opts) self.assertEqual(args.simspec, simspec) self.assertEqual(args.fibermap, fibermap) self.assertEqual(args.spectrograph, 3) self.assertEqual(args.config, 'desi') with self.assertRaises(SystemExit): quickgen.parse([])
def setUpClass(cls): cls.nspec = 6 cls.nwave = 20 id = 1 cls.night = '20160101' cls.expid = 1 # Paths os.environ['LVM_SPECTRO_REDUX'] = os.environ['HOME'] os.environ['SPECPROD'] = 'desi_test_qa' # Files cls.testDir = specprod_root() cls.qafile_b0 = findfile('qa_data', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.qafile_b1 = findfile('qa_data', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.qafile_exp = cls.testDir+'/exposures/'+cls.night+'/{:08d}/qa-{:08d}'.format(id,id) cls.qafile_brick = cls.testDir+'/brick/3582m005/qa-3582m005.yaml' cls.flux_pdf = cls.testDir+'/exposures/'+cls.night+'/{:08d}/qa-flux-{:08d}.pdf'.format(id,id) cls.frame_pdf = cls.testDir+'/exposures/'+cls.night+'/{:08d}/qa-frame-{:08d}.pdf'.format(id,id) # Files for exposure fibermap QA figure cls.frame_b0 = findfile('frame', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.frame_b1 = findfile('frame', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.fflat_b0 = findfile('fiberflat', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.fflat_b1 = findfile('fiberflat', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.exp_fmap_plot = cls.testDir+'/test_exp_fibermap_plot.png'
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) # YEARMMDD string, rolls over at noon not midnight # TODO: fix usage of night to be something other than today if night is None: #night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # create production # FIXME: someday run PSF estimation too... ### com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf" com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf" sp.check_call(com, shell=True) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.yaml_read(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec pipe.yaml_write(optpath, opts) # run the generated shell scripts # FIXME: someday run PSF estimation too... # print("Running bootcalib script...") # com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh") # sp.check_call(["bash", com]) # print("Running specex script...") # com = os.path.join(proddir, "run", "scripts", "specex_all.sh") # sp.check_call(["bash", com]) # print("Running psfcombine script...") # com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh") # sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "run_shell.sh") print("Running extraction through calibration: " + com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "spectra.sh") print("Running spectral regrouping: " + com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "redshift.sh") print("Running redshift script " + com) sp.check_call(["bash", com]) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'TRUTH') from lvmmodel.footprint import radec2pix nside = 64 pixels = np.unique( radec2pix(nside, fibermap['RA_TARGET'], fibermap['DEC_TARGET'])) print() print("--------------------------------------------------") print("Pixel True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for pix in pixels: zbest = io.read_zbest(io.findfile('zbest', groupname=pix)) for i in range(len(zbest.z)): objtype = zbest.spectype[i] z, zwarn = zbest.z[i], zbest.zwarn[i] j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0] truetype = siminfo['OBJTYPE'][j] oiiflux = siminfo['OIIFLUX'][j] truez = siminfo['REDSHIFT'][j] dv = 3e5 * (z - truez) / (1 + truez) if truetype == 'SKY' and zwarn > 0: status = 'ok' elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) elif zwarn == 0: if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GALAXY': if abs(dv) < 150: status = ok elif oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) else: status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux) elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype in ('STD', 'FSTD') and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'. format(pix, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def main(args): # imports import glob from lvmspec.io import findfile from lvmspec.io import get_exposures from lvmspec.io import get_files, get_nights from lvmspec.io import read_frame from lvmspec.io import get_reduced_frames from lvmspec.io.sky import read_sky from lvmspec.io import specprod_root from lvmspec.qa import utils as qa_utils import copy import pdb # Log log = get_logger() log.info("starting") # Path if args.reduxdir is not None: specprod_dir = args.reduxdir else: specprod_dir = specprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b', 'r', 'z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Exposure plot? if args.expid is not None: # Nights if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres( cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux, 1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from lvmspec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series( channel_dict[channel], 'wave', outfile=args.outdir + '/QA_skyresid_wave_expid_{:d}{:s}.png'.format( args.expid, channel)) skysub_resid_series( channel_dict[channel], 'flux', outfile=args.outdir + '/QA_skyresid_flux_expid_{:d}{:s}.png'.format( args.expid, channel)) return # Skyline if args.skyline: from lvmspec.qa.qa_plots import skyline_resid # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) if len(cframes) == 1: log.error('len(cframes)==1; starting debugging') pdb.set_trace() # Need to call differently else: sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes, flatten=False) # Plot outfile = args.outdir + '/skyline_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return # Full Prod Plot? if args.prod: from lvmspec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile = args.outdir + '/skyresid_prod_dual_{:s}.png'.format( channel) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Full Prod Plot? if args.gauss: from lvmspec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes) # Plot log.info("Plotting..") outfile = args.outdir + '/skyresid_prod_gauss_{:s}.png'.format( channel) skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def main(args, comm=None): rank = 0 nproc = 1 if comm is not None: rank = comm.rank nproc = comm.size # Determine which nights we are using nights = None if args.nights is not None: nights = args.nights.split(",") else: if rank == 0: rawdir = os.path.abspath(specio.rawdata_root()) nights = [] nightpat = re.compile(r"\d{8}") for root, dirs, files in os.walk(rawdir, topdown=True): for d in dirs: nightmat = nightpat.match(d) if nightmat is not None: nights.append(d) break if comm is not None: nights = comm.bcast(nights, root=0) # Get the list of exposures for each night night_expid = {} all_expid = [] exp_to_night = {} if rank == 0: for nt in nights: night_expid[nt] = specio.get_exposures(nt, raw=True) all_expid.extend(night_expid[nt]) for ex in night_expid[nt]: exp_to_night[ex] = nt if comm is not None: night_expid = comm.bcast(night_expid, root=0) all_expid = comm.bcast(all_expid, root=0) exp_to_night = comm.bcast(exp_to_night, root=0) expids = np.array(all_expid, dtype=np.int32) nexp = len(expids) # Get the list of cameras cams = None if args.cameras is not None: cams = args.cameras.split(",") else: cams = [] # Do this in spectrograph order first, so that # later when we distribute cameras each group will have # the minimal set of spectrographs to store. for spec in range(10): for band in ['b', 'r', 'z']: cams.append('{}{}'.format(band, spec)) # number of cameras ncamera = len(cams) # check that our communicator is an appropriate size if comm is not None: if ncamera * args.camera_procs > comm.size: if comm.rank == 0: print( "Communicator size ({}) too small for {} cameras each with {} procs" .format(comm.size, ncamera, args.camera_procs), flush=True) comm.Abort() # create a set of reproducible seeds for each exposure np.random.seed(args.seed) maxexp = np.max(expids) seeds = np.random.randint(2**32, size=(maxexp + 1)) taskproc = ncamera * args.camera_procs comm_group = comm comm_rank = None group = 0 ngroup = 1 group_rank = 0 if comm is not None: from mpi4py import MPI if taskproc > 1: ngroup = int(comm.size / taskproc) group = int(comm.rank / taskproc) group_rank = comm.rank % taskproc comm_group = comm.Split(color=group, key=group_rank) comm_rank = comm.Split(color=group_rank, key=group) else: comm_group = MPI.COMM_SELF comm_rank = comm else: taskproc = 1 if ngroup * taskproc != nproc: msg = "Using group size {}, which does not divide evenly into total ({}). Some processes idle.".format( taskproc, nproc) if rank == 0: warnings.warn(msg, RuntimeWarning) nactive = ngroup if len(expids) < ngroup: msg = "There are fewer exposures ({}) than process groups ({}). Some groups idle.".format( len(expids), ngroup) if rank == 0: warnings.warn(msg, RuntimeWarning) nactive = len(expids) if group > nactive - 1: myexpids = np.array([], dtype=np.int32) else: myexpids = np.array_split(expids, nactive)[group] for ex in myexpids: nt = exp_to_night[ex] # path to raw file simspecfile = simio.findfile('simspec', nt, ex) rawfile = specio.findfile('raw', nt, ex) rawfile = os.path.join(os.path.dirname(simspecfile), rawfile) # Is this exposure already finished? done = True if group_rank == 0: if not os.path.isfile(rawfile): done = False if args.preproc: for c in cams: pixfile = specio.findfile('pix', night=nt, expid=ex, camera=c) if not os.path.isfile(pixfile): done = False if comm_group is not None: done = comm_group.bcast(done, root=0) if done and not args.overwrite: if group_rank == 0: print("Skipping completed exposure {:08d} on night {}".format( ex, nt)) continue # Write per-process logs to a separate directory, # since there are so many of them. logdir = "{}_logs".format(rawfile) if group_rank == 0: if not os.path.isdir(logdir): os.makedirs(logdir) if comm_group is not None: comm_group.barrier() tasklog = os.path.join(logdir, "pixsim") with stdouterr_redirected(to=tasklog, comm=comm_group): try: options = {} options["night"] = nt options["expid"] = int(ex) options["cosmics"] = args.cosmics options["seed"] = seeds[ex] options["cameras"] = ",".join(cams) options["mpi_camera"] = args.camera_procs options["verbose"] = args.verbose options["preproc"] = args.preproc optarray = option_list(options) pixargs = pixsim.parse(optarray) pixsim.main(pixargs, comm_group) except: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) print("".join(lines), flush=True)
def expand_config(self): """ config: lvmspec.quicklook.qlconfig.Config object """ self.log.debug("Building Full Configuration") self.program = self.conf["Program"] self.flavor = self.conf["Flavor"] self.debuglevel = self.conf["Debuglevel"] self.period = self.conf["Period"] self.timeout = self.conf["Timeout"] self.fiberflatexpid = self.conf["FiberflatExpid"] self.psftype = self.conf["PSFType"] self.templateexpid = self.conf["TemplateExpid"] #- some global variables: self.rawfile = findfile("raw", night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir) self.fibermap = findfile("fibermap", night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir) self.fiberflat = findfile( "fiberflat", night=self.night, expid=self.fiberflatexpid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir ) #- TODO: Assuming same night for calibration files (here and psf) self.psf = findfile(self.psftype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir) #- Get reference metrics from template yaml file if self.flavor == 'arcs': template = findfile('ql_mergedQAarc_file', night=self.night, expid=self.templateexpid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir) else: template = findfile('ql_mergedQA_file', night=self.night, expid=self.templateexpid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir) self.reference = None if os.path.isfile(template): with open(template) as reference: refdict = yaml.load(reference) nights = refdict['NIGHTS'] for night in nights: if self.night == night['NIGHT']: exposures = night['EXPOSURES'] for exposure in exposures: if self.templateexpid == exposure['EXPID']: cameras = exposure['CAMERAS'] for camera in cameras: if self.camera == camera['CAMERA']: self.reference = camera[ 'PIPELINE_STEPS'] if self.reference is None: self.log.warning("WARNING template file is malformed %s" % template) else: self.log.warning("WARNING can't open template file %s" % template) outconfig = {} outconfig['Night'] = self.night outconfig['Program'] = self.program outconfig['Flavor'] = self.flavor outconfig['Camera'] = self.camera outconfig['Expid'] = self.expid outconfig['DumpIntermediates'] = self.dumpintermediates outconfig['FiberMap'] = self.fibermap outconfig['FiberFlatFile'] = self.fiberflat outconfig['PSFFile'] = self.psf outconfig['Period'] = self.period pipeline = [] for ii, PA in enumerate(self.palist): pipe = { 'OutputFile': self.dump_qa()[1][0][PA] } #- integrated QAs for that PA. pipe['PA'] = { 'ClassName': PA, 'ModuleName': self.pamodule, 'kwargs': self.paargs[PA] } pipe['QAs'] = [] for jj, QA in enumerate(self.qalist[PA]): pipe_qa = { 'ClassName': QA, 'ModuleName': self.qamodule, 'kwargs': self.qaargs[QA] } pipe['QAs'].append(pipe_qa) pipe['StepName'] = PA pipeline.append(pipe) outconfig['PipeLine'] = pipeline outconfig['RawImage'] = self.rawfile outconfig["OutputFile"] = self.outputfile outconfig['Timeout'] = self.timeout #- Check if all the files exist for this QL configuraion check_config(outconfig) return outconfig
def io_qa_pa(self, paname): """ Specify the filenames: yaml and png of the pa level qa files" """ if self.conf["Flavor"] == 'arcs': filemap = { 'Initialize': 'ql_initial_arc', 'Preproc': 'ql_preproc_arc', 'BootCalibration': 'ql_bootcalib', 'BoxcarExtract': 'ql_boxextract_arc', 'ResolutionFit': 'ql_resfit_arc' } elif self.conf["Flavor"] == 'flat': filemap = { 'Initialize': 'ql_initial', 'Preproc': 'ql_preproc', 'BoxcarExtract': 'ql_boxextract', 'ComputeFiberflat_QL': 'ql_computeflat', } elif self.conf["Flavor"] == 'bias': filemap = { 'Initialize': 'ql_initial_bias', 'Preproc': 'ql_preproc_bias' } elif self.conf["Flavor"] == 'dark': filemap = { 'Initialize': 'ql_initial_dark', 'Preproc': 'ql_preproc_dark' } else: filemap = { 'Initialize': 'ql_initial', 'Preproc': 'ql_preproc', 'BootCalibration': 'ql_bootcalib', 'BoxcarExtract': 'ql_boxextract', 'ResolutionFit': 'ql_resfit', 'ApplyFiberFlat_QL': 'ql_fiberflat', 'SkySub_QL': 'ql_skysub' } if paname in filemap: filetype = filemap[paname] + '_file' figtype = filemap[paname] + '_fig' else: raise IOError( "PA name does not match any file type. Check PA name in config for {}" .format(paname)) outfile = findfile(filetype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) outfig = findfile(figtype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) return (outfile, outfig)
def paargs(self, psfspfile=None): """ Many arguments for the PAs are taken default. Some of these may need to be variable psfspfile is for offline extraction case """ wavelength = self.wavelength if self.wavelength is None: #- setting default wavelength for extraction for different cam if self.camera[0] == 'b': self.wavelength = '3570,5730,0.8' elif self.camera[0] == 'r': self.wavelength = '5630,7740,0.8' elif self.camera[0] == 'z': self.wavelength = '7420,9830,0.8' #- Make kwargs less verbose using '%%' marker for global variables. Pipeline will map them back paopt_initialize = {'camera': self.camera} if self.writepixfile: pixfile = self.dump_pa("Preproc") else: pixfile = None paopt_preproc = {'camera': self.camera, 'dumpfile': pixfile} if self.dumpintermediates: if self.conf["Flavor"] == 'arcs': calibdir = os.path.join(os.environ['QL_SPEC_REDUX'], 'calib2d', self.night) framefile = findfile('frame', night=self.night, expid=self.expid, camera=self.camera, outdir=calibdir) else: framefile = self.dump_pa("BoxcarExtract") fframefile = self.dump_pa("ApplyFiberFlat_QL") sframefile = self.dump_pa("SkySub_QL") else: framefile = None fframefile = None sframefile = None if self.conf["Flavor"] == 'arcs': arcimg = findfile('pix', night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir) flatimg = findfile('pix', night=self.night, expid=self.conf["FiberflatExpid"], camera=self.camera, rawdata_dir=self.rawdata_dir) bootfile = findfile('psfboot', night=self.night, camera=self.camera, specprod_dir=self.specprod_dir) psfnightfile = findfile('psfnight', night=self.night, camera=self.camera, specprod_dir=self.specprod_dir) else: arcimg = None flatimg = None bootfile = None psfnightfile = None paopt_bootcalib = { 'ArcLampImage': arcimg, 'FlatImage': flatimg, 'outputFile': bootfile } paopt_extract = { 'BoxWidth': 2.5, 'FiberMap': self.fibermap, 'Wavelength': self.wavelength, 'Nspec': 500, 'PSFFile': self.psf, 'usesigma': self.usesigma, 'dumpfile': framefile } paopt_resfit = { 'PSFbootfile': bootfile, 'PSFoutfile': psfnightfile, 'usesigma': self.usesigma } if self.conf["Flavor"] == 'flat': fiberflatfile = findfile('fiberflat', night=self.night, expid=self.conf["FiberflatExpid"], camera=self.camera, specprod_dir=self.specprod_dir) else: fiberflatfile = None paopt_comflat = {'outputFile': fiberflatfile} paopt_apfflat = { 'FiberFlatFile': self.fiberflat, 'dumpfile': fframefile } if self.writeskymodelfile: outskyfile = findfile('sky', night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) else: outskyfile = None paopt_skysub = { 'Outskyfile': outskyfile, 'dumpfile': sframefile, 'Apply_resolution': self.usesigma } paopts = {} defList = { 'Initialize': paopt_initialize, 'Preproc': paopt_preproc, 'BootCalibration': paopt_bootcalib, 'BoxcarExtract': paopt_extract, 'ResolutionFit': paopt_resfit, 'ComputeFiberflat_QL': paopt_comflat, 'ApplyFiberFlat_QL': paopt_apfflat, 'SkySub_QL': paopt_skysub } def getPAConfigFromFile(PA, algs): def mergeDicts(source, dest): for k in source: if k not in dest: dest[k] = source[k] userconfig = {} if PA in algs: fc = algs[PA] for k in fc: #do a deep copy leave QA config out if k != "QA": userconfig[k] = fc[k] defconfig = {} if PA in defList: defconfig = defList[PA] mergeDicts(defconfig, userconfig) return userconfig for PA in self.palist: paopts[PA] = getPAConfigFromFile(PA, self.algorithms) #- Ignore intermediate dumping and write explicitly the outputfile for self.outputfile = self.dump_pa(self.palist[-1]) return paopts
def runpipeline(pl, convdict, conf, mergeQA=False): """ Runs the quicklook pipeline as configured Args: pl: is a list of [pa,qas] where pa is a pipeline step and qas the corresponding qas for that pa convdict: converted dictionary e.g : conf["IMAGE"] is the real psf file but convdict["IMAGE"] is like lvmspec.image.Image object and so on. details in setup_pipeline method below for examples. conf: a configured dictionary, read from the configuration yaml file. e.g: conf=configdict=yaml.load(open('configfile.yaml','rb')) mergedQA: if True, outputs the merged QA after the execution of pipeline. Perhaps, this should always be True, but leaving as option, until configuration and IO settles. """ qlog = qllogger.QLLogger() log = qlog.getlog() hb = QLHB.QLHeartbeat(log, conf["Period"], conf["Timeout"]) inp = convdict["rawimage"] paconf = conf["PipeLine"] qlog = qllogger.QLLogger() log = qlog.getlog() passqadict = None #- pass this dict to QAs downstream schemaMerger = QL_QAMerger(conf['Night'], conf['Expid'], conf['Flavor'], conf['Camera']) QAresults = [ ] #- merged QA list for the whole pipeline. This will be reorganized for databasing after the pipeline executes for s, step in enumerate(pl): log.info("Starting to run step {}".format(paconf[s]["StepName"])) pa = step[0] pargs = mapkeywords(step[0].config["kwargs"], convdict) schemaStep = schemaMerger.addPipelineStep(paconf[s]["StepName"]) try: hb.start("Running {}".format(step[0].name)) oldinp = inp #- copy for QAs that need to see earlier input inp = pa(inp, **pargs) except Exception as e: log.critical("Failed to run PA {} error was {}".format( step[0].name, e), exc_info=True) sys.exit("Failed to run PA {}".format(step[0].name)) qaresult = {} for qa in step[1]: try: qargs = mapkeywords(qa.config["kwargs"], convdict) hb.start("Running {}".format(qa.name)) qargs[ "dict_countbins"] = passqadict #- pass this to all QA downstream if qa.name == "RESIDUAL" or qa.name == "Sky_Residual": res = qa(inp[0], inp[1], **qargs) else: if isinstance(inp, tuple): res = qa(inp[0], **qargs) else: res = qa(inp, **qargs) if qa.name == "COUNTBINS" or qa.name == "CountSpectralBins": #TODO -must run this QA for now. change this later. passqadict = res if "qafile" in qargs: qawriter.write_qa_ql(qargs["qafile"], res) log.debug("{} {}".format(qa.name, inp)) qaresult[qa.name] = res schemaStep.addParams(res['PARAMS']) schemaStep.addMetrics(res['METRICS']) except Exception as e: log.warning("Failed to run QA {}. Got Exception {}".format( qa.name, e), exc_info=True) if len(qaresult): if conf["DumpIntermediates"]: f = open(paconf[s]["OutputFile"], "w") f.write(yaml.dump(yamlify(qaresult))) hb.stop("Step {} finished. Output is in {} ".format( paconf[s]["StepName"], paconf[s]["OutputFile"])) else: hb.stop("Step {} finished.".format(paconf[s]["StepName"])) QAresults.append([pa.name, qaresult]) hb.stop("Pipeline processing finished. Serializing result") #- merge QAs for this pipeline execution if mergeQA is True: # from lvmspec.quicklook.util import merge_QAs # log.info("Merging all the QAs for this pipeline execution") # merge_QAs(QAresults,conf) log.debug("Dumping mergedQAs") from lvmspec.io import findfile ftype = 'ql_mergedQA_file' specprod_dir = os.environ[ 'QL_SPEC_REDUX'] if 'QL_SPEC_REDUX' in os.environ else "" if conf['Flavor'] == 'arcs': ftype = 'ql_mergedQAarc_file' destFile = findfile(ftype, night=conf['Night'], expid=conf['Expid'], camera=conf['Camera'], specprod_dir=specprod_dir) # this will overwrite the file. above function returns same name for different QL executions # results will be erased. schemaMerger.writeToFile(destFile) log.info("Wrote merged QA file {}".format(destFile)) if isinstance(inp, tuple): return inp[0] else: return inp
def io_qa(self, qaname): """ Specify the filenames: yaml and png for the given qa output """ if self.conf["Flavor"] == 'arcs': filemap = { 'Bias_From_Overscan': 'ql_getbias_arc', 'Get_RMS': 'ql_getrms_arc', 'Count_Pixels': 'ql_countpix_arc', 'Calc_XWSigma': 'ql_xwsigma_arc', 'CountSpectralBins': 'ql_countbins_arc' } elif self.conf["Flavor"] == 'bias': filemap = { 'Bias_From_Overscan': 'ql_getbias_bias', 'Get_RMS': 'ql_getrms_bias', 'Count_Pixels': 'ql_countpix_bias' } elif self.conf["Flavor"] == 'dark': filemap = { 'Bias_From_Overscan': 'ql_getbias_dark', 'Get_RMS': 'ql_getrms_dark', 'Count_Pixels': 'ql_countpix_dark' } else: filemap = { 'Bias_From_Overscan': 'ql_getbias', 'Get_RMS': 'ql_getrms', 'Count_Pixels': 'ql_countpix', 'Calc_XWSigma': 'ql_xwsigma', 'CountSpectralBins': 'ql_countbins', 'Sky_Continuum': 'ql_skycont', 'Sky_Peaks': 'ql_skypeak', 'Sky_Residual': 'ql_skyresid', 'Integrate_Spec': 'ql_integ', 'Calculate_SNR': 'ql_snr' } if qaname in filemap: filetype = filemap[qaname] + '_file' figtype = filemap[qaname] + '_fig' else: raise IOError( "QA name does not match any file type. Check QA name in config for {}" .format(qaname)) outfile = findfile(filetype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) outfig = findfile(figtype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) return (outfile, outfig)
def test_quickgen_simspec(self): night = self.night expid = self.expid camera = 'r0' # flavors = ['flat','dark','gray','bright','bgs','mws','elg','lrg','qso','arc'] flavors = ['arc', 'flat', 'dark', 'bright'] for i in range(len(flavors)): flavor = flavors[i] obs.new_exposure(flavor, night=night, expid=expid, nspec=2) #- output to same directory as input os.environ['LVM_SPECTRO_REDUX'] = os.path.join( os.getenv('LVM_SPECTRO_SIM'), os.getenv('PIXPROD')) #- run quickgen simspec = io.findfile('simspec', night, expid) fibermap = io.findfile('simfibermap', night, expid) self.assertTrue(os.path.exists(simspec)) self.assertTrue(os.path.exists(fibermap)) if flavor == 'flat': try: cmd = "quickgen --simspec {} --fibermap {}".format( simspec, fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) except SystemExit: pass #- verify flat outputs fiberflatfile = lvmspec.io.findfile('fiberflat', night, expid, camera) self.assertTrue(os.path.exists(fiberflatfile)) elif flavor == 'arc': try: cmd = "quickgen --simspec {} --fibermap {}".format( simspec, fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) except SystemExit: pass #- verify arc outputs framefile = lvmspec.io.findfile('frame', night, expid, camera) self.assertTrue(os.path.exists(framefile)) else: cmd = "quickgen --simspec {} --fibermap {}".format( simspec, fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) #- verify outputs framefile = lvmspec.io.findfile('frame', night, expid, camera) self.assertTrue(os.path.exists(framefile)) cframefile = lvmspec.io.findfile('cframe', night, expid, camera) self.assertTrue(os.path.exists(cframefile)) skyfile = lvmspec.io.findfile('sky', night, expid, camera) self.assertTrue(os.path.exists(skyfile)) fluxcalibfile = lvmspec.io.findfile('calib', night, expid, camera) self.assertTrue(os.path.exists(fluxcalibfile)) os.remove(simspec) os.remove(fibermap)