def test_quickgen_moonzenith_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures obs.new_exposure('bgs',night=night,expid=expid0,nspec=1,seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obs.new_exposure('bgs',night=night,expid=expid1,nspec=1,seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) # generate quickgen output for each moon phase cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 0".format(simspec0,fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 90".format(simspec1,fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0=desispec.io.findfile("cframe",night,expid0,camera) cframe1=desispec.io.findfile("cframe",night,expid1,camera) cf0=desispec.io.read_frame(cframe0) cf1=desispec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar),np.median(cf1.ivar))
def _write_qaframe(self, camera='b0', expid=1, night='20160101', ZPval=24., flavor='science'): """Write QA data frame files""" frm = self._make_frame(camera=camera, expid=expid, night=night, flavor=flavor) qafrm = QA_Frame(frm) # SKY qafrm.init_skysub() qafrm.qa_data['SKYSUB']['METRICS'] = {} qafrm.qa_data['SKYSUB']['METRICS']['NSKY_FIB'] = 10 # FLUX qafrm.init_fluxcalib() qafrm.qa_data['FLUXCALIB']['METRICS'] = {} qafrm.qa_data['FLUXCALIB']['METRICS']['ZP'] = ZPval qafrm.qa_data['FLUXCALIB']['METRICS']['RMS_ZP'] = 0.05 # Outfile qafile = findfile('qa_data', night=night, expid=expid, specprod_dir=self.testDir, camera=camera) # WRITE write_qa_frame(qafile, qafrm) self.files_written.append(qafile) # Generate frame too (for QA_Exposure) frame = self._make_frame(camera=camera, flavor=flavor, night=night, expid=expid) frame_file = findfile('frame', night=night, expid=expid, specprod_dir=self.testDir, camera=camera) _ = write_frame(frame_file, frame) self.files_written.append(frame_file) # return qafile
def io_qa_pa(self,paname): """ Specify the filenames: json and png of the pa level qa files" """ filemap={'Initialize': 'initial', 'Preproc': 'preproc', 'Flexure': 'flexure', 'BoxcarExtract': 'boxextract', 'Extract_QP': 'extractqp', 'ComputeFiberflat_QL': 'computeflat', 'ComputeFiberflat_QP': 'computeflatqp', 'ApplyFiberFlat_QL': 'fiberflat', 'ApplyFiberFlat_QP': 'fiberflatqp', 'SkySub_QL': 'skysub', 'SkySub_QP': 'skysubqp', 'ResolutionFit': 'resfit', 'ApplyFluxCalibration': 'fluxcalib' } if paname in filemap: outfile=findfile('ql_file',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfile=outfile.replace('qlfile',filemap[paname]) outfig=findfile('ql_fig',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfig=outfig.replace('qlfig',filemap[paname]) else: raise IOError("PA name does not match any file type. Check PA name in config for {}".format(paname)) return (outfile,outfig)
def test_quickgen_airmass_simspec(self): night=self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures of varying airmass obscond = simexp.reference_conditions['DARK'] obscond['AIRMASS'] = 1.5 obs.new_exposure('dark',night=night,expid=expid0,nspec=1,seed=1,obsconditions=obscond) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obscond['AIRMASS'] = 1.0 obs.new_exposure('dark',night=night,expid=expid1,nspec=1,seed=1,obsconditions=obscond) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) # generate quickgen output for each airmass cmd = "quickgen --simspec {} --fibermap {}".format(simspec0,fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {}".format(simspec1,fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0=desispec.io.findfile("cframe",night,expid0,camera) cframe1=desispec.io.findfile("cframe",night,expid1,camera) cf0=desispec.io.read_frame(cframe0) cf1=desispec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar),np.median(cf1.ivar))
def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('newexp-random failed for {} exposure {}'.format(program, expid)) cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) outputs.append(io.findfile('raw', night, expid)) if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(program, expid)) return
def io_qa(self,qaname): """ Specify the filenames: json and png for the given qa output """ filemap={'Check_HDUs':'checkHDUs', 'Trace_Shifts':'trace', 'Bias_From_Overscan': 'getbias', 'Get_RMS' : 'getrms', 'Count_Pixels': 'countpix', 'Calc_XWSigma': 'xwsigma', 'CountSpectralBins': 'countbins', 'Sky_Continuum': 'skycont', 'Sky_Rband': 'skyRband', 'Sky_Peaks': 'skypeak', 'Sky_Residual': 'skyresid', 'Integrate_Spec': 'integ', 'Calculate_SNR': 'snr', 'Check_Resolution': 'checkres', 'Check_FiberFlat': 'checkfibflat' } if qaname in filemap: outfile=findfile('ql_file',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfile=outfile.replace('qlfile',filemap[qaname]) outfig=findfile('ql_fig',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfig=outfig.replace('qlfig',filemap[qaname]) else: raise IOError("QA name does not match any file type. Check QA name in config for {}".format(qaname)) return (outfile,outfig)
def _write_flat_file(self, camera='b0', night=None, expid=None): # Init if night is None: night = self.nights[0] if expid is None: expid = self.expids[0] # Filename frame_file = findfile('frame', night=night, expid=expid, specprod_dir=self.testDir, camera=camera) fflat_file = findfile('fiberflat', night=night, expid=expid, specprod_dir=self.testDir, camera=camera) # Frames fb = self._make_frame(camera=camera, flavor='flat', nspec=10) _ = write_frame(frame_file, fb) self.files_written.append(frame_file) # Fiberflats ff = get_fiberflat_from_frame(fb) write_fiberflat(fflat_file, ff) self.files_written.append(fflat_file) # Return return frame_file, fflat_file
def test_quickgen_moonzenith_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures obs.new_exposure('bgs', night=night, expid=expid0, nspec=1, seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obs.new_exposure('bgs', night=night, expid=expid1, nspec=1, seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) # generate quickgen output for each moon phase cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 0".format( simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --moon-zenith 90".format( simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = desispec.io.findfile("cframe", night, expid0, camera) cframe1 = desispec.io.findfile("cframe", night, expid1, camera) cf0 = desispec.io.read_frame(cframe0) cf1 = desispec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def test_quickgen_seed_simspec(self): night=self.night camera = 'r0' expid0 = 100 expid1 = 101 expid2 = 102 # generate exposures seed 1 & 2 obs.new_exposure('dark',night=night,expid=expid0,nspec=1,seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obs.new_exposure('dark',night=night,expid=expid1,nspec=1,seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) obs.new_exposure('dark',night=night,expid=expid2,nspec=1,seed=2) simspec2 = io.findfile('simspec', night, expid2) fibermap2 = desispec.io.findfile('fibermap', night, expid2) s0=fits.open(simspec0) s1=fits.open(simspec1) s2=fits.open(simspec2) self.assertEqual(s0['TRUTH'].data['OBJTYPE'][0], s1['TRUTH'].data['OBJTYPE'][0]) self.assertEqual(s0['TRUTH'].data['REDSHIFT'][0], s1['TRUTH'].data['REDSHIFT'][0]) self.assertNotEqual(s0['TRUTH'].data['REDSHIFT'][0], s2['TRUTH'].data['REDSHIFT'][0]) self.assertTrue(np.all(s0['WAVE'].data == s1['WAVE'].data)) self.assertTrue(np.all(s0['FLUX'].data == s1['FLUX'].data)) # generate quickgen output for each exposure # spawn with subprocess so that it will really be restarting fresh; # using quickgen.main re-uses a specsim Simulator that consumes random # state when using it. # See https://github.com/desihub/specsim/issues/94 cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 1".format( sys.executable, self.binDir, simspec0, fibermap0) ### quickgen.main(quickgen.parse(cmd.split()[1:])) subprocess.check_call(cmd.split()) cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 1".format( sys.executable, self.binDir, simspec1, fibermap1) ### quickgen.main(quickgen.parse(cmd.split()[1:])) subprocess.check_call(cmd.split()) cmd = "{} {}/quickgen --simspec {} --fibermap {} --seed 2".format( sys.executable, self.binDir, simspec2, fibermap2) ### quickgen.main(quickgen.parse(cmd.split()[1:])) subprocess.check_call(cmd.split()) cframe0=desispec.io.findfile("cframe",night,expid0,camera) cframe1=desispec.io.findfile("cframe",night,expid1,camera) cframe2=desispec.io.findfile("cframe",night,expid2,camera) cf0=desispec.io.read_frame(cframe0) cf1=desispec.io.read_frame(cframe1) cf2=desispec.io.read_frame(cframe2) self.assertTrue(np.all(cf0.flux == cf1.flux)) #- same seed self.assertTrue(np.all(cf0.ivar == cf1.ivar)) self.assertTrue(np.any(cf0.flux != cf2.flux)) #- different seed
def load_s2n_values(objtype, nights, channel, sub_exposures=None): fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night) #, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format( flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data, masked=True) sps_hdu.close() objs = sps_tab['TEMPLATETYPE'] == objtype if np.sum(objs) == 0: continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel + str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: cframe = read_frame(cframe_path) except: log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave, 1) dwave[0] = dwave[1] # iobjs = objs[cframe.fibers] if np.sum(iobjs) == 0: continue s2n = cframe.flux[iobjs, :] * np.sqrt( cframe.ivar[iobjs, :]) / np.sqrt(dwave) # Save fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]]) if objtype == 'ELG': fdict['OII'].append( sps_tab['OIIFLUX'][cframe.fibers[iobjs]]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def test_quickgen_simspec(self): night = self.night expid = self.expid camera = 'r0' # flavors = ['flat','dark','gray','bright','bgs','mws','elg','lrg','qso','arc'] flavors = ['flat', 'dark', 'bright'] for i in range(len(flavors)): flavor = flavors[i] obs.new_exposure(flavor, night=night, expid=expid, nspec=2) #- output to same directory as input os.environ['DESI_SPECTRO_REDUX'] = os.path.join( os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD')) #- run quickgen simspec = io.findfile('simspec', night, expid) fibermap = io.findfile('simfibermap', night, expid) self.assertTrue(os.path.exists(simspec)) self.assertTrue(os.path.exists(fibermap)) if flavor == 'flat': try: cmd = "quickgen --simspec {} --fibermap {}".format( simspec, fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) except SystemExit: pass #- verify flat outputs fiberflatfile = desispec.io.findfile('fiberflat', night, expid, camera) self.assertTrue(os.path.exists(fiberflatfile)) else: cmd = "quickgen --simspec {} --fibermap {}".format( simspec, fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) #- verify outputs framefile = desispec.io.findfile('frame', night, expid, camera) self.assertTrue(os.path.exists(framefile)) cframefile = desispec.io.findfile('cframe', night, expid, camera) self.assertTrue(os.path.exists(cframefile)) skyfile = desispec.io.findfile('sky', night, expid, camera) self.assertTrue(os.path.exists(skyfile)) fluxcalibfile = desispec.io.findfile('calib', night, expid, camera) self.assertTrue(os.path.exists(fluxcalibfile)) os.remove(simspec) os.remove(fibermap)
def test_quickgen_seed_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 expid2 = 102 # generate exposures seed 1 & 2 obs.new_exposure('dark', night=night, expid=expid0, nspec=1, seed=1) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obs.new_exposure('dark', night=night, expid=expid1, nspec=1, seed=1) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) obs.new_exposure('dark', night=night, expid=expid2, nspec=1, seed=2) simspec2 = io.findfile('simspec', night, expid2) fibermap2 = desispec.io.findfile('fibermap', night, expid2) # generate quickgen output for each exposure cmd = "quickgen --simspec {} --fibermap {} --seed 1".format( simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --seed 1".format( simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {} --seed 2".format( simspec2, fibermap2) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = desispec.io.findfile("cframe", night, expid0, camera) cframe1 = desispec.io.findfile("cframe", night, expid1, camera) cframe2 = desispec.io.findfile("cframe", night, expid2, camera) cf0 = desispec.io.read_frame(cframe0) cf1 = desispec.io.read_frame(cframe1) cf2 = desispec.io.read_frame(cframe2) self.assertTrue(np.all(cf0.flux == cf1.flux)) #- same seed self.assertTrue(np.all(cf0.ivar == cf1.ivar)) self.assertTrue(np.any(cf0.flux != cf2.flux)) #- different seed s0 = fits.open(simspec0) s1 = fits.open(simspec1) s2 = fits.open(simspec2) self.assertEqual(s0['TRUTH'].data['OBJTYPE'][0], s1['TRUTH'].data['OBJTYPE'][0]) self.assertEqual(s0['TRUTH'].data['REDSHIFT'][0], s1['TRUTH'].data['REDSHIFT'][0]) self.assertNotEqual(s0['TRUTH'].data['REDSHIFT'][0], s2['TRUTH'].data['REDSHIFT'][0])
def find_most_recent(night, file_type='psfnight', cam='r', n_nights=30): ''' Searches back in time for either psfnight or fiberflatnight (or anything supported by desispec.calibfinder.findcalibfile. This only works on nightly-based files, so exposure id information is not used. Inputs: night : str. YYYYMMDD night to look back from file_type : str. psfnight or fiberflatnight cam : str. camera (b, r, or z). n_nights : int. number of nights to step back before giving up returns: nightfile : str. Full pathname to calibration file of interest. If none found, None is returned. ''' # Set the time as Noon on the day in question today = time.strptime('{} 12'.format(night), '%Y%m%d %H') one_day = 60 * 60 * 24 # seconds test_night_struct = today # Search a month in the past for daysback in range(n_nights): test_night_struct = time.strptime( time.ctime(time.mktime(test_night_struct) - one_day)) test_night_str = time.strftime('%Y%m%d', test_night_struct) nightfile = findfile(file_type, test_night_str, camera=cam) if os.path.isfile(nightfile): return nightfile return None
def _write_qaframe(self, camera='b0', expid=1, night='20160101', ZPval=24.): """Write QA data frame files""" frm = self._make_frame(camera=camera) qafrm = QA_Frame(frm) # SKY qafrm.init_skysub() qafrm.qa_data['SKYSUB']['METRICS'] = {} qafrm.qa_data['SKYSUB']['METRICS']['NSKY_FIB'] = 10 # FLUX qafrm.init_fluxcalib() qafrm.qa_data['FLUXCALIB']['METRICS'] = {} qafrm.qa_data['FLUXCALIB']['METRICS']['ZP'] = ZPval qafrm.qa_data['FLUXCALIB']['METRICS']['RMS_ZP'] = 0.05 # Outfile qafile = findfile('qa_data', night=night, expid=expid, specprod_dir=self.testDir, camera=camera) # WRITE write_qa_frame(qafile, qafrm) self.files_written.append(qafile) return qafile
def qaargs(self): qaopts = {} referencemetrics=[] for PA in self.palist: for qa in self.qalist[PA]: #- individual QA for that PA pa_yaml = PA.upper() params=self._qaparams(qa) qaopts[qa]={'night' : self.night, 'expid' : self.expid, 'camera': self.camera, 'paname': PA, 'PSFFile': self.psf_filename, 'amps': self.amps, #'qafile': self.dump_qa()[0][qa], 'qafig': self.dump_qa()[qa], 'FiberMap': self.fibermap, 'param': params, 'refKey':self._qaRefKeys[qa], 'singleqa' : self.singqa, 'plotconf':self.plotconf, 'hardplots': self.hardplots } if qa == 'Calc_XWSigma': qaopts[qa]['Peaks']=self.algorithms['Initialize']['PEAKS'] qaopts[qa]['Flavor']=self.flavor qaopts[qa]['PSFFile']=self.calibpsf if qa == 'Sky_Peaks': qaopts[qa]['Peaks']=self.algorithms['Initialize']['PEAKS'] if self.singqa is not None: qaopts[qa]['rawdir']=self.rawdata_dir qaopts[qa]['specdir']=self.specprod_dir if qa == 'Sky_Residual': skyfile = findfile('sky',night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) qaopts[qa]['SkyFile']=skyfile if self.reference != None: refkey=qaopts[qa]['refKey'] for padict in range(len(self.reference)): pa_metrics=self.reference[padict].keys() if refkey in pa_metrics: qaopts[qa]['ReferenceMetrics']={'{}'.format(refkey): self.reference[padict][refkey]} return qaopts
def read_fiberflat(filename): """Read fiberflat from filename Args: filename (str): Name of fiberflat file, or (night, expid, camera) tuple Returns: FiberFlat object with attributes fiberflat, ivar, mask, meanspec, wave, header Notes: fiberflat, ivar, mask are 2D [nspec, nwave] meanspec and wave are 1D [nwave] """ #- check if outfile is (night, expid, camera) tuple instead if isinstance(filename, (tuple, list)) and len(filename) == 3: night, expid, camera = filename filename = findfile('fiberflat', night, expid, camera) header = fits.getheader(filename, 0) fiberflat = native_endian(fits.getdata(filename, 0)).astype('f8') ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8')) mask = native_endian(fits.getdata(filename, "MASK", uint=True)) meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8')) wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8')) return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def tearDown(self): for expid in [self.expid, 100, 101, 102]: fibermap = desispec.io.findfile('fibermap', self.night, expid) if os.path.exists(fibermap): os.remove(fibermap) simspecfile = io.findfile('simspec', self.night, expid) if os.path.exists(simspecfile): os.remove(simspecfile) for camera in ('b0', 'r0', 'z0'): framefile = desispec.io.findfile('frame', self.night, expid, camera=camera) if os.path.exists(framefile): os.remove(framefile) cframefile = desispec.io.findfile('cframe', self.night, expid, camera=camera) if os.path.exists(cframefile): os.remove(cframefile) skyfile = desispec.io.findfile('sky', self.night, expid, camera=camera) if os.path.exists(skyfile): os.remove(skyfile) fluxcalibfile = desispec.io.findfile('calib', self.night, expid, camera=camera) if os.path.exists(fluxcalibfile): os.remove(fluxcalibfile)
def read_fiberflat(filename): """Read fiberflat from filename Args: filename (str): Name of fiberflat file, or (night, expid, camera) tuple Returns: FiberFlat object with attributes fiberflat, ivar, mask, meanspec, wave, header Notes: fiberflat, ivar, mask are 2D [nspec, nwave] meanspec and wave are 1D [nwave] """ #- check if outfile is (night, expid, camera) tuple instead if isinstance(filename, (tuple, list)) and len(filename) == 3: night, expid, camera = filename filename = findfile('fiberflat', night, expid, camera) header = fits.getheader(filename, 0) fiberflat = native_endian(fits.getdata(filename, 0)) ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8')) mask = native_endian(fits.getdata(filename, "MASK", uint=True)) meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8')) wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8')) return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def qafile_from_framefile(frame_file, qaprod_dir=None, output_dir=None): """ Derive the QA filename from an input frame file Args: frame_file: str output_dir: str, optional Over-ride default output path qa_dir: str, optional Over-ride default QA Returns: """ frame_meta = read_meta_frame(frame_file) night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = int(frame_meta['EXPID']) if frame_meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' # Name qafile = findfile(qatype, night=night, camera=camera, expid=expid, outdir=output_dir, qaprod_dir=qaprod_dir) # Return return qafile, qatype
def dump_pa(self, paname): """ dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case. """ pafilemap = { 'Preproc': 'pix', 'BoxcarExtract': 'frame', 'ApplyFiberFlat_QL': 'fframe', 'SkySub_QL': 'sframe' } if paname in pafilemap: filetype = pafilemap[paname] else: raise IOError( "PA name does not match any file type. Check PA name in config" ) pafile = findfile(filetype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) return pafile
def test_quickgen_simspec(self): night = self.night expid = self.expid camera = 'r0' # flavors = ['flat','dark','gray','bright','bgs','mws','elg','lrg','qso','arc'] flavors = ['flat', 'dark', 'bright'] for i in range(len(flavors)): flavor = flavors[i] obs.new_exposure(flavor, night=night, expid=expid, nspec=2) #- output to same directory as input os.environ['DESI_SPECTRO_REDUX'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD')) #- run quickgen simspec = io.findfile('simspec', night, expid) fibermap = io.findfile('simfibermap', night, expid) self.assertTrue(os.path.exists(simspec)) self.assertTrue(os.path.exists(fibermap)) if flavor == 'flat': try: cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) except SystemExit: pass #- verify flat outputs fiberflatfile = desispec.io.findfile('fiberflat', night, expid, camera) self.assertTrue(os.path.exists(fiberflatfile)) else: cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap) quickgen.main(quickgen.parse(cmd.split()[1:])) #- verify outputs framefile = desispec.io.findfile('frame', night, expid, camera) self.assertTrue(os.path.exists(framefile)) cframefile = desispec.io.findfile('cframe', night, expid, camera) self.assertTrue(os.path.exists(cframefile)) skyfile = desispec.io.findfile('sky', night, expid, camera) self.assertTrue(os.path.exists(skyfile)) fluxcalibfile = desispec.io.findfile('calib', night, expid, camera) self.assertTrue(os.path.exists(fluxcalibfile)) os.remove(simspec) os.remove(fibermap)
def io_qa_pa(self,paname): """ Specify the filenames: yaml and png of the pa level qa files" """ if self.conf["Flavor"] == 'arcs': filemap={'Initialize': 'ql_initial_arc', 'Preproc': 'ql_preproc_arc', 'BootCalibration': 'ql_bootcalib', 'BoxcarExtract': 'ql_boxextract_arc', 'ResolutionFit': 'ql_resfit_arc' } elif self.conf["Flavor"] == 'flat': filemap={'Initialize': 'ql_initial', 'Preproc': 'ql_preproc', 'BoxcarExtract': 'ql_boxextract', 'ComputeFiberflat_QL': 'ql_computeflat', } elif self.conf["Flavor"] == 'bias': filemap={'Initialize': 'ql_initial_bias', 'Preproc': 'ql_preproc_bias' } elif self.conf["Flavor"] == 'dark': filemap={'Initialize': 'ql_initial_dark', 'Preproc': 'ql_preproc_dark' } else: filemap={'Initialize': 'ql_initial', 'Preproc': 'ql_preproc', 'BootCalibration': 'ql_bootcalib', 'BoxcarExtract': 'ql_boxextract', 'ResolutionFit': 'ql_resfit', 'ApplyFiberFlat_QL': 'ql_fiberflat', 'SkySub_QL': 'ql_skysub' } if paname in filemap: filetype=filemap[paname]+'_file' figtype=filemap[paname]+'_fig' else: raise IOError("PA name does not match any file type. Check PA name in config for {}".format(paname)) outfile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfig=findfile(figtype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) return (outfile,outfig)
def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, flavor in zip([0, 1, 2], ['flat', 'arc', 'dark']): cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, flavor=flavor, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError( 'pixsim newexp failed for {} exposure {}'.format( flavor, expid)) cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) for camera in ['b0', 'r0', 'z0']: pixfile = io.findfile('pix', night, expid, camera) outputs.append(pixfile) #outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-'))) if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format( flavor, expid)) return
def io_qa(self,qaname): """ Specify the filenames: yaml and png for the given qa output """ if self.conf["Flavor"] == 'arcs': filemap={'Bias_From_Overscan': 'ql_getbias_arc', 'Get_RMS' : 'ql_getrms_arc', 'Count_Pixels': 'ql_countpix_arc', 'Calc_XWSigma': 'ql_xwsigma_arc', 'CountSpectralBins': 'ql_countbins_arc' } elif self.conf["Flavor"] == 'bias': filemap={'Bias_From_Overscan': 'ql_getbias_bias', 'Get_RMS' : 'ql_getrms_bias', 'Count_Pixels': 'ql_countpix_bias' } elif self.conf["Flavor"] == 'dark': filemap={'Bias_From_Overscan': 'ql_getbias_dark', 'Get_RMS' : 'ql_getrms_dark', 'Count_Pixels': 'ql_countpix_dark' } else: filemap={'Bias_From_Overscan': 'ql_getbias', 'Get_RMS' : 'ql_getrms', 'Count_Pixels': 'ql_countpix', 'Calc_XWSigma': 'ql_xwsigma', 'CountSpectralBins': 'ql_countbins', 'Sky_Continuum': 'ql_skycont', 'Sky_Peaks': 'ql_skypeak', 'Sky_Residual': 'ql_skyresid', 'Integrate_Spec': 'ql_integ', 'Calculate_SNR': 'ql_snr' } if qaname in filemap: filetype=filemap[qaname]+'_file' figtype=filemap[qaname]+'_fig' else: raise IOError("QA name does not match any file type. Check QA name in config for {}".format(qaname)) outfile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) outfig=findfile(figtype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) return (outfile,outfig)
def dump_pa(self,filetype): """ dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case. """ if filetype in ["fframe","sframe"]: #- fiberflat fielded or sky subtracted intermediate files pafile=os.path.join(self.specprod_dir,'exposures',self.night,"%08d"%self.expid,"%s-%s-%08d.fits"%(filetype,self.camera,self.expid)) else: pafile=findfile(filetype,night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) return pafile
def read_frame(filename, nspec=None): """Reads a frame fits file and returns its data. Args: filename: path to a file, or (night, expid, camera) tuple where night = string YEARMMDD expid = integer exposure ID camera = b0, r1, .. z9 Returns: desispec.Frame object with attributes wave, flux, ivar, etc. """ #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, (str, unicode)): night, expid, camera = filename filename = findfile('frame', night, expid, camera) if not os.path.isfile(filename) : raise IOError("cannot open"+filename) fx = fits.open(filename, uint=True, memmap=False) hdr = fx[0].header flux = native_endian(fx['FLUX'].data.astype('f8')) ivar = native_endian(fx['IVAR'].data.astype('f8')) wave = native_endian(fx['WAVELENGTH'].data.astype('f8')) if 'MASK' in fx: mask = native_endian(fx['MASK'].data) else: mask = None #- let the Frame object create the default mask resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8')) if 'FIBERMAP' in fx: fibermap = fx['FIBERMAP'].data else: fibermap = None if 'CHI2PIX' in fx: chi2pix = native_endian(fx['CHI2PIX'].data.astype('f8')) else: chi2pix = None fx.close() if nspec is not None: flux = flux[0:nspec] ivar = ivar[0:nspec] resolution_data = resolution_data[0:nspec] if chi2pix is not None: chi2pix = chi2pix[0:nspec] if mask is not None: mask = mask[0:nspec] # return flux,ivar,wave,resolution_data, hdr return Frame(wave, flux, ivar, mask, resolution_data, meta=hdr, fibermap=fibermap, chi2pix=chi2pix)
def test_expand_args(self): night = self.night expid = self.expid obs.new_exposure('arc', night=night, expid=expid, nspec=4) simspec = io.findfile('simspec', self.night, self.expid) fibermap = desispec.io.findfile('fibermap', self.night, self.expid) opts = ['--simspec', simspec, '--fibermap', fibermap] args = quickgen.parse(opts) self.assertEqual(args.simspec, simspec) self.assertEqual(args.fibermap, fibermap)
def test_qa_frame_write_load_data(self): # Write frm0 = self._make_frame() qafrm0 = QA_Frame(frm0) # Write outfile = findfile('qa_data', night=self.nights[0], expid=self.expids[0], specprod_dir=self.testDir, camera='b0') write_qa_frame(outfile, qafrm0) self.files_written.append(outfile) # Load qafrm2 = load_qa_frame(outfile, frame_meta=frm0.meta) assert qafrm2.night == qafrm0.night
def test_qa_frame_write_load_data(self): # Write frm0 = self._make_frame() qafrm0 = QA_Frame(frm0) # Write outfile = findfile('qa_data', night=self.nights[0], expid=self.expids[0], specprod_dir=self.testDir, camera='b0') write_qa_frame(outfile, qafrm0) self.files_written.append(outfile) # Load qafrm2 = load_qa_frame(outfile, frm0) assert qafrm2.night == qafrm0.night
def __init__(self, night,flavor,expid,camera,palist,debuglevel=20,period=5.,psfboot=None,wavelength=None, dumpintermediates=True,amps=True,rawdata_dir=None,specprod_dir=None, outdir=None,timeout=120., fiberflat=None,outputfile=None,qlf=False): """ psfboot- does not seem to have a desispec.io.findfile entry, so passing this in argument. May be this will be useful even so. palist: Palist object. See class Palist below Note: fiberflat will have a different expid. Passing the file directly in the path """ self.night=night self.expid=expid self.flavor=flavor self.camera=camera self.psfboot=psfboot self.fiberflat=fiberflat self.outputfile=outputfile #- final outputfile. self.wavelength=wavelength self.debuglevel=debuglevel self.period=period self.dumpintermediates=dumpintermediates self.amps=amps if rawdata_dir is None: rawdata_dir=os.getenv('DESI_SPECTRO_DATA') self.rawdata_dir=rawdata_dir if specprod_dir is None: specprod_dir=os.path.join(os.getenv('DESI_SPECTRO_REDUX'), os.getenv('SPECPROD')) self.specprod_dir=specprod_dir self.outdir=outdir self.timeout=timeout self._palist=palist self.pamodule=palist.pamodule self.qamodule=palist.qamodule self._qlf=qlf #- some global variables self.rawfile=findfile("raw",night=self.night,expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir) self.fibermap=findfile("fibermap", night=self.night,expid=self.expid,camera=self.camera, rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir)
def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, program in zip([0, 1, 2], ['flat', 'arc', 'dark']): cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, program=program, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError( 'newexp-random failed for {} exposure {}'.format( program, expid)) cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) outputs.append(io.findfile('raw', night, expid)) if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format( program, expid)) return
def read_frame(filename, nspec=None): """Reads a frame fits file and returns its data. Args: filename: path to a file, or (night, expid, camera) tuple where night = string YEARMMDD expid = integer exposure ID camera = b0, r1, .. z9 Returns: desispec.Frame object with attributes wave, flux, ivar, etc. """ #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, (str, unicode)): night, expid, camera = filename filename = findfile('frame', night, expid, camera) if not os.path.isfile(filename): raise IOError("cannot open" + filename) fx = fits.open(filename, uint=True, memmap=False) hdr = fx[0].header flux = native_endian(fx['FLUX'].data.astype('f8')) ivar = native_endian(fx['IVAR'].data.astype('f8')) wave = native_endian(fx['WAVELENGTH'].data.astype('f8')) if 'MASK' in fx: mask = native_endian(fx['MASK'].data) else: mask = None #- let the Frame object create the default mask resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8')) if 'FIBERMAP' in fx: fibermap = fx['FIBERMAP'].data else: fibermap = None fx.close() if nspec is not None: flux = flux[0:nspec] ivar = ivar[0:nspec] resolution_data = resolution_data[0:nspec] # return flux,ivar,wave,resolution_data, hdr return Frame(wave, flux, ivar, mask, resolution_data, meta=hdr, fibermap=fibermap)
def sim(night, nspec=5, clobber=False): """ Simulate data as part of the integration test. Args: night (str): YEARMMDD nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() # Create input fibermaps, spectra, and pixel-level raw data for expid, flavor in zip([0,1,2], ['flat', 'arc', 'dark']): cmd = "newexp-desi --flavor {flavor} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, flavor=flavor, nspec=nspec, night=night) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid)) cmd = "pixsim-desi --preproc --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night) inputs = [fibermap, simspec] outputs = list() outputs.append(fibermap.replace('fibermap-', 'simpix-')) for camera in ['b0', 'r0', 'z0']: pixfile = io.findfile('pix', night, expid, camera) outputs.append(pixfile) #outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-'))) if pipe.runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid)) return
def test_quickgen_airmass_simspec(self): night = self.night camera = 'r0' expid0 = 100 expid1 = 101 # generate exposures of varying airmass obscond = simexp.reference_conditions['DARK'] obscond['AIRMASS'] = 1.5 obs.new_exposure('dark', night=night, expid=expid0, nspec=1, seed=1, obsconditions=obscond) simspec0 = io.findfile('simspec', night, expid0) fibermap0 = desispec.io.findfile('fibermap', night, expid0) obscond['AIRMASS'] = 1.0 obs.new_exposure('dark', night=night, expid=expid1, nspec=1, seed=1, obsconditions=obscond) simspec1 = io.findfile('simspec', night, expid1) fibermap1 = desispec.io.findfile('fibermap', night, expid1) # generate quickgen output for each airmass cmd = "quickgen --simspec {} --fibermap {}".format(simspec0, fibermap0) quickgen.main(quickgen.parse(cmd.split()[1:])) cmd = "quickgen --simspec {} --fibermap {}".format(simspec1, fibermap1) quickgen.main(quickgen.parse(cmd.split()[1:])) cframe0 = desispec.io.findfile("cframe", night, expid0, camera) cframe1 = desispec.io.findfile("cframe", night, expid1, camera) cf0 = desispec.io.read_frame(cframe0) cf1 = desispec.io.read_frame(cframe1) self.assertLess(np.median(cf0.ivar), np.median(cf1.ivar))
def batch_run(infile, outdir, cameras, queue, batchtime, batchopts): """Submits batch job to `nightwatch run infile outdir ...` Args: infile (str): input DESI raw data file outdir (str): base output directory cameras (list or None): None, or list of cameras to include queue (str): slurm queue name batchtime (int): batch job time limit [minutes] batchopts (str): additional batch options Returns error code from sbatch submission Note: this is a non-blocking call and will return before the batch processing is finished """ night, expid = io.get_night_expid(infile) expdir = io.findfile('expdir', night=night, expid=expid, basedir=outdir) infile = os.path.abspath(infile) expdir = os.path.abspath(expdir) outdir = os.path.abspath(outdir) if cameras is None: camera_options = "" elif isinstance(cameras, (list, tuple)): camera_options = "--cameras {}".format(','.join(cameras)) elif isinstance(cameras, str): camera_options = f"--cameras {cameras}" else: raise ValueError('Unable to parse cameras {}'.format(cameras)) jobname = f'nightwatch-{expid:08d}' batchfile = f'{expdir}/{jobname}.slurm' with open(batchfile, 'w') as fx: fx.write(f"""#!/bin/bash -l #SBATCH {batchopts} #SBATCH --qos {queue} #SBATCH --time {batchtime} #SBATCH --job-name {jobname} #SBATCH --output {expdir}/{jobname}-%j.joblog #SBATCH --exclusive nightwatch run --infile {infile} --outdir {outdir} {camera_options} """) err = subprocess.call(["sbatch", batchfile]) return err
def io_qa_pa(self, paname): """ Specify the filenames: yaml and png of the pa level qa files" """ filemap = { 'Initialize': 'ql_initial', 'Preproc': 'ql_preproc', 'BoxcarExtract': 'ql_boxextract', 'ApplyFiberFlat_QL': 'ql_fiberflat', 'SkySub_QL': 'ql_skysub' } if paname in filemap: filetype = filemap[paname] + '_file' figtype = filemap[paname] + '_fig' else: raise IOError( "PA name does not match any file type. Check PA name in config for {}" .format(paname)) outfile = findfile(filetype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) outfig = findfile(figtype, night=self.night, expid=self.expid, camera=self.camera, rawdata_dir=self.rawdata_dir, specprod_dir=self.specprod_dir, outdir=self.outdir) return (outfile, outfig)
def read_qa_frame(filename): """Generate a QA_Frame object from a data file """ #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, basestring): night, expid, camera = filename filename = findfile('qa', night, expid, camera) # Read qa_data = read_qa_data(filename) # Instantiate qaframe = QA_Frame(qa_data) return qaframe
def dump_pa(self,paname): """ dump the PA outputs to respective files. This has to be updated for fframe and sframe files as QL anticipates for dumpintermediate case. """ pafilemap={'Preproc': 'preproc', 'Flexure': None, 'BoxcarExtract': 'frame','ResolutionFit': None, 'Extract_QP': 'qframe', 'ComputeFiberflat_QL': 'fiberflat', 'ComputeFiberflat_QP': 'fiberflat', 'ApplyFiberFlat_QL': 'fframe', 'ApplyFiberFlat_QP': 'fframe', 'SkySub_QL': 'sframe', 'SkySub_QP': 'sframe', 'ApplyFluxCalibration': 'cframe'} if paname in pafilemap: filetype=pafilemap[paname] else: raise IOError("PA name does not match any file type. Check PA name in config") pafile=None if filetype is not None: pafile=findfile(filetype,night=self.night,expid=self.expid,camera=self.camera,rawdata_dir=self.rawdata_dir,specprod_dir=self.specprod_dir,outdir=self.outdir) return pafile
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c + str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError( 'must use only one camera with --bias, --pixflat, --mask options' ) if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def test_parse(self): night = self.night expid = self.expid obs.new_exposure('dark', night=night, expid=expid, nspec=4) simspec = io.findfile('simspec', self.night, self.expid) fibermap = desispec.io.findfile('fibermap', self.night, self.expid) opts = ['--simspec', simspec, '--fibermap', fibermap] opts += ['--spectrograph', '3', '--config', 'desi'] args = quickgen.parse(opts) self.assertEqual(args.simspec, simspec) self.assertEqual(args.fibermap, fibermap) self.assertEqual(args.spectrograph, 3) self.assertEqual(args.config, 'desi') with self.assertRaises(SystemExit): quickgen.parse([])
def read_sky(filename) : """Read sky model and return SkyModel object with attributes wave, flux, ivar, mask, header. skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave] """ #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, (str, unicode)): night, expid, camera = filename filename = findfile('sky', night, expid, camera) hdr = fits.getheader(filename, 0) wave = native_endian(fits.getdata(filename, "WAVELENGTH")) skyflux = native_endian(fits.getdata(filename, "SKY")) ivar = native_endian(fits.getdata(filename, "IVAR")) mask = native_endian(fits.getdata(filename, "MASK", uint=True)) skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr) return skymodel
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --pixflat, --mask options') if (args.pixfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --pixfile option') if args.outdir is None: args.outdir = os.getcwd() for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=args.bias, pixflat=args.pixflat, mask=args.mask) except IOError: log.error('Camera {} not in {}'.format(camera, args.infile)) continue if args.pixfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] pixfile = io.findfile('pix', night=night, expid=expid, camera=camera, outdir=args.outdir) else: pixfile = args.pixfile io.write_image(pixfile, img)
def read_sky(filename) : """Read sky model and return SkyModel object with attributes wave, flux, ivar, mask, header. skymodel.wave is 1D common wavelength grid, the others are 2D[nspec, nwave] """ #- check if filename is (night, expid, camera) tuple instead if not isinstance(filename, (str, unicode)): night, expid, camera = filename filename = findfile('sky', night, expid, camera) fx = fits.open(filename, memmap=False, uint=True) hdr = fx[0].header wave = native_endian(fx["WAVELENGTH"].data.astype('f8')) skyflux = native_endian(fx["SKY"].data.astype('f8')) ivar = native_endian(fx["IVAR"].data.astype('f8')) mask = native_endian(fx["MASK"].data) fx.close() skymodel = SkyModel(wave, skyflux, ivar, mask, header=hdr) return skymodel
def qafile_from_framefile(frame_file, qaprod_dir=None, output_dir=None): """ Derive the QA filename from an input frame file Args: frame_file: str output_dir: str, optional Over-ride default output path qa_dir: str, optional Over-ride default QA Returns: """ frame_meta = read_meta_frame(frame_file) night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] if frame_meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' # Name qafile = findfile(qatype, night=night, camera=camera, expid=expid, outdir=output_dir, qaprod_dir=qaprod_dir) # Return return qafile, qatype
def runpipeline(pl,convdict,conf): """ Runs the quicklook pipeline as configured Args: pl: is a list of [pa,qas] where pa is a pipeline step and qas the corresponding qas for that pa convdict: converted dictionary e.g : conf["IMAGE"] is the real psf file but convdict["IMAGE"] is like desispec.image.Image object and so on. details in setup_pipeline method below for examples. conf: a configured dictionary, read from the configuration yaml file. e.g: conf=configdict=yaml.load(open('configfile.yaml','rb')) """ qlog=qllogger.QLLogger() log=qlog.getlog() hb=QLHB.QLHeartbeat(log,conf["Period"],conf["Timeout"]) inp=convdict["rawimage"] singqa=conf["singleqa"] paconf=conf["PipeLine"] qlog=qllogger.QLLogger() log=qlog.getlog() passqadict=None #- pass this dict to QAs downstream schemaMerger=QL_QAMerger(conf['Night'],conf['Expid'],conf['Flavor'],conf['Camera'],conf['Program'],convdict) QAresults=[] if singqa is None: for s,step in enumerate(pl): log.info("Starting to run step {}".format(paconf[s]["StepName"])) pa=step[0] pargs=mapkeywords(step[0].config["kwargs"],convdict) schemaStep=schemaMerger.addPipelineStep(paconf[s]["StepName"]) try: hb.start("Running {}".format(step[0].name)) oldinp=inp #- copy for QAs that need to see earlier input inp=pa(inp,**pargs) if step[0].name == 'Initialize': schemaStep.addMetrics(inp[1]) except Exception as e: log.critical("Failed to run PA {} error was {}".format(step[0].name,e),exc_info=True) sys.exit("Failed to run PA {}".format(step[0].name)) qaresult={} for qa in step[1]: try: qargs=mapkeywords(qa.config["kwargs"],convdict) hb.start("Running {}".format(qa.name)) qargs["dict_countbins"]=passqadict #- pass this to all QA downstream if qa.name=="RESIDUAL" or qa.name=="Sky_Residual": res=qa(inp[0],inp[1],**qargs) else: if isinstance(inp,tuple): res=qa(inp[0],**qargs) else: res=qa(inp,**qargs) if qa.name=="COUNTBINS" or qa.name=="CountSpectralBins": passqadict=res if "qafile" in qargs: qawriter.write_qa_ql(qargs["qafile"],res) log.debug("{} {}".format(qa.name,inp)) qaresult[qa.name]=res schemaStep.addParams(res['PARAMS']) schemaStep.addMetrics(res['METRICS']) except Exception as e: log.warning("Failed to run QA {}. Got Exception {}".format(qa.name,e),exc_info=True) hb.stop("Step {} finished.".format(paconf[s]["StepName"])) QAresults.append([pa.name,qaresult]) hb.stop("Pipeline processing finished. Serializing result") else: import numpy as np qa=None qas=[[],['Bias_From_Overscan','Get_RMS','Count_Pixels','Calc_XWSigma'],'Trace_Shifts','CountSpectralBins',['Sky_Continuum','Sky_Peaks'],['Calculate_SNR'],['Sky_Rband','Integrate_Spec']] singleqaperpa=['Bias_From_Overscan','Check_HDUs','Trace_Shifts','CountSpectralBins'] for palg in range(len(qas)): if singqa in qas[palg]: pa=pl[palg][0] pac=paconf[palg] if singqa in singleqaperpa: qa = pl[palg][1][0] else: for qalg in range(len(qas[palg])): if qas[palg][qalg] == singqa: qa=pl[palg][1][qalg] if qa is None: log.critical("Unknown input QA... Valid QAs are: {}".format(qas)) sys.exit() log.info("Starting to run step {}".format(pac["StepName"])) pargs=mapkeywords(pa.config["kwargs"],convdict) schemaStep=schemaMerger.addPipelineStep(pac["StepName"]) qaresult={} try: qargs=mapkeywords(qa.config["kwargs"],convdict) hb.start("Running {}".format(qa.name)) if singqa=="Sky_Residual": res=qa(inp[0],inp[1],**qargs) else: if isinstance(inp,tuple): res=qa(inp[0],**qargs) else: res=qa(inp,**qargs) if singqa=="CountSpectralBins": passqadict=res if "qafile" in qargs: qawriter.write_qa_ql(qargs["qafile"],res) log.debug("{} {}".format(qa.name,inp)) schemaStep.addMetrics(res['METRICS']) except Exception as e: log.warning("Failed to run QA {}. Got Exception {}".format(qa.name,e),exc_info=True) if len(qaresult): if conf["DumpIntermediates"]: f = open(pac["OutputFile"],"w") f.write(yaml.dump(yamlify(qaresult))) log.info("{} finished".format(qa.name)) #- merge QAs for this pipeline execution #- RS: don't write merged file if running single QA if singqa is None: log.debug("Dumping mergedQAs") from desispec.io import findfile specprod_dir=os.environ['QL_SPEC_REDUX'] if 'QL_SPEC_REDUX' in os.environ else "" destFile=findfile('ql_mergedQA_file',night=conf['Night'], expid=conf['Expid'], camera=conf['Camera'], specprod_dir=specprod_dir) schemaMerger.writeTojsonFile(destFile) log.info("Wrote merged QA file {}".format(destFile)) if isinstance(inp,tuple): return inp[0] else: return inp
parser = optparse.OptionParser(usage = "%prog [options]") parser.add_option("-b", "--brick", type=str, help="input brickname") parser.add_option("-n", "--nspec", type=int, help="number of spectra to fit [default: all]") parser.add_option("-o", "--outfile", type=str, help="output file name") parser.add_option("--zspec", help="also include spectra in output file", action="store_true") opts, args = parser.parse_args() log = get_logger() #- Read brick files for each channel log.info("Reading bricks") brick = dict() for channel in ('b', 'r', 'z'): filename = io.findfile('brick', band=channel, brickid=opts.brick) brick[channel] = io.Brick(filename) #- Assume all channels have the same number of targets #- TODO: generalize this to allow missing channels if opts.nspec is None: opts.nspec = brick['b'].get_num_targets() log.info("Fitting {} targets".format(opts.nspec)) else: log.info("Fitting {} of {} targets".format(opts.nspec, brick['b'].get_num_targets())) nspec = opts.nspec #- Coadd individual exposures and combine channels #- Full coadd code is a bit slow, so try something quick and dirty for #- now to get something going for redshifting
def main() : """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID') parser.add_argument('--fibermap', type = str, help = 'path of fibermap file') parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits') parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9') parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux') args = parser.parse_args() log = get_logger() # Call necessary environment variables. No need if add argument to give full file path. if 'DESI_SPECTRO_REDUX' not in os.environ: raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles') DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX'] PRODNAME=os.environ['PRODNAME'] if 'DESISIM' not in os.environ: raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration') DESISIM=os.environ['DESISIM'] # to read the filter transmission files if args.fibermap is None or args.models is None or \ args.spectrograph is None or args.outfile is None or \ args.fiberflatexpid is None: log.critical('Missing a required argument') parser.print_help() sys.exit(12) # read Standard Stars from the fibermap file # returns the Fiber id, filter names and mags for the standard stars fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True) #- Trim to just fibers on this spectrograph ii = (500*args.spectrograph <= fiber_tbdata["FIBER"]) ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1)) fiber_tbdata = fiber_tbdata[ii] #- Get info for the standard stars refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD") refFibers=fiber_tbdata["FIBER"][refStarIdx] refFilters=fiber_tbdata["FILTER"][refStarIdx] refMags=fiber_tbdata["MAG"] fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags} NIGHT=fiber_header['NIGHT'] EXPID=fiber_header['EXPID'] filters=fibers["FILTER"] if 'DESISIM' not in os.environ: raise RuntimeError('Set environment DESISIM. Can not find filter response files') basepath=DESISIM+"/data/" #now load all the skyfiles, framefiles, fiberflatfiles etc # all three channels files are simultaneously treated for model fitting skyfile={} framefile={} fiberflatfile={} for i in ["b","r","z"]: camera = i+str(args.spectrograph) skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera) framefile[i] = io.findfile('frame', NIGHT, EXPID, camera) fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera) #Read Frames, Flats and Sky files frameFlux={} frameIvar={} frameWave={} frameResolution={} framehdr={} fiberFlat={} ivarFlat={} maskFlat={} meanspecFlat={} waveFlat={} headerFlat={} sky={} skyivar={} skymask={} skywave={} skyhdr={} for i in ["b","r","z"]: #arg=(night,expid,'%s%s'%(i,spectrograph)) #- minimal code change for refactored I/O, while not taking advantage of simplified structure frame = io.read_frame(framefile[i]) frameFlux[i] = frame.flux frameIvar[i] = frame.ivar frameWave[i] = frame.wave frameResolution[i] = frame.resolution_data framehdr[i] = frame.header ff = io.read_fiberflat(fiberflatfile[i]) fiberFlat[i] = ff.fiberflat ivarFlat[i] = ff.ivar maskFlat[i] = ff.mask meanspecFlat[i] = ff.meanspec waveFlat[i] = ff.wave headerFlat[i] = ff.header skymodel = io.read_sky(skyfile[i]) sky[i] = skymodel.flux skyivar[i] = skymodel.ivar skymask[i] = skymodel.mask skywave[i] = skymodel.wave skyhdr[i] = skymodel.header # Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]} # Read the standard Star data and divide by flat and subtract sky stars=[] ivars=[] for i in fibers["FIBER"]: #flat and sky should have same wavelength binning as data, otherwise should be rebinned. stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]], "r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]], "z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i])) ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]})) stdwave,stdflux,templateid=io.read_stdstar_templates(args.models) #- Trim standard star wavelengths to just the range we need minwave = min([min(w) for w in frameWave.values()]) maxwave = max([max(w) for w in frameWave.values()]) ii = (minwave-10 < stdwave) & (stdwave < maxwave+10) stdwave = stdwave[ii] stdflux = stdflux[:, ii] log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars))) if len(stars) == 0: log.critical("No standard stars! Exiting") sys.exit(1) # Now for each star, find the best model and normalize. normflux=[] bestModelIndex=np.arange(len(stars)) templateID=np.arange(len(stars)) chi2dof=np.zeros(len(stars)) #- TODO: don't use 'l' as a variable name. Can look like a '1' for k,l in enumerate(stars): log.info("checking best model for star {0}".format(l[0])) starindex=l[0] mags=l[2] filters=fibers["FILTER"][k] rflux=stars[k][1]["r"][0] bflux=stars[k][1]["b"][0] zflux=stars[k][1]["z"][0] flux={"b":bflux,"r":rflux,"z":zflux} #print ivars rivar=ivars[k][1]["r"][0] bivar=ivars[k][1]["b"][0] zivar=ivars[k][1]["z"][0] ivar={"b":bivar,"r":rivar,"z":zivar} resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]} # Now find the best Model bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux) log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k])) # Normalize the best model using reported magnitude modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath) normflux.append(normalizedflux) # Now write the normalized flux for all best models to a file normflux=np.array(normflux) stdfibers=fibers["FIBER"] data={} data['BESTMODEL']=bestModelIndex data['CHI2DOF']=chi2dof data['TEMPLATEID']=templateid[bestModelIndex] norm_model_file=args.outfile io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = get_logger() #- YEARMMDD string, rolls over at noon not midnight if night is None: night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) #- check for required environment variables check_env() #- parameter dictionary that will later be used for formatting commands params = dict(night=night, nspec=nspec) #----- #- Input fibermaps, spectra, and pixel-level raw data for expid, flavor in zip([0,1,2], ['flat', 'arc', 'science']): cmd = "pixsim-desi --newexp {flavor} --nspec {nspec} --night {night} --expid {expid}".format( expid=expid, flavor=flavor, **params) fibermap = io.findfile('fibermap', night, expid) simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid) inputs = [] outputs = [fibermap, simspec] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('pixsim newexp failed for {} exposure {}'.format(flavor, expid)) cmd = "pixsim-desi --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, **params) inputs = [fibermap, simspec] outputs = list() for camera in ['b0', 'r0', 'z0']: pixfile = io.findfile('pix', night, expid, camera) outputs.append(pixfile) outputs.append(os.path.join(os.path.dirname(pixfile), os.path.basename(pixfile).replace('pix-', 'simpix-'))) if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('pixsim failed for {} exposure {}'.format(flavor, expid)) #----- #- Extract waverange = dict( b = "3570,5940,1.0", r = "5630,7740,1.0", z = "7440,9830,1.0", ) for expid in [0,1,2]: for channel in ['b', 'r', 'z']: camera = channel+'0' pixfile = io.findfile('pix', night, expid, camera) psffile = '{}/data/specpsf/psf-{}.fits'.format(os.getenv('DESIMODEL'), channel) framefile = io.findfile('frame', night, expid, camera) cmd = "exspec -i {pix} -p {psf} --specrange 0,{nspec} -w {wave} -o {frame}".format( pix=pixfile, psf=psffile, wave=waverange[channel], frame=framefile, **params) inputs = [pixfile, psffile] outputs = [framefile,] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('extraction failed for {} expid {}'.format(camera, expid)) #----- #- Fiber flat expid = 0 for channel in ['b', 'r', 'z']: camera = channel+"0" framefile = io.findfile('frame', night, expid, camera) fiberflat = io.findfile('fiberflat', night, expid, camera) cmd = "desi_compute_fiberflat.py --infile {frame} --outfile {fiberflat}".format( frame=framefile, fiberflat=fiberflat, **params) inputs = [framefile,] outputs = [fiberflat,] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('fiberflat failed for '+camera) #----- #- Sky model flat_expid = 0 expid = 2 for channel in ['b', 'r', 'z']: camera = channel+"0" framefile = io.findfile('frame', night, expid, camera) fibermap = io.findfile('fibermap', night, expid) fiberflat = io.findfile('fiberflat', night, flat_expid, camera) skyfile = io.findfile('sky', night, expid, camera) cmd="desi_compute_sky.py --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --outfile {sky}".format( frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, **params) inputs = [framefile, fibermap, fiberflat] outputs = [skyfile, ] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('sky model failed for '+camera) #----- #- Fit standard stars if 'STD_TEMPLATES' in os.environ: std_templates = os.getenv('STD_TEMPLATES') else: std_templates = os.getenv('DESI_ROOT')+'/spectro/templates/stellar_templates/v1.0/stdstar_templates_v1.0.fits' stdstarfile = io.findfile('stdstars', night, expid, spectrograph=0) cmd = """desi_fit_stdstars.py --spectrograph 0 \ --fibermap {fibermap} \ --fiberflatexpid {flat_expid} \ --models {std_templates} --outfile {stdstars}""".format( flat_expid=flat_expid, fibermap=fibermap, std_templates=std_templates, stdstars=stdstarfile) inputs = [fibermap, std_templates] outputs = [stdstarfile,] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('fitting stdstars failed') #----- #- Flux calibration for channel in ['b', 'r', 'z']: camera = channel+"0" framefile = io.findfile('frame', night, expid, camera) fibermap = io.findfile('fibermap', night, expid) fiberflat = io.findfile('fiberflat', night, flat_expid, camera) skyfile = io.findfile('sky', night, expid, camera) calibfile = io.findfile('calib', night, expid, camera) #- Compute flux calibration vector cmd = """desi_compute_fluxcalibration.py \ --infile {frame} --fibermap {fibermap} --fiberflat {fiberflat} --sky {sky} \ --models {stdstars} --outfile {calib}""".format( frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, stdstars=stdstarfile, calib=calibfile, ) inputs = [framefile, fibermap, fiberflat, skyfile, stdstarfile] outputs = [calibfile,] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('flux calibration failed for '+camera) #- Apply the flux calibration to write a cframe file cframefile = io.findfile('cframe', night, expid, camera) cmd = """desi_process_exposure.py \ --infile {frame} --fiberflat {fiberflat} --sky {sky} --calib {calib} \ --outfile {cframe}""".format(frame=framefile, fibermap=fibermap, fiberflat=fiberflat, sky=skyfile, calib=calibfile, cframe=cframefile) inputs = [framefile, fiberflat, skyfile, calibfile] outputs = [cframefile, ] if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('combining calibration steps failed for '+camera) #----- #- Bricks inputs = list() for camera in ['b0', 'r0', 'z0']: inputs.append( io.findfile('cframe', night, expid, camera) ) outputs = list() fibermap = io.read_fibermap(io.findfile('fibermap', night, expid)) bricks = set(fibermap['BRICKNAME']) for b in bricks: for channel in ['b', 'r', 'z']: outputs.append( io.findfile('brick', brickid=b, band=channel)) cmd = "desi_make_bricks.py --night "+night if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('brick generation failed') #----- #- Redshifts! for b in bricks: inputs = [io.findfile('brick', brickid=b, band=channel) for channel in ['b', 'r', 'z']] zbestfile = io.findfile('zbest', brickid=b) outputs = [zbestfile, ] cmd = "desi_zfind.py --brick {} -o {}".format(b, zbestfile) if runcmd(cmd, inputs, outputs, clobber) != 0: raise RuntimeError('redshifts failed for brick '+b) #----- #- Did it work? #- (this combination of fibermap, simspec, and zbest is a pain) simdir = os.path.dirname(io.findfile('fibermap', night=night, expid=expid)) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'METADATA') print() print("--------------------------------------------------") print("Brick True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for b in bricks: zbest = io.read_zbest(io.findfile('zbest', brickid=b)) for i in range(len(zbest.z)): if zbest.type[i] == 'ssp_em_galaxy': objtype = 'GAL' elif zbest.type[i] == 'spEigenStar': objtype = 'STAR' else: objtype = zbest.type[i] z, zwarn = zbest.z[i], zbest.zwarn[i] j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0] truetype = siminfo['OBJTYPE'][j] truez = siminfo['REDSHIFT'][j] dv = 3e5*(z-truez)/(1+truez) if truetype == 'SKY' and zwarn > 0: status = 'ok' elif zwarn == 0: if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype == 'STD' and objtype == 'STAR': status = 'ok' else: status = 'oops' else: status = 'oops' print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format( b, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ import argparse parser = argparse.ArgumentParser(usage = "{prog} [options]") # parser.add_argument("-i", "--input", type=str, help="input data") # parser.add_argument("-o", "--output", type=str, help="output data") parser.add_argument("--skip-psf", action="store_true", help="Skip PSF fitting step") args = parser.parse_args() log = logging.get_logger() # YEARMMDD string, rolls over at noon not midnight if night is None: night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # create production if clobber and os.path.isdir(proddir): shutil.rmtree(proddir) dbfile = io.get_pipe_database() if not os.path.exists(dbfile): com = "desi_pipe create --db-sqlite" log.info('Running {}'.format(com)) sp.check_call(com, shell=True) else: log.info("Using pre-existing production database {}".format(dbfile)) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.prod.yaml_read(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec opts['psf']['specmin'] = 0 opts['psf']['nspec'] = nspec opts['traceshift']['nfibers'] = nspec pipe.prod.yaml_write(optpath, opts) if args.skip_psf: #- Copy desimodel psf into this production instead of fitting psf import shutil for channel in ['b', 'r', 'z']: refpsf = '{}/data/specpsf/psf-{}.fits'.format( os.getenv('DESIMODEL'), channel) nightpsf = io.findfile('psfnight', night, camera=channel+'0') shutil.copy(refpsf, nightpsf) for expid in [0,1,2]: exppsf = io.findfile('psf', night, expid, camera=channel+'0') shutil.copy(refpsf, exppsf) #- Resync database to current state dbpath = io.get_pipe_database() db = pipe.load_db(dbpath, mode="w") db.sync(night) # Run the pipeline tasks in order from desispec.pipeline.tasks.base import default_task_chain for tasktype in default_task_chain: #- if we skip psf/psfnight/traceshift, update state prior to extractions if tasktype == 'traceshift' and args.skip_psf: db.getready() run_pipeline_step(tasktype) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'TRUTH') try: elginfo = fits.getdata(simspec, 'TRUTH_ELG') except: elginfo = None from desimodel.footprint import radec2pix nside=64 pixels = np.unique(radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC'])) num_missing = 0 for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) num_missing += 1 if num_missing > 0: log.critical('{} zbest files missing'.format(num_missing)) sys.exit(1) print() print("--------------------------------------------------") print("Pixel True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) continue zfx = fits.open(zfile, memmap=False) zbest = zfx['ZBEST'].data for i in range(len(zbest['Z'])): objtype = zbest['SPECTYPE'][i] z, zwarn = zbest['Z'][i], zbest['ZWARN'][i] j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0] truetype = siminfo['OBJTYPE'][j] oiiflux = 0.0 if truetype == 'ELG': k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0] oiiflux = elginfo['OIIFLUX'][k] truez = siminfo['REDSHIFT'][j] dv = C_LIGHT*(z-truez)/(1+truez) status = None if truetype == 'SKY' and zwarn > 0: status = 'ok' elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) elif zwarn == 0: if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GALAXY': if abs(dv) < 150: status = 'ok' elif oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) else: status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux) elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype in ('STD', 'FSTD') and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format( pix, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.pixfile is not None): log.warning('--pixfile is deprecated; please use --outfile instead') if args.outfile is None: args.outfile = args.pixfile else: log.critical("Set --outfile not --pixfile and certainly not both") sys.exit(1) if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, fill_header=args.fill_header ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile io.write_image(outfile, img)
def main(args, comm=None) : log = get_logger() if args.npoly < 0 : log.warning("Need npoly>=0, changing this %d -> 1"%args.npoly) args.npoly=0 if args.nproc < 1 : log.warning("Need nproc>=1, changing this %d -> 1"%args.nproc) args.nproc=1 if comm is not None: if args.nproc != 1: if comm.rank == 0: log.warning("Using MPI, forcing multiprocessing nproc -> 1") args.nproc = 1 if args.objtype is not None: args.objtype = args.objtype.split(',') #- Read brick files for each channel if (comm is None) or (comm.rank == 0): log.info("Reading bricks") brick = dict() if args.brick is not None: if len(args.brickfiles) != 0: raise RuntimeError('Give -b/--brick or input brickfiles but not both') for channel in ('b', 'r', 'z'): filename = None if (comm is None) or (comm.rank == 0): filename = io.findfile('brick', band=channel, brickname=args.brick, specprod_dir=args.specprod_dir) if comm is not None: filename = comm.bcast(filename, root=0) brick[channel] = io.Brick(filename) else: for filename in args.brickfiles: bx = io.Brick(filename) if bx.channel not in brick: brick[bx.channel] = bx else: if (comm is None) or (comm.rank == 0): log.error('Channel {} in multiple input files'.format(bx.channel)) sys.exit(2) filters=brick.keys() for fil in filters: if (comm is None) or (comm.rank == 0): log.info("Filter found: "+fil) #- Assume all channels have the same number of targets #- TODO: generalize this to allow missing channels #if args.nspec is None: # args.nspec = brick['b'].get_num_targets() # log.info("Fitting {} targets".format(args.nspec)) #else: # log.info("Fitting {} of {} targets".format(args.nspec, brick['b'].get_num_targets())) #- Coadd individual exposures and combine channels #- Full coadd code is a bit slow, so try something quick and dirty for #- now to get something going for redshifting if (comm is None) or (comm.rank == 0): log.info("Combining individual channels and exposures") wave=[] for fil in filters: wave=np.concatenate([wave,brick[fil].get_wavelength_grid()]) np.ndarray.sort(wave) nwave = len(wave) #- flux and ivar arrays to fill for all targets #flux = np.zeros((nspec, nwave)) #ivar = np.zeros((nspec, nwave)) flux = [] ivar = [] good_targetids=[] targetids = brick['b'].get_target_ids() fpinfo = None if args.print_info is not None: if (comm is None) or (comm.rank == 0): fpinfo = open(args.print_info,"w") for i, targetid in enumerate(targetids): #- wave, flux, and ivar for this target; concatenate xwave = list() xflux = list() xivar = list() good=True for channel in filters: exp_flux, exp_ivar, resolution, info = brick[channel].get_target(targetid) weights = np.sum(exp_ivar, axis=0) ii, = np.where(weights > 0) if len(ii)==0: good=False break xwave.extend(brick[channel].get_wavelength_grid()[ii]) #- Average multiple exposures on the same wavelength grid for each channel xflux.extend(np.average(exp_flux[:,ii], weights=exp_ivar[:,ii], axis=0)) xivar.extend(weights[ii]) if not good: continue xwave = np.array(xwave) xivar = np.array(xivar) xflux = np.array(xflux) ii = np.argsort(xwave) #flux[i], ivar[i] = resample_flux(wave, xwave[ii], xflux[ii], xivar[ii]) fl, iv = resample_flux(wave, xwave[ii], xflux[ii], xivar[ii]) flux.append(fl) ivar.append(iv) good_targetids.append(targetid) if not args.print_info is None: s2n = np.median(fl[:-1]*np.sqrt(iv[:-1])/np.sqrt(wave[1:]-wave[:-1])) if (comm is None) or (comm.rank == 0): print targetid,s2n fpinfo.write(str(targetid)+" "+str(s2n)+"\n") if not args.print_info is None: if (comm is None) or (comm.rank == 0): fpinfo.close() sys.exit() good_targetids=good_targetids[args.first_spec:] flux=np.array(flux[args.first_spec:]) ivar=np.array(ivar[args.first_spec:]) nspec=len(good_targetids) if (comm is None) or (comm.rank == 0): log.info("number of good targets = %d"%nspec) if (args.nspec is not None) and (args.nspec < nspec): if (comm is None) or (comm.rank == 0): log.info("Fitting {} of {} targets".format(args.nspec, nspec)) nspec=args.nspec good_targetids=good_targetids[:nspec] flux=flux[:nspec] ivar=ivar[:nspec] else : if (comm is None) or (comm.rank == 0): log.info("Fitting {} targets".format(nspec)) if (comm is None) or (comm.rank == 0): log.debug("flux.shape={}".format(flux.shape)) zf = None if comm is None: # Use multiprocessing built in to RedMonster. zf = RedMonsterZfind(wave= wave,flux= flux,ivar=ivar, objtype=args.objtype,zrange_galaxy= args.zrange_galaxy, zrange_qso=args.zrange_qso,zrange_star=args.zrange_star, nproc=args.nproc,npoly=args.npoly) else: # Use MPI # distribute the spectra among processes my_firstspec, my_nspec = dist_uniform(nspec, comm.size, comm.rank) my_specs = slice(my_firstspec, my_firstspec + my_nspec) for p in range(comm.size): if p == comm.rank: if my_nspec > 0: log.info("process {} fitting spectra {} - {}".format(p, my_firstspec, my_firstspec+my_nspec-1)) else: log.info("process {} idle".format(p)) sys.stdout.flush() comm.barrier() # do redshift fitting on each process myzf = None if my_nspec > 0: savelevel = os.environ["DESI_LOGLEVEL"] os.environ["DESI_LOGLEVEL"] = "WARNING" myzf = RedMonsterZfind(wave=wave, flux=flux[my_specs,:], ivar=ivar[my_specs,:], objtype=args.objtype,zrange_galaxy= args.zrange_galaxy, zrange_qso=args.zrange_qso,zrange_star=args.zrange_star, nproc=args.nproc,npoly=args.npoly) os.environ["DESI_LOGLEVEL"] = savelevel # Combine results into a single ZFindBase object on the root process. # We could do this with a gather, but we are using a small number of # processes, and point-to-point communication is easier for people to # understand. if comm.rank == 0: zf = ZfindBase(myzf.wave, np.zeros((nspec, myzf.nwave)), np.zeros((nspec, myzf.nwave)), R=None, results=None) for p in range(comm.size): if comm.rank == 0: if p == 0: # root process copies its own data into output zf.flux[my_specs] = myzf.flux zf.ivar[my_specs] = myzf.ivar zf.model[my_specs] = myzf.model zf.z[my_specs] = myzf.z zf.zerr[my_specs] = myzf.zerr zf.zwarn[my_specs] = myzf.zwarn zf.spectype[my_specs] = myzf.spectype zf.subtype[my_specs] = myzf.subtype else: # root process receives from process p and copies # it into the output. p_nspec = comm.recv(source=p, tag=0) # only proceed if the sending process actually # has some spectra assigned to it. if p_nspec > 0: p_firstspec = comm.recv(source=p, tag=1) p_slice = slice(p_firstspec, p_firstspec+p_nspec) p_flux = comm.recv(source=p, tag=2) zf.flux[p_slice] = p_flux p_ivar = comm.recv(source=p, tag=3) zf.ivar[p_slice] = p_ivar p_model = comm.recv(source=p, tag=4) zf.model[p_slice] = p_model p_z = comm.recv(source=p, tag=5) zf.z[p_slice] = p_z p_zerr = comm.recv(source=p, tag=6) zf.zerr[p_slice] = p_zerr p_zwarn = comm.recv(source=p, tag=7) zf.zwarn[p_slice] = p_zwarn p_type = comm.recv(source=p, tag=8) zf.spectype[p_slice] = p_type p_subtype = comm.recv(source=p, tag=9) zf.subtype[p_slice] = p_subtype else: if p == comm.rank: # process p sends to root comm.send(my_nspec, dest=0, tag=0) if my_nspec > 0: comm.send(my_firstspec, dest=0, tag=1) comm.send(myzf.flux, dest=0, tag=2) comm.send(myzf.ivar, dest=0, tag=3) comm.send(myzf.model, dest=0, tag=4) comm.send(myzf.z, dest=0, tag=5) comm.send(myzf.zerr, dest=0, tag=6) comm.send(myzf.zwarn, dest=0, tag=7) comm.send(myzf.spectype, dest=0, tag=8) comm.send(myzf.subtype, dest=0, tag=9) comm.barrier() if (comm is None) or (comm.rank == 0): # The full results exist only on the rank zero process. # reformat results dtype = list() dtype = [ ('Z', zf.z.dtype), ('ZERR', zf.zerr.dtype), ('ZWARN', zf.zwarn.dtype), ('SPECTYPE', zf.spectype.dtype), ('SUBTYPE', zf.subtype.dtype), ] formatted_data = np.empty(nspec, dtype=dtype) formatted_data['Z'] = zf.z formatted_data['ZERR'] = zf.zerr formatted_data['ZWARN'] = zf.zwarn formatted_data['SPECTYPE'] = zf.spectype formatted_data['SUBTYPE'] = zf.subtype # Create a ZfindBase object with formatted results zfi = ZfindBase(None, None, None, results=formatted_data) zfi.nspec = nspec # QA if (args.qafile is not None) or (args.qafig is not None): log.info("performing skysub QA") # Load qabrick = load_qa_brick(args.qafile) # Run qabrick.run_qa('ZBEST', (zfi,brick)) # Write if args.qafile is not None: write_qa_brick(args.qafile, qabrick) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: raise IOError("Not yet implemented") qa_plots.brick_zbest(args.qafig, zfi, qabrick) #- Write some output if args.outfile is None: args.outfile = io.findfile('zbest', brickname=args.brick) log.info("Writing "+args.outfile) #io.write_zbest(args.outfile, args.brick, targetids, zfi, zspec=args.zspec) io.write_zbest(args.outfile, args.brick, good_targetids, zfi, zspec=args.zspec) return
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) # YEARMMDD string, rolls over at noon not midnight # TODO: fix usage of night to be something other than today if night is None: #night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # create production # FIXME: someday run PSF estimation too... ### com = "desi_pipe --env env.txt --spectrographs 0 --fakeboot --fakepsf" rawdir = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD')) com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf --raw {}".format(rawdir) sp.check_call(com, shell=True) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.read_options(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec pipe.write_options(optpath, opts) # run the generated shell scripts # FIXME: someday run PSF estimation too... # print("Running bootcalib script...") # com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh") # sp.check_call(["bash", com]) # print("Running specex script...") # com = os.path.join(proddir, "run", "scripts", "specex_all.sh") # sp.check_call(["bash", com]) # print("Running psfcombine script...") # com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh") # sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "extract_all.sh") print("Running extraction script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "fiberflat-procexp_all.sh") print("Running calibration script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "bricks.sh") print("Running makebricks script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "zfind_all.sh") print("Running zfind script "+com) sp.check_call(["bash", com]) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'METADATA') brickdirs = glob.glob(os.path.join(proddir, "bricks", "*")) bricks = [ os.path.basename(x) for x in brickdirs ] print() print("--------------------------------------------------") print("Brick True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for b in bricks: zbest = io.read_zbest(io.findfile('zbest', brickname=b)) for i in range(len(zbest.z)): if zbest.spectype[i] == 'ssp_em_galaxy': objtype = 'GAL' elif zbest.spectype[i] == 'spEigenStar': objtype = 'STAR' else: objtype = zbest.spectype[i] z, zwarn = zbest.z[i], zbest.zwarn[i] j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0] truetype = siminfo['OBJTYPE'][j] truez = siminfo['REDSHIFT'][j] dv = 3e5*(z-truez)/(1+truez) if truetype == 'SKY' and zwarn > 0: status = 'ok' elif zwarn == 0: if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype == 'STD' and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format( b, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def load_all_s2n_values(nights, channel, sub_exposures=None): """ Calculate S/N values for a set of spectra from an input list of nights Args: nights: list channel: str ('b','r','z') sub_exposures: Returns: fdict: dict Contains all the S/N info for all nights in the given channel """ fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night)#, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') log.debug('Getting truth from {}'.format(simspec_file)) sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data,masked=True) #- Get OIIFLUX from separate HDU and join if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu): elg_truth = Table(sps_hdu['TRUTH_ELG'].data) sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'], keys='TARGETID', join_type='left') else: sps_tab['OIIFLUX'] = 0.0 sps_hdu.close() #objs = sps_tab['TEMPLATETYPE'] == objtype #if np.sum(objs) == 0: # continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel+str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: log.debug('Reading from {}'.format(cframe_path)) cframe = read_frame(cframe_path) except (IOError, OSError): log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave,1) dwave[0] = dwave[1] # Calculate s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave) #s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave) # Save fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers]) fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers]) fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def main(args) : # imports import glob from desispec.io import findfile, makepath from desispec.io import get_exposures from desispec.io import get_files, get_nights from desispec.io import get_reduced_frames from desispec.io import specprod_root from desispec.io import qaprod_root from desispec.qa import utils as qa_utils import copy import pdb # Init specprod_dir = specprod_root() # Log log=get_logger() log.info("starting") # Path if args.qaprod_dir is not None: qaprod_dir = args.qaprod_dir else: qaprod_dir = qaprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b','r','z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Exposure plot? if args.expid is not None: # Nights if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres(cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series(channel_dict[channel], 'wave', outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel)) skysub_resid_series(channel_dict[channel], 'flux', outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel)) return # Skyline if args.skyline: from desispec.qa.qa_plots import skyline_resid # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) if len(cframes) == 1: log.error('len(cframes)==1; starting debugging') pdb.set_trace() # Need to call differently else: sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes, flatten=False) # Plot outfile=args.outdir+'/skyline_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel) makepath(outfile) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Test sky noise for Gaussianity if args.gauss: from desispec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes) # Plot log.info("Plotting..") outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel) makepath(outfile) skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return
def main(args) : # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log=get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir = args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(),1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))