def setUp(self): self.fileio = "test_spectra.fits" self.fileappend = "test_spectra_append.fits" self.filebuild = "test_spectra_build.fits" self.meta = { "KEY1" : "VAL1", "KEY2" : "VAL2" } self.nwave = 100 self.nspec = 5 self.ndiag = 3 fmap = empty_fibermap(self.nspec) fmap = add_columns(fmap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(0), np.int32(0), np.int32(0)], ) for s in range(self.nspec): fmap[s]["TARGETID"] = 456 + s fmap[s]["FIBER"] = 123 + s fmap[s]["NIGHT"] = s fmap[s]["EXPID"] = s self.fmap1 = encode_table(fmap) fmap = empty_fibermap(self.nspec) fmap = add_columns(fmap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(0), np.int32(0), np.int32(0)], ) for s in range(self.nspec): fmap[s]["TARGETID"] = 789 + s fmap[s]["FIBER"] = 200 + s fmap[s]["NIGHT"] = 1000 fmap[s]["EXPID"] = s self.fmap2 = encode_table(fmap) self.bands = ["b", "r", "z"] self.wave = {} self.flux = {} self.ivar = {} self.mask = {} self.res = {} self.extra = {} for s in range(self.nspec): for b in self.bands: self.wave[b] = np.arange(self.nwave) self.flux[b] = np.repeat(np.arange(self.nspec), self.nwave).reshape( (self.nspec, self.nwave) ) + 3.0 self.ivar[b] = 1.0 / self.flux[b] self.mask[b] = np.tile(np.arange(2, dtype=np.uint32), (self.nwave * self.nspec) // 2).reshape( (self.nspec, self.nwave) ) self.res[b] = np.zeros( (self.nspec, self.ndiag, self.nwave), dtype=np.float64) self.res[b][:,1,:] = 1.0 self.extra[b] = {} self.extra[b]["FOO"] = self.flux[b]
def _write_fibermap(self): """Write a fake fiberflat""" fibermap = io.empty_fibermap(self.nspec) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' io.write_fibermap(self.fibermapfile, fibermap)
def get_frame_data(nspec=10, objtype=None): """ Return basic test data for desispec.frame object: """ nwave = 100 wavemin, wavemax = 4000, 4100 wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax) resol_data = set_resolmatrix(nspec, nwave) calib = np.sin((wave - wavemin) * np.pi / np.max(wave)) flux = np.zeros((nspec, nwave)) for i in range(nspec): flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib) sigma = 0.01 # flux += np.random.normal(scale=sigma, size=flux.shape) ivar = np.ones(flux.shape) / sigma**2 mask = np.zeros(flux.shape, dtype=int) fibermap = empty_fibermap(nspec, 1500) if objtype is None: fibermap['OBJTYPE'] = 'QSO' fibermap['OBJTYPE'][0:3] = 'STD' # For flux tests else: fibermap['OBJTYPE'] = objtype frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap) frame.meta = {} frame.meta['EXPTIME'] = 1. # For flux tests return frame
def get_frame_data(nspec=10): """ Return basic test data for desispec.frame object: """ nwave = 100 wavemin, wavemax = 4000, 4100 wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax) resol_data=set_resolmatrix(nspec,nwave) calib = np.sin((wave-wavemin) * np.pi / np.max(wave)) flux = np.zeros((nspec, nwave)) for i in range(nspec): flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib) sigma = 0.01 # flux += np.random.normal(scale=sigma, size=flux.shape) ivar = np.ones(flux.shape) / sigma**2 mask = np.zeros(flux.shape, dtype=int) fibermap = empty_fibermap(nspec, 1500) fibermap['OBJTYPE'] = 'TGT' fibermap['DESI_TARGET'] = desi_mask.QSO fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests meta = dict(EXPTIME = 1.0) frame = Frame(wave, flux, ivar, mask,resol_data,fibermap=fibermap, meta=meta) return frame
def _get_fibermap(self): fibermap = io.empty_fibermap(self.nspec, 1500) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' fibermap['DESI_TARGET'][i] = desi_mask.SKY fibermap['OBJTYPE'][i + 1] = 'TGT' fibermap['DESI_TARGET'][i + 1] = desi_mask.STD_FAINT return fibermap
def _get_fibermap(self): fibermap = io.empty_fibermap(self.nspec, 1500) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' fibermap['DESI_TARGET'][i] = desi_mask.SKY fibermap['OBJTYPE'][i+1] = 'TGT' fibermap['DESI_TARGET'][i+1] = desi_mask.STD_FAINT return fibermap
def _get_spectra(self, with_gradient=False): #- Setup data for a Resolution matrix sigma2 = 4.0 ndiag = 21 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((self.nspec, ndiag, self.nwave)) for i in range(self.nspec): kernel = np.exp(-(xx + float(i) / self.nspec * 0.3)**2 / (2 * sigma2)) #kernel = np.exp(-xx**2/(2*sigma2)) kernel /= sum(kernel) for j in range(self.nwave): Rdata[i, :, j] = kernel flux = np.zeros((self.nspec, self.nwave), dtype=float) ivar = np.ones((self.nspec, self.nwave), dtype=float) # Add a random component for i in range(self.nspec): ivar[i] += 0.4 * np.random.uniform(size=self.nwave) mask = np.zeros((self.nspec, self.nwave), dtype=int) fibermap = empty_fibermap(self.nspec, 1500) fibermap['OBJTYPE'][0::2] = 'SKY' x = fibermap["FIBERASSIGN_X"] y = fibermap["FIBERASSIGN_Y"] x = x - np.mean(x) y = y - np.mean(y) if np.std(x) > 0: x /= np.std(x) if np.std(y) > 0: y /= np.std(y) w = (self.wave - self.wave[0]) / (self.wave[-1] - self.wave[0]) * 2. - 1 for i in range(self.nspec): R = Resolution(Rdata[i]) if with_gradient: scale = 1. + (0.1 * x[i] + 0.2 * y[i]) * (1 + 0.4 * w) flux[i] = R.dot(scale * self.flux) else: flux[i] = R.dot(self.flux) meta = {"camera": "r2"} return Frame(self.wave, flux, ivar, mask, Rdata, spectrograph=2, fibermap=fibermap, meta=meta)
def _random_spectra(self, ns=3, nw=10): wave = np.linspace(5000, 5100, nw) flux = np.random.uniform(0, 1, size=(ns,nw)) ivar = np.random.uniform(0, 1, size=(ns,nw)) #mask = np.zeros((ns,nw),dtype=int) mask = None rdat = np.ones((ns,3,nw)) rdat[:,0] *= 0.25 rdat[:,1] *= 0.5 rdat[:,2] *= 0.25 fmap = empty_fibermap(ns) fmap["TARGETID"][:]=12 # same target return Spectra(bands=["x"],wave={"x":wave},flux={"x":flux},ivar={"x":ivar}, mask=None, resolution_data={"x":rdat} , fibermap=fmap)
def test_main(self): """ Test the main program. """ # generate the frame data wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 11 xx = np.linspace(-(ndiag - 1) / 2.0, +(ndiag - 1) / 2.0, ndiag) Rdata = np.zeros((nspec, ndiag, nwave)) kernel = np.exp(-xx**2 / (2 * sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i, :, j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) # create a fake fibermap fibermap = io.empty_fibermap(nspec, nwave) for i in range(0, nspec): fibermap['OBJTYPE'][i] = 'FAKE' io.write_fibermap(self.testfibermap, fibermap) #- write out the frame frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap, meta=dict(FLAVOR='flat')) write_frame(self.testframe, frame, fibermap=fibermap) # set program arguments argstr = ['--infile', self.testframe, '--outfile', self.testflat] # run it args = ffscript.parse(options=argstr) ffscript.main(args)
def _get_fibermap(self, gaia_only=False): fibermap = io.empty_fibermap(self.nspec, 1500) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' fibermap['DESI_TARGET'][i] = desi_mask.SKY fibermap['OBJTYPE'][i + 1] = 'TGT' fibermap['DESI_TARGET'][i + 1] = desi_mask.STD_FAINT fibermap['GAIA_PHOT_G_MEAN_MAG'][i + 1] = 15 fibermap['GAIA_PHOT_BP_MEAN_MAG'][i + 1] = 15 fibermap['GAIA_PHOT_RP_MEAN_MAG'][i + 1] = 15 if gaia_only: fibermap['PHOTSYS'] = 'G' else: fibermap['FLUX_G'][i + 1] = 100 fibermap['FLUX_R'][i + 1] = 100 fibermap['FLUX_Z'][i + 1] = 100 return fibermap
def test_main(self): """ Test the main program. """ # generate the frame data wave, flux, ivar, mask = _get_data() nspec, nwave = flux.shape #- Setup data for a Resolution matrix sigma = 4.0 ndiag = 11 xx = np.linspace(-(ndiag-1)/2.0, +(ndiag-1)/2.0, ndiag) Rdata = np.zeros( (nspec, ndiag, nwave) ) kernel = np.exp(-xx**2/(2*sigma)) kernel /= sum(kernel) for i in range(nspec): for j in range(nwave): Rdata[i,:,j] = kernel #- Convolve the data with the resolution matrix convflux = np.empty_like(flux) for i in range(nspec): convflux[i] = Resolution(Rdata[i]).dot(flux[i]) # create a fake fibermap fibermap = io.empty_fibermap(nspec, nwave) for i in range(0, nspec): fibermap['OBJTYPE'][i] = 'FAKE' io.write_fibermap(self.testfibermap, fibermap) #- write out the frame frame = Frame(wave, convflux, ivar, mask, Rdata, spectrograph=0, fibermap=fibermap) write_frame(self.testframe, frame, fibermap=fibermap) # set program arguments argstr = [ '--infile', self.testframe, '--outfile', self.testflat ] # run it args = ffscript.parse(options=argstr) ffscript.main(args)
def get_frame_data(nspec=10, wavemin=4000, wavemax=4100, nwave=100, meta={}): """ Return basic test data for desispec.frame object: """ wave, model_flux = get_models(nspec, nwave, wavemin=wavemin, wavemax=wavemax) resol_data = set_resolmatrix(nspec, nwave) calib = np.sin((wave - wavemin) * np.pi / np.max(wave)) flux = np.zeros((nspec, nwave)) for i in range(nspec): flux[i] = Resolution(resol_data[i]).dot(model_flux[i] * calib) sigma = 0.01 # flux += np.random.normal(scale=sigma, size=flux.shape) ivar = np.ones(flux.shape) / sigma**2 mask = np.zeros(flux.shape, dtype=int) fibermap = empty_fibermap(nspec, 1500) fibermap['OBJTYPE'] = 'TGT' fibermap['DESI_TARGET'] = desi_mask.QSO fibermap['DESI_TARGET'][0:3] = desi_mask.STD_FAINT # For flux tests fibermap['FIBER_X'] = np.arange(nspec) * 400. / nspec #mm fibermap['FIBER_Y'] = np.arange(nspec) * 400. / nspec #mm fibermap['DELTA_X'] = 0.005 * np.ones(nspec) #mm fibermap['DELTA_Y'] = 0.003 * np.ones(nspec) #mm if "EXPTIME" not in meta.keys(): meta['EXPTIME'] = 1.0 frame = Frame(wave, flux, ivar, mask, resol_data, fibermap=fibermap, meta=meta) return frame
def _get_fibermap(self): fibermap = io.empty_fibermap(self.nspec, 1500) for i in range(0, self.nspec, 3): fibermap['OBJTYPE'][i] = 'SKY' fibermap['OBJTYPE'][i+1] = 'STD' return fibermap
def setUp(cls): cls.program = program = 'dark' cls.flavor = flavor = 'bias' cls.night = night = '20150105' cls.camera = camera = 'r0' cls.expid = expid = 314 cls.psfExpid = psfExpid = 313 cls.flatExpid = flatExpid = 312 cls.templateExpid = templateExpid = 311 cls.nspec = nspec = 5 cls.exptime = exptime = 100 #- Setup environment and override default environment variables #- python 2.7 location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # py/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') if not os.path.isdir(cls.binDir): #- python 3.x setup.py test location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # build/ os.path.dirname( # lib/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') #- last attempt if not os.path.isdir(cls.binDir): cls.topDir = os.getcwd() cls.binDir = os.path.join(cls.topDir, 'bin') if not os.path.isdir(cls.binDir): raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__)) id = uuid4().hex cls.fibermapfile = 'fibermap-'+id+'.fits' cls.framefile = 'frame-'+id+'.fits' cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io') datanightDir = os.path.join(testDir,night) dataDir = os.path.join(datanightDir,'{:08d}'.format(expid)) expDir = os.path.join(testDir,'exposures') expnightDir = os.path.join(expDir,night) reduxDir = os.path.join(expnightDir,'{:08d}'.format(expid)) calibDir = os.path.join(testDir, 'ql_calib') configDir = os.path.join(testDir, 'ql_config') os.environ['QL_CALIB_DIR'] = calibDir os.environ['QL_CONFIG_DIR'] = configDir if not os.path.exists(testDir): os.makedirs(testDir) os.makedirs(datanightDir) os.makedirs(dataDir) os.makedirs(expDir) os.makedirs(expnightDir) os.makedirs(reduxDir) os.makedirs(calibDir) os.makedirs(configDir) #- Write dummy configuration and input files to test merging configdict = {'name': 'Test Configuration', 'Program': program, 'Flavor': flavor, 'PSFExpid': psfExpid, 'PSFType': 'psf', 'FiberflatExpid': flatExpid, 'TemplateExpid': templateExpid, 'TemplateNight': night, 'WritePreprocfile': False, 'WriteSkyModelfile': False, 'WriteIntermediatefiles': False, 'WriteStaticPlots': False, 'Debuglevel': 20, 'UseResolution': False, 'Period': 5.0, 'Timeout': 120.0, 'Pipeline': ['Initialize','Preproc'], 'Algorithms': {'Initialize':{ 'QA':{'Check_HDUs':{'PARAMS':{}} }}, 'Preproc':{ 'QA':{'Bias_From_Overscan':{'PARAMS':{'BIAS_AMP_NORMAL_RANGE':[-100.0,100.0],'BIAS_AMP_WARN_RANGE':[-200.0,200.0]}}, 'Get_RMS':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'NOISE_AMP_NORMAL_RANGE':[-1.0,1.0],'NOISE_AMP_WARN_RANGE':[-2.0,2.0]}}, 'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[-0.1,0.1],'LITFRAC_WARN_RANGE':[-0.2,0.2]}}}}} } with open('{}/test_config.yaml'.format(testDir),'w') as config: yaml.dump(configdict,config) cls.configfile = '{}/test_config.yaml'.format(testDir) #- Generate raw file rawfile = os.path.join(dataDir,'desi-00000314.fits.fz') raw_hdr = {} raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988' raw_hdr['NIGHT'] = night raw_hdr['PROGRAM'] = program raw_hdr['FLAVOR'] = flavor raw_hdr['CAMERA'] = camera raw_hdr['EXPID'] = expid raw_hdr['EXPTIME'] = exptime raw_hdr['DOSVER'] = 'SIM' raw_hdr['FEEVER'] = 'SIM' raw_hdr['DETECTOR'] = 'SIM' raw_hdr['PRESEC1'] = '[1:4,1:2048]' raw_hdr['DATASEC1'] = '[5:2052,1:2048]' raw_hdr['BIASSEC1'] = '[2053:2102,1:2048]' raw_hdr['CCDSEC1'] = '[1:2048,1:2048]' raw_hdr['PRESEC2'] = '[4201:4204,1:2048]' raw_hdr['DATASEC2'] = '[2153:4200,1:2048]' raw_hdr['BIASSEC2'] = '[2103:2152,1:2048]' raw_hdr['CCDSEC2'] = '[2049:4096,1:2048]' raw_hdr['PRESEC3'] = '[1:4,2049:4096]' raw_hdr['DATASEC3'] = '[5:2052,2049:4096]' raw_hdr['BIASSEC3'] = '[2053:2102,2049:4096]' raw_hdr['CCDSEC3'] = '[1:2048,2049:4096]' raw_hdr['PRESEC4'] = '[4201:4204,2049:4096]' raw_hdr['DATASEC4'] = '[2153:4200,2049:4096]' raw_hdr['BIASSEC4'] = '[2103:2152,2049:4096]' raw_hdr['CCDSEC4'] = '[2049:4096,2049:4096]' raw_hdr['GAIN1'] = 1.0 raw_hdr['GAIN2'] = 1.0 raw_hdr['GAIN3'] = 1.0 raw_hdr['GAIN4'] = 1.0 raw_hdr['RDNOISE1'] = 3.0 raw_hdr['RDNOISE2'] = 3.0 raw_hdr['RDNOISE3'] = 3.0 raw_hdr['RDNOISE4'] = 3.0 primary_header={'PROGRAM':program} data=np.zeros((4096,4204))+200. raw_data=data.astype(int) write_raw(rawfile,raw_data,raw_hdr,primary_header=primary_header) #- Generate fibermap file fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits') fibermap = empty_fibermap(nspec) write_fibermap(fibermapfile,fibermap) #- Generate calib data for camera in ['b0', 'r0', 'z0']: #- Fiberflat has to exist but can be a dummpy file filename = '{}/fiberflat-{}.fits'.format(calibDir, camera) fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close() #- PSF has to be real file psffile = '{}/psf-{}.fits'.format(calibDir, camera) example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera)) shutil.copy(example_psf, psffile) #- Copy test calibration-data.yaml file specdir=calibDir+"spec/sp0" if not os.path.isdir(specdir) : os.makedirs(specdir) for c in "brz" : shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c))) #- Set calibration environment variable os.environ['DESI_SPECTRO_CALIB'] = calibDir
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR="./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR='specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR=os.environ['SPECPROD'] prod_Dir=specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory"%prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap=read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx=np.where(objtype=='STD') # match STD with STAR mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR bgsindx=np.where(objtype=='BGS') # match BGS with LRG objtype[stdindx]='STAR' objtype[mwsindx]='STAR' objtype[bgsindx]='LRG' NIGHT=fibermap.meta['NIGHT'] EXPID=fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj =='STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave>5800.0-1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32) continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32) for spec in range(args.nspec) : flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line*ref_integrated_flux, CONSTFLUXDENSITY=continuum*ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj,[]) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX'] gray_objects = ['gray','grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax= args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax,3,maxbin)) # object photons nsky = np.zeros((nmax,3,maxbin)) # sky photons nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i,camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: phot = np.vstack(phot) meanspec = resample_flux( waves[channel], camwave, np.average(phot/dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec+args.nstart-1)//500+1): camera = channel+str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start=max(500*kk,args.nstart) end=min(500*(kk+1),nmax) if (args.spectrograph <= kk): log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end)) ff = FiberFlat( waves[channel], fiberflat[start:end,:], ivar[start:end,:], mask[start:end,:], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in( 'Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in( 'Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0] # Get results for our flux-calibrated output file. cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0] cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0] noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] + output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[j, i, :num_pixels] = 25.0 / ( output['variance_electrons'][:,0] - output['num_source_electrons'][:,0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels) armName={"b":0,"r":1,"z":2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave=np.gradient(waves[channel]) nobj[:,armName[channel],:num_pixels]/=dwave frame_rand_noise[:,armName[channel],:num_pixels]/=dwave nivar[:,armName[channel],:num_pixels]*=dwave**2 nsky[:,armName[channel],:num_pixels]/=dwave sky_rand_noise[:,armName[channel],:num_pixels]/=dwave sky_ivar[:,armName[channel],:num_pixels]/=dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec+args.nstart-1)//500+1): start=max(500*ii,args.nstart) # first spectrum for a given spectrograph end=min(500*(ii+1),nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end)) num_pixels = len(waves[channel]) # Write frame file framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar=nivar[start:end,armName[channel],:num_pixels] sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart==start): resol=resolution[channel][:sh1,:,:] else: resol=resolution[channel][-sh1:,:,:] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera) cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels] cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName,cframe) cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar=sky_ivar[start:end,armName[channel],:num_pixels] skymask=np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera) flux = cframe_observedflux[start:end,armName[channel],:num_pixels] phot = nobj[start:end,armName[channel],:num_pixels] calibration = np.zeros_like(phot) jj = (flux>0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels] #mask=(1/calibivar>0).astype(int)?? mask=np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera) log.info("Wrote file {}".format(calibfilePath))
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR = "./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory" % DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR = 'specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR = os.environ['SPECPROD'] prod_Dir = specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory" % prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap = read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx = np.where(objtype == 'STD') # match STD with STAR mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR bgsindx = np.where(objtype == 'BGS') # match BGS with LRG objtype[stdindx] = 'STAR' objtype[mwsindx] = 'STAR' objtype[bgsindx] = 'LRG' NIGHT = fibermap.meta['NIGHT'] EXPID = fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_elg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs, rmagrange=args.rmagrange_bgs, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave > 5800.0 - 1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec) % 2 == 0).astype( np.float32) continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32) for spec in range(args.nspec): flux[spec, indx] = single_line[ spec] * ref_integrated_flux / np.gradient(wavelengths)[ indx] # single line flux[spec] += continuum[ spec] * ref_cst_flux_density # flat continuum meta1 = Table( dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx] * np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line * ref_integrated_flux, CONSTFLUXDENSITY=continuum * ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj, []) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column( Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX'] gray_objects = ['gray', 'grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning( 'Maybe using an outdated simspec file with flavor={}'.format( flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header[ 'MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax = args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty( (args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty( (args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i, camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: phot = np.vstack(phot) meanspec = resample_flux(waves[channel], camwave, np.average(phot / dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec + args.nstart - 1) // 500 + 1): camera = channel + str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start = max(500 * kk, args.nstart) end = min(500 * (kk + 1), nmax) if (args.spectrograph <= kk): log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, kk, start, end)) ff = FiberFlat(waves[channel], fiberflat[start:end, :], ivar[start:end, :], mask[start:end, :], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID, camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in('Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec + args.nstart - 1) // 500 + 1): start = max(500 * ii, args.nstart) # first spectrum for a given spectrograph end = min(500 * (ii + 1), nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, ii, start, end)) num_pixels = len(waves[channel]) # Write frame file framefileName = desispec.io.findfile("frame", NIGHT, EXPID, camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar = nivar[start:end, armName[channel], :num_pixels] sh1 = frame_flux.shape[ 0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart == start): resol = resolution[channel][:sh1, :, :] else: resol = resolution[channel][-sh1:, :, :] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath = desispec.io.findfile("frame", NIGHT, EXPID, camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID, camera) cframeFlux = cframe_observedflux[ start:end, armName[channel], :num_pixels] + cframe_rand_noise[ start:end, armName[channel], :num_pixels] cframeIvar = cframe_ivar[start:end, armName[channel], :num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName, cframe) cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID, camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar = sky_ivar[start:end, armName[channel], :num_pixels] skymask = np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID, camera) flux = cframe_observedflux[start:end, armName[channel], :num_pixels] phot = nobj[start:end, armName[channel], :num_pixels] calibration = np.zeros_like(phot) jj = (flux > 0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar = 10 / cframe_ivar[start:end, armName[channel], :num_pixels] #mask=(1/calibivar>0).astype(int)?? mask = np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID, camera) log.info("Wrote file {}".format(calibfilePath))
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT=(night, 'Night of observation YEARMMDD'), EXPID=(expid, 'DESI exposure ID'), TILEID=(tileid, 'DESI tile ID'), FLAVOR=(flavor, 'Flavor [arc, flat, science, ...]'), TELRA=(telera, 'Telescope pointing RA [degrees]'), TELDEC=(teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = ['3412p195' for x in range(nspec)] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10, 10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir
def fake_raw(): dirhash = uuid.uuid4() night = fake_night() rawdir = "test_pipe_raw_{}".format(dirhash) if os.path.exists(rawdir): shutil.rmtree(rawdir) os.makedirs(rawdir) nightdir = os.path.join(rawdir, night) # set up one spectrograph (500 fibers) nspec = 500 # arc expid = "00000000" tileid = "0" flavor = "arc" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ARC' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # flat expid = "00000001" tileid = "1" flavor = "flat" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'FLAT' io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) # science expid = "00000002" tileid = "2" flavor = "dark" telera = "0.0" teledec = "0.0" fmfile = os.path.join(nightdir, "fibermap-{}.fits".format(expid)) hdr = dict( NIGHT = (night, 'Night of observation YEARMMDD'), EXPID = (expid, 'DESI exposure ID'), TILEID = (tileid, 'DESI tile ID'), FLAVOR = (flavor, 'Flavor [arc, flat, science, ...]'), TELRA = (telera, 'Telescope pointing RA [degrees]'), TELDEC = (teledec, 'Telescope pointing dec [degrees]'), ) fibermap = io.empty_fibermap(nspec) fibermap['OBJTYPE'] = 'ELG' fibermap['FIBER'] = np.arange(nspec, dtype='i4') fibermap['TARGETID'] = np.random.randint(sys.maxsize, size=nspec) fibermap['BRICKNAME'] = [ '3412p195' for x in range(nspec) ] io.write_fibermap(fmfile, fibermap, header=hdr) for cam in ["r0", "b0", "z0"]: pfile = os.path.join(nightdir, "pix-{}-{}.fits".format(cam, expid)) pix = np.random.normal(0, 3.0, size=(10,10)) ivar = np.ones_like(pix) / 3.0**2 mask = np.zeros(pix.shape, dtype=np.uint32) img = dimg.Image(pix, ivar, mask, camera=cam) io.write_image(pfile, img) return rawdir
def setUp(self): #- catch specific warnings so that we can find and fix # warnings.filterwarnings("error", ".*did not parse as fits unit.*") #- Test data and files to work with self.fileio = "test_spectra.fits" self.fileappend = "test_spectra_append.fits" self.filebuild = "test_spectra_build.fits" self.meta = {"KEY1": "VAL1", "KEY2": "VAL2"} self.nwave = 100 self.nspec = 5 self.ndiag = 3 fmap = empty_fibermap(self.nspec) fmap = add_columns( fmap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(0), np.int32(0), np.int32(0)], ) for s in range(self.nspec): fmap[s]["TARGETID"] = 456 + s fmap[s]["FIBER"] = 123 + s fmap[s]["NIGHT"] = s fmap[s]["EXPID"] = s self.fmap1 = encode_table(fmap) fmap = empty_fibermap(self.nspec) fmap = add_columns( fmap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(0), np.int32(0), np.int32(0)], ) for s in range(self.nspec): fmap[s]["TARGETID"] = 789 + s fmap[s]["FIBER"] = 200 + s fmap[s]["NIGHT"] = 1000 fmap[s]["EXPID"] = 1000 + s self.fmap2 = encode_table(fmap) for s in range(self.nspec): fmap[s]["TARGETID"] = 1234 + s fmap[s]["FIBER"] = 300 + s fmap[s]["NIGHT"] = 2000 fmap[s]["EXPID"] = 2000 + s self.fmap3 = encode_table(fmap) self.bands = ["b", "r", "z"] self.wave = {} self.flux = {} self.ivar = {} self.mask = {} self.res = {} self.extra = {} for s in range(self.nspec): for b in self.bands: self.wave[b] = np.arange(self.nwave) self.flux[b] = np.repeat(np.arange(self.nspec), self.nwave).reshape( (self.nspec, self.nwave)) + 3.0 self.ivar[b] = 1.0 / self.flux[b] self.mask[b] = np.tile(np.arange(2, dtype=np.uint32), (self.nwave * self.nspec) // 2).reshape( (self.nspec, self.nwave)) self.res[b] = np.zeros((self.nspec, self.ndiag, self.nwave), dtype=np.float64) self.res[b][:, 1, :] = 1.0 self.extra[b] = {} self.extra[b]["FOO"] = self.flux[b] self.scores = dict(BLAT=np.arange(self.nspec), FOO=np.ones(self.nspec)) self.extra_catalog = Table() self.extra_catalog['A'] = np.arange(self.nspec) self.extra_catalog['B'] = np.ones(self.nspec)
def setUp(cls): cls.program = program = 'dark' cls.flavor = flavor = 'bias' cls.night = night = '20150105' cls.camera = camera = 'r0' cls.expid = expid = 314 cls.psfExpid = psfExpid = 313 cls.flatExpid = flatExpid = 312 cls.templateExpid = templateExpid = 311 cls.nspec = nspec = 5 cls.exptime = exptime = 100 #- Seup environment and override default environment variables #- python 2.7 location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # py/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') if not os.path.isdir(cls.binDir): #- python 3.x setup.py test location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # build/ os.path.dirname( # lib/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') #- last attempt if not os.path.isdir(cls.binDir): cls.topDir = os.getcwd() cls.binDir = os.path.join(cls.topDir, 'bin') if not os.path.isdir(cls.binDir): raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__)) id = uuid4().hex cls.fibermapfile = 'fibermap-'+id+'.fits' cls.framefile = 'frame-'+id+'.fits' cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io') dataDir = os.path.join(testDir,night) expDir = os.path.join(testDir,'exposures') nightDir = os.path.join(expDir,night) reduxDir = os.path.join(nightDir,'{:08d}'.format(expid)) if not os.path.exists(testDir): os.makedirs(testDir) os.makedirs(dataDir) os.makedirs(expDir) os.makedirs(nightDir) os.makedirs(reduxDir) #- Write dummy configuration and input files to test merging configdict = {'name': 'Test Configuration', 'Program': program, 'Flavor': flavor, 'PSFExpid': psfExpid, 'PSFType': 'psf', 'FiberflatExpid': flatExpid, 'TemplateExpid': templateExpid, 'WritePixfile': False, 'WriteSkyModelfile': False, 'WriteIntermediatefiles': False, 'WriteStaticPlots': False, 'Debuglevel': 20, 'UseResolution': False, 'Period': 5.0, 'Timeout': 120.0, 'Pipeline': ['Initialize','Preproc'], 'Algorithms': {'Initialize':{ 'QA':{ 'Bias_From_Overscan':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'DIFF_WARN_RANGE':[-1.0,1.0],'DIFF_ALARM_RANGE':[-2.0,2.0]}}}}, 'Preproc':{ 'QA':{ 'Get_RMS':{'PARAMS':{'RMS_WARN_RANGE':[-1.0,1.0],'RMS_ALARM_RANGE':[-2.0,2.0]}}, 'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[200.0,500.0],'LITFRAC_WARN_RANGE':[50.0,650.0]}}}}} } with open('{}/test_config.yaml'.format(testDir),'w') as config: yaml.dump(configdict,config) cls.configfile = '{}/test_config.yaml'.format(testDir) #- Generate raw file rawfile = os.path.join(dataDir,'desi-00000314.fits.fz') raw_hdr = {} raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988' raw_hdr['NIGHT'] = night raw_hdr['PROGRAM'] = program raw_hdr['FLAVOR'] = flavor raw_hdr['CAMERA'] = camera raw_hdr['EXPID'] = expid raw_hdr['EXPTIME'] = exptime raw_hdr['DOSVER'] = 'SIM' raw_hdr['FEEVER'] = 'SIM' raw_hdr['DETECTOR'] = 'SIM' raw_hdr['PRESEC1'] = '[1:4,1:2048]' raw_hdr['DATASEC1'] = '[5:2052,1:2048]' raw_hdr['BIASSEC1'] = '[2053:2102,1:2048]' raw_hdr['CCDSEC1'] = '[1:2048,1:2048]' raw_hdr['PRESEC2'] = '[4201:4204,1:2048]' raw_hdr['DATASEC2'] = '[2153:4200,1:2048]' raw_hdr['BIASSEC2'] = '[2103:2152,1:2048]' raw_hdr['CCDSEC2'] = '[2049:4096,1:2048]' raw_hdr['PRESEC3'] = '[1:4,2049:4096]' raw_hdr['DATASEC3'] = '[5:2052,2049:4096]' raw_hdr['BIASSEC3'] = '[2053:2102,2049:4096]' raw_hdr['CCDSEC3'] = '[1:2048,2049:4096]' raw_hdr['PRESEC4'] = '[4201:4204,2049:4096]' raw_hdr['DATASEC4'] = '[2153:4200,2049:4096]' raw_hdr['BIASSEC4'] = '[2103:2152,2049:4096]' raw_hdr['CCDSEC4'] = '[2049:4096,2049:4096]' raw_hdr['GAIN1'] = 1.0 raw_hdr['GAIN2'] = 1.0 raw_hdr['GAIN3'] = 1.0 raw_hdr['GAIN4'] = 1.0 raw_hdr['RDNOISE1'] = 3.0 raw_hdr['RDNOISE2'] = 3.0 raw_hdr['RDNOISE3'] = 3.0 raw_hdr['RDNOISE4'] = 3.0 data=np.zeros((4096,4204))+200. raw_data=data.astype(int) write_raw(rawfile,raw_data,raw_hdr) #- Generate fibermap file fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits') fibermap = empty_fibermap(nspec) write_fibermap(fibermapfile,fibermap)
def simulate(airmass=None, exptime=None, seeing=None, moon_frac=None, moon_sep=None, moon_alt=None, seed=1234, nspec=5000, brickname='testbrick', galsim=False, ra=None, dec=None): #- construct the simulator qsim = simulator.Simulator('desi') # Initialize random number generator to use. random_state = np.random.RandomState(seed) #- Create a blank fake fibermap for bricks fibermap = empty_fibermap(nspec) targetids = random_state.randint(2**62, size=nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 #- working out only ELG objtype = 'ELG' true_objtype = np.tile(np.array([objtype]), (nspec)) #- Initialize the output truth table. spectra = [] wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax dw = 0.2 wavelengths = np.arange(round(wavemin, 1), wavemax, dw) npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = 'ELG' * nspec truth['FLUX'] = np.zeros((nspec, npix)) truth['WAVE'] = wavelengths #- get the templates flux, tmpwave, meta1 = get_templates(wavelengths, seed=seed, nmodel=nspec) truth['FLUX'] = flux meta = vstack([meta, meta1]) #- Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) #- Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} #- Now simulate maxbin = 0 nmax = nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (nspec, nwave)) # calibrated brick flux noisyflux[camera.name] = np.empty( (nspec, nwave)) # brick flux with noise obsivar[camera.name] = np.empty( (nspec, nwave)) # inverse variance of brick flux sflux = np.empty((nspec, npix)) #- Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) spectra = truth['FLUX'] * 1.0e17 print("Simulating Spectra") for j in range(nspec): print("Simulating %s/%s spectra" % (j, nspec), end='\r') thisobjtype = 'ELG' sys.stdout.flush() #- update qsim using conditions if airmass is None: thisairmass = None else: thisairmass = airmass[j] if seeing is None: thisseeing = None else: thisseeing = seeing[j] if moon_frac is None: thismoon_frac = None else: thismoon_frac = moon_frac[j] if moon_sep is None: thismoon_sep = None else: thismoon_sep = moon_sep[j] if moon_alt is None: thismoon_alt = None else: thismoon_alt = moon_alt[j] if exptime is None: thisexptime = None else: thisexptime = exptime[j] nqsim = update_simulator(qsim, airmass=thisairmass, exptime=thisexptime, seeing=thisseeing, moon_frac=thismoon_frac, moon_sep=thismoon_sep, moon_alt=thismoon_alt, galsim=galsim) nqsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) nqsim.source.update_out() nqsim.simulate() nqsim.generate_random_noise(random_state) sflux[j][:] = 1e17 * qsim.source.flux_in.to(fluxunits).value for i, output in enumerate(nqsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) #return output obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) cframe_flux = cframe_observedflux[ j, i, :num_pixels] + cframe_rand_noise[j, i, :num_pixels] armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Output brick files if ra is None or dec is None: filename = 'brick-{}-{}.fits'.format(channel, brickname) filepath = os.path.normpath( os.path.join('{}'.format(brickname), filename)) if os.path.exists(filepath): os.remove(filepath) print('Writing {}'.format(filepath)) header = dict(BRICKNAM=brickname, CHANNEL=channel) brick = Brick(filepath, mode='update', header=header) brick.add_objects(noisyflux[channel], obsivar[channel], waves[channel], resolution[channel], fibermap, night, expid) brick.close() """ # Append truth to the file. Note: we add the resolution-convolved true # flux, not the high resolution source flux, which makes chi2 # calculations easier. header = fitsheader(header) fx = fits.open(filepath, mode='append') _add_truth(fx, header, meta, trueflux, sflux, wavelengths, channel) fx.flush() fx.close() #sys.stdout.close() """ print("Wrote file {}".format(filepath)) else: bricknames = get_bricknames(ra, dec) fibermap['BRICKNAME'] = bricknames bricknames = set(bricknames) print("No. of bricks: {}".format(len(bricknames))) print("Writing brick files") for brick_name in bricknames: thisbrick = (fibermap['BRICKNAME'] == brick_name) brickdata = fibermap[thisbrick] fibers = brickdata['FIBER'] #np.mod(brickdata['FIBER'],nspec) filename = 'brick-{}-{}.fits'.format(channel, brick_name) filepath = os.path.normpath( os.path.join('./{}'.format(brick_name), filename)) if os.path.exists(filepath): os.remove(filepath) #print('Writing {}'.format(filepath)) header = dict(BRICKNAM=brick_name, CHANNEL=channel) brick = Brick(filepath, mode='update', header=header) brick.add_objects(noisyflux[channel][fibers], obsivar[channel][fibers], waves[channel], resolution[channel][fibers], brickdata, night, expid) brick.close() print("Finished writing brick files for {} bricks".format( len(bricknames))) #- make a truth file header = fitsheader(header) make_truthfile(header, meta, trueflux, sflux, wavelengths)
def main(args=None): if args is None: args = parse() elif isinstance(args, (list, tuple)): args = parse(args) bias=True if args.bias : bias=args.bias if args.nobias : bias=False dark=True if args.dark : dark=args.dark if args.nodark : dark=False pixflat=True if args.pixflat : pixflat=args.pixflat if args.nopixflat : pixflat=False mask=True if args.mask : mask=args.mask if args.nomask : mask=False if args.cameras is None: args.cameras = [c+str(i) for c in 'brz' for i in range(10)] else: args.cameras = args.cameras.split(',') if (args.bias is not None) or (args.pixflat is not None) or (args.mask is not None) or (args.dark is not None): if len(args.cameras) > 1: raise ValueError('must use only one camera with --bias, --dark, --pixflat, --mask options') if (args.outfile is not None) and len(args.cameras) > 1: raise ValueError('must use only one camera with --outfile option') if args.outdir is None: args.outdir = os.getcwd() log.warning('--outdir not specified; using {}'.format(args.outdir)) ccd_calibration_filename = None if args.no_ccd_calib_filename : ccd_calibration_filename = False elif args.ccd_calib_filename is not None : ccd_calibration_filename = args.ccd_calib_filename if args.fibermap and not os.path.exists(fibermap): raise ValueError('--fibermap {} not found'.format(args.fibermap)) if args.fibermap is None: datadir, infile = os.path.split(os.path.abspath(args.infile)) fibermapfile = infile.replace('desi-', 'fibermap-').replace('.fits.fz', '.fits') args.fibermap = os.path.join(datadir, fibermapfile) if args.nofibermap: fibermap = None elif os.path.exists(args.fibermap): fibermap = io.read_fibermap(args.fibermap) else: log.warning('fibermap file not found; creating blank fibermap') fibermap = io.empty_fibermap(5000) for camera in args.cameras: try: img = io.read_raw(args.infile, camera, bias=bias, dark=dark, pixflat=pixflat, mask=mask, bkgsub=args.bkgsub, nocosmic=args.nocosmic, cosmics_nsig=args.cosmics_nsig, cosmics_cfudge=args.cosmics_cfudge, cosmics_c2fudge=args.cosmics_c2fudge, ccd_calibration_filename=ccd_calibration_filename, nocrosstalk=args.nocrosstalk, nogain=args.nogain, nodarktrail=args.nodarktrail, fill_header=args.fill_header, ) except IOError: log.error('Error while reading or preprocessing camera {} in {}'.format(camera, args.infile)) continue if(args.zero_masked) : img.pix *= (img.mask==0) if args.outfile is None: night = img.meta['NIGHT'] expid = img.meta['EXPID'] outfile = io.findfile('preproc', night=night, expid=expid, camera=camera, outdir=args.outdir) else: outfile = args.outfile if fibermap: petal_loc = int(img.camera[1]) ii = (fibermap['PETAL_LOC'] == petal_loc) img.fibermap = fibermap[ii] io.write_image(outfile, img) log.info("Wrote {}".format(outfile))
def setUp(cls): cls.program = program = 'dark' cls.flavor = flavor = 'bias' cls.night = night = '20150105' cls.camera = camera = 'r0' cls.expid = expid = 314 cls.psfExpid = psfExpid = 313 cls.flatExpid = flatExpid = 312 cls.templateExpid = templateExpid = 311 cls.nspec = nspec = 5 cls.exptime = exptime = 100 #- Setup environment and override default environment variables #- python 2.7 location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # py/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') if not os.path.isdir(cls.binDir): #- python 3.x setup.py test location: cls.topDir = os.path.dirname( # top-level os.path.dirname( # build/ os.path.dirname( # lib/ os.path.dirname( # desispec/ os.path.dirname(os.path.abspath(__file__)) # test/ ) ) ) ) cls.binDir = os.path.join(cls.topDir,'bin') #- last attempt if not os.path.isdir(cls.binDir): cls.topDir = os.getcwd() cls.binDir = os.path.join(cls.topDir, 'bin') if not os.path.isdir(cls.binDir): raise RuntimeError('Unable to auto-locate desispec/bin from {}'.format(__file__)) id = uuid4().hex cls.fibermapfile = 'fibermap-'+id+'.fits' cls.framefile = 'frame-'+id+'.fits' cls.testDir = testDir = os.path.join(os.environ['HOME'],'ql_test_io') datanightDir = os.path.join(testDir,night) dataDir = os.path.join(datanightDir,'{:08d}'.format(expid)) expDir = os.path.join(testDir,'exposures') expnightDir = os.path.join(expDir,night) reduxDir = os.path.join(expnightDir,'{:08d}'.format(expid)) calibDir = os.path.join(testDir, 'ql_calib') configDir = os.path.join(testDir, 'ql_config') os.environ['QL_CALIB_DIR'] = calibDir os.environ['QL_CONFIG_DIR'] = configDir if not os.path.exists(testDir): os.makedirs(testDir) os.makedirs(datanightDir) os.makedirs(dataDir) os.makedirs(expDir) os.makedirs(expnightDir) os.makedirs(reduxDir) os.makedirs(calibDir) os.makedirs(configDir) #- Write dummy configuration and input files to test merging configdict = {'name': 'Test Configuration', 'Program': program, 'Flavor': flavor, 'PSFExpid': psfExpid, 'PSFType': 'psf', 'FiberflatExpid': flatExpid, 'TemplateExpid': templateExpid, 'TemplateNight': night, 'WritePreprocfile': False, 'WriteSkyModelfile': False, 'WriteIntermediatefiles': False, 'WriteStaticPlots': False, 'Debuglevel': 20, 'UseResolution': False, 'Period': 5.0, 'Timeout': 120.0, 'Pipeline': ['Initialize','Preproc'], 'Algorithms': {'Initialize':{ 'QA':{'Check_HDUs':{'PARAMS':{}} }}, 'Preproc':{ 'QA':{'Bias_From_Overscan':{'PARAMS':{'BIAS_AMP_NORMAL_RANGE':[-100.0,100.0],'BIAS_AMP_WARN_RANGE':[-200.0,200.0]}}, 'Get_RMS':{'PARAMS':{'PERCENTILES':[68.2,95.4,99.7],'NOISE_AMP_NORMAL_RANGE':[-1.0,1.0],'NOISE_AMP_WARN_RANGE':[-2.0,2.0]}}, 'Count_Pixels':{'PARAMS':{'CUTPIX':500,'LITFRAC_NORMAL_RANGE':[-0.1,0.1],'LITFRAC_WARN_RANGE':[-0.2,0.2]}}}}} } with open('{}/test_config.yaml'.format(testDir),'w') as config: yaml.dump(configdict,config) cls.configfile = '{}/test_config.yaml'.format(testDir) #- Generate raw file rawfile = os.path.join(dataDir,'desi-00000314.fits.fz') raw_hdr = {} raw_hdr['DATE-OBS'] = '2015-01-05T08:17:03.988' raw_hdr['NIGHT'] = night raw_hdr['PROGRAM'] = program raw_hdr['FLAVOR'] = flavor raw_hdr['CAMERA'] = camera raw_hdr['EXPID'] = expid raw_hdr['EXPTIME'] = exptime raw_hdr['DOSVER'] = 'SIM' raw_hdr['FEEVER'] = 'SIM' raw_hdr['DETECTOR'] = 'SIM' raw_hdr['PRESECA'] = '[1:4,1:2048]' raw_hdr['DATASECA'] = '[5:2052,1:2048]' raw_hdr['BIASSECA'] = '[2053:2102,1:2048]' raw_hdr['CCDSECA'] = '[1:2048,1:2048]' raw_hdr['PRESECB'] = '[4201:4204,1:2048]' raw_hdr['DATASECB'] = '[2153:4200,1:2048]' raw_hdr['BIASSECB'] = '[2103:2152,1:2048]' raw_hdr['CCDSECB'] = '[2049:4096,1:2048]' raw_hdr['PRESECC'] = '[1:4,2049:4096]' raw_hdr['DATASECC'] = '[5:2052,2049:4096]' raw_hdr['BIASSECC'] = '[2053:2102,2049:4096]' raw_hdr['CCDSECC'] = '[1:2048,2049:4096]' raw_hdr['PRESECD'] = '[4201:4204,2049:4096]' raw_hdr['DATASECD'] = '[2153:4200,2049:4096]' raw_hdr['BIASSECD'] = '[2103:2152,2049:4096]' raw_hdr['CCDSECD'] = '[2049:4096,2049:4096]' raw_hdr['GAINA'] = 1.0 raw_hdr['GAINB'] = 1.0 raw_hdr['GAINC'] = 1.0 raw_hdr['GAIND'] = 1.0 raw_hdr['RDNOISEA'] = 3.0 raw_hdr['RDNOISEB'] = 3.0 raw_hdr['RDNOISEC'] = 3.0 raw_hdr['RDNOISED'] = 3.0 primary_header={'PROGRAM':program} data=np.zeros((4096,4204))+200. raw_data=data.astype(int) write_raw(rawfile,raw_data,raw_hdr,primary_header=primary_header) #- Generate fibermap file fibermapfile = os.path.join(dataDir,'fibermap-00000314.fits') fibermap = empty_fibermap(nspec) write_fibermap(fibermapfile,fibermap) #- Generate calib data for camera in ['b0', 'r0', 'z0']: #- Fiberflat has to exist but can be a dummpy file filename = '{}/fiberflat-{}.fits'.format(calibDir, camera) fx = open(filename, 'w'); fx.write('fiberflat file'); fx.close() #- PSF has to be real file psffile = '{}/psf-{}.fits'.format(calibDir, camera) example_psf = resource_filename('desispec', 'test/data/ql/psf-{}.fits'.format(camera)) shutil.copy(example_psf, psffile) #- Copy test calibration-data.yaml file specdir=calibDir+"spec/sp0" if not os.path.isdir(specdir) : os.makedirs(specdir) for c in "brz" : shutil.copy(resource_filename('desispec', 'test/data/ql/{}0.yaml'.format(c)),os.path.join(specdir,"{}0.yaml".format(c))) #- Set calibration environment variable os.environ['DESI_SPECTRO_CALIB'] = calibDir