def test_resolution(self, n = 100): dense = np.arange(n*n).reshape(n,n) R1 = Resolution(dense) assert scipy.sparse.isspmatrix_dia(R1),'Resolution is not recognized as a scipy.sparse.dia_matrix.' assert len(R1.offsets) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size' R2 = Resolution(R1) assert np.array_equal(R1.toarray(),R2.toarray()),'Constructor broken for dia_matrix input.' R3 = Resolution(R1.data) assert np.array_equal(R1.toarray(),R3.toarray()),'Constructor broken for array data input.' sparse = scipy.sparse.dia_matrix((R1.data[::-1],R1.offsets[::-1]),(n,n)) R4 = Resolution(sparse) assert np.array_equal(R1.toarray(),R4.toarray()),'Constructor broken for permuted offsets input.' R5 = Resolution(R1.to_fits_array()) assert np.array_equal(R1.toarray(),R5.toarray()),'to_fits_array() is broken.' #- test different sizes of input diagonals for ndiag in [3,5,11]: R6 = Resolution(np.ones((ndiag, n))) assert len(R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format(ndiag) #- An even number if diagonals is not allowed try: ndiag = 10 R7 = Resolution(np.ones((ndiag, n))) raise RuntimeError('Incorrectly created Resolution with even number of diagonals') except ValueError, err: #- it correctly raised an error, so pass pass
def test_resolution(self, n=100): dense = np.arange(n * n).reshape(n, n) R1 = Resolution(dense) assert scipy.sparse.isspmatrix_dia( R1), 'Resolution is not recognized as a scipy.sparse.dia_matrix.' assert len( R1.offsets ) == desispec.resolution.default_ndiag, 'Resolution.offsets has wrong size' R2 = Resolution(R1) assert np.array_equal( R1.toarray(), R2.toarray()), 'Constructor broken for dia_matrix input.' R3 = Resolution(R1.data) assert np.array_equal( R1.toarray(), R3.toarray()), 'Constructor broken for array data input.' sparse = scipy.sparse.dia_matrix((R1.data[::-1], R1.offsets[::-1]), (n, n)) R4 = Resolution(sparse) assert np.array_equal( R1.toarray(), R4.toarray()), 'Constructor broken for permuted offsets input.' R5 = Resolution(R1.to_fits_array()) assert np.array_equal(R1.toarray(), R5.toarray()), 'to_fits_array() is broken.' #- test different sizes of input diagonals for ndiag in [3, 5, 11]: R6 = Resolution(np.ones((ndiag, n))) assert len( R6.offsets) == ndiag, 'Constructor broken for ndiag={}'.format( ndiag) #- An even number if diagonals is not allowed try: ndiag = 10 R7 = Resolution(np.ones((ndiag, n))) raise RuntimeError( 'Incorrectly created Resolution with even number of diagonals') except ValueError as err: #- it correctly raised an error, so pass pass #- Test creation with sigmas - it should conserve flux R9 = Resolution(np.linspace(1.0, 2.0, n)) self.assertTrue(np.allclose(np.sum(R9.data, axis=0), 1.0))
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR = "./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory" % DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR = 'specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR = os.environ['SPECPROD'] prod_Dir = specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory" % prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap = read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx = np.where(objtype == 'STD') # match STD with STAR mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR bgsindx = np.where(objtype == 'BGS') # match BGS with LRG objtype[stdindx] = 'STAR' objtype[mwsindx] = 'STAR' objtype[bgsindx] = 'LRG' NIGHT = fibermap.meta['NIGHT'] EXPID = fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_elg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs, rmagrange=args.rmagrange_bgs, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave > 5800.0 - 1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec) % 2 == 0).astype( np.float32) continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32) for spec in range(args.nspec): flux[spec, indx] = single_line[ spec] * ref_integrated_flux / np.gradient(wavelengths)[ indx] # single line flux[spec] += continuum[ spec] * ref_cst_flux_density # flat continuum meta1 = Table( dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx] * np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line * ref_integrated_flux, CONSTFLUXDENSITY=continuum * ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj, []) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column( Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX'] gray_objects = ['gray', 'grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning( 'Maybe using an outdated simspec file with flavor={}'.format( flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header[ 'MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax = args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty( (args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty( (args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i, camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: phot = np.vstack(phot) meanspec = resample_flux(waves[channel], camwave, np.average(phot / dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec + args.nstart - 1) // 500 + 1): camera = channel + str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start = max(500 * kk, args.nstart) end = min(500 * (kk + 1), nmax) if (args.spectrograph <= kk): log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, kk, start, end)) ff = FiberFlat(waves[channel], fiberflat[start:end, :], ivar[start:end, :], mask[start:end, :], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID, camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in('Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec + args.nstart - 1) // 500 + 1): start = max(500 * ii, args.nstart) # first spectrum for a given spectrograph end = min(500 * (ii + 1), nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, ii, start, end)) num_pixels = len(waves[channel]) # Write frame file framefileName = desispec.io.findfile("frame", NIGHT, EXPID, camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar = nivar[start:end, armName[channel], :num_pixels] sh1 = frame_flux.shape[ 0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart == start): resol = resolution[channel][:sh1, :, :] else: resol = resolution[channel][-sh1:, :, :] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath = desispec.io.findfile("frame", NIGHT, EXPID, camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID, camera) cframeFlux = cframe_observedflux[ start:end, armName[channel], :num_pixels] + cframe_rand_noise[ start:end, armName[channel], :num_pixels] cframeIvar = cframe_ivar[start:end, armName[channel], :num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName, cframe) cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID, camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar = sky_ivar[start:end, armName[channel], :num_pixels] skymask = np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID, camera) flux = cframe_observedflux[start:end, armName[channel], :num_pixels] phot = nobj[start:end, armName[channel], :num_pixels] calibration = np.zeros_like(phot) jj = (flux > 0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar = 10 / cframe_ivar[start:end, armName[channel], :num_pixels] #mask=(1/calibivar>0).astype(int)?? mask = np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID, camera) log.info("Wrote file {}".format(calibfilePath))
def simulate(airmass=None, exptime=None, seeing=None, moon_frac=None, moon_sep=None, moon_alt=None, seed=1234, nspec=5000, brickname='testbrick', galsim=False, ra=None, dec=None): #- construct the simulator qsim = simulator.Simulator('desi') # Initialize random number generator to use. random_state = np.random.RandomState(seed) #- Create a blank fake fibermap for bricks fibermap = empty_fibermap(nspec) targetids = random_state.randint(2**62, size=nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 #- working out only ELG objtype = 'ELG' true_objtype = np.tile(np.array([objtype]), (nspec)) #- Initialize the output truth table. spectra = [] wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax dw = 0.2 wavelengths = np.arange(round(wavemin, 1), wavemax, dw) npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = 'ELG' * nspec truth['FLUX'] = np.zeros((nspec, npix)) truth['WAVE'] = wavelengths #- get the templates flux, tmpwave, meta1 = get_templates(wavelengths, seed=seed, nmodel=nspec) truth['FLUX'] = flux meta = vstack([meta, meta1]) #- Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) #- Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} #- Now simulate maxbin = 0 nmax = nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (nspec, nwave)) # calibrated brick flux noisyflux[camera.name] = np.empty( (nspec, nwave)) # brick flux with noise obsivar[camera.name] = np.empty( (nspec, nwave)) # inverse variance of brick flux sflux = np.empty((nspec, npix)) #- Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) spectra = truth['FLUX'] * 1.0e17 print("Simulating Spectra") for j in range(nspec): print("Simulating %s/%s spectra" % (j, nspec), end='\r') thisobjtype = 'ELG' sys.stdout.flush() #- update qsim using conditions if airmass is None: thisairmass = None else: thisairmass = airmass[j] if seeing is None: thisseeing = None else: thisseeing = seeing[j] if moon_frac is None: thismoon_frac = None else: thismoon_frac = moon_frac[j] if moon_sep is None: thismoon_sep = None else: thismoon_sep = moon_sep[j] if moon_alt is None: thismoon_alt = None else: thismoon_alt = moon_alt[j] if exptime is None: thisexptime = None else: thisexptime = exptime[j] nqsim = update_simulator(qsim, airmass=thisairmass, exptime=thisexptime, seeing=thisseeing, moon_frac=thismoon_frac, moon_sep=thismoon_sep, moon_alt=thismoon_alt, galsim=galsim) nqsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) nqsim.source.update_out() nqsim.simulate() nqsim.generate_random_noise(random_state) sflux[j][:] = 1e17 * qsim.source.flux_in.to(fluxunits).value for i, output in enumerate(nqsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) #return output obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) cframe_flux = cframe_observedflux[ j, i, :num_pixels] + cframe_rand_noise[j, i, :num_pixels] armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Output brick files if ra is None or dec is None: filename = 'brick-{}-{}.fits'.format(channel, brickname) filepath = os.path.normpath( os.path.join('{}'.format(brickname), filename)) if os.path.exists(filepath): os.remove(filepath) print('Writing {}'.format(filepath)) header = dict(BRICKNAM=brickname, CHANNEL=channel) brick = Brick(filepath, mode='update', header=header) brick.add_objects(noisyflux[channel], obsivar[channel], waves[channel], resolution[channel], fibermap, night, expid) brick.close() """ # Append truth to the file. Note: we add the resolution-convolved true # flux, not the high resolution source flux, which makes chi2 # calculations easier. header = fitsheader(header) fx = fits.open(filepath, mode='append') _add_truth(fx, header, meta, trueflux, sflux, wavelengths, channel) fx.flush() fx.close() #sys.stdout.close() """ print("Wrote file {}".format(filepath)) else: bricknames = get_bricknames(ra, dec) fibermap['BRICKNAME'] = bricknames bricknames = set(bricknames) print("No. of bricks: {}".format(len(bricknames))) print("Writing brick files") for brick_name in bricknames: thisbrick = (fibermap['BRICKNAME'] == brick_name) brickdata = fibermap[thisbrick] fibers = brickdata['FIBER'] #np.mod(brickdata['FIBER'],nspec) filename = 'brick-{}-{}.fits'.format(channel, brick_name) filepath = os.path.normpath( os.path.join('./{}'.format(brick_name), filename)) if os.path.exists(filepath): os.remove(filepath) #print('Writing {}'.format(filepath)) header = dict(BRICKNAM=brick_name, CHANNEL=channel) brick = Brick(filepath, mode='update', header=header) brick.add_objects(noisyflux[channel][fibers], obsivar[channel][fibers], waves[channel], resolution[channel][fibers], brickdata, night, expid) brick.close() print("Finished writing brick files for {} bricks".format( len(bricknames))) #- make a truth file header = fitsheader(header) make_truthfile(header, meta, trueflux, sflux, wavelengths)
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None, sourcetype=None, targetid=None, redshift=None, expid=0, seed=0, skyerr=0.0, ra=None, dec=None): """ Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can be used as input to the redshift fitter. Args: wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted) flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size flux has to be in units of 10^-17 ergs/s/cm2/A spectra_filename : path to output FITS file in the Spectra format program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs ignored if obsconditions is not None Optional: obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star targetid : list of targetids for each target. default of None has them generated as str(range(nspec)) redshift : list/array with each index being the redshifts for that target expid : this expid number will be saved in the Spectra fibermap seed : random seed skyerr : fractional sky subtraction error ra : numpy array with targets RA (deg) dec : numpy array with targets Dec (deg) """ log = get_logger() if len(flux.shape) == 1: flux = flux.reshape((1, flux.size)) nspec = flux.shape[0] log.info("Starting simulation of {} spectra".format(nspec)) if sourcetype is None: sourcetype = np.array(["star" for i in range(nspec)]) log.debug("sourcetype = {}".format(sourcetype)) tileid = 0 telera = 0 teledec = 0 dateobs = time.gmtime() night = desisim.obs.get_night(utc=dateobs) program = program.lower() frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec) frame_fibermap.meta["FLAVOR"] = "custom" frame_fibermap.meta["NIGHT"] = night frame_fibermap.meta["EXPID"] = expid # add DESI_TARGET tm = desitarget.targetmask.desi_mask frame_fibermap['DESI_TARGET'][sourcetype == "star"] = tm.STD_FSTAR frame_fibermap['DESI_TARGET'][sourcetype == "lrg"] = tm.LRG frame_fibermap['DESI_TARGET'][sourcetype == "elg"] = tm.ELG frame_fibermap['DESI_TARGET'][sourcetype == "qso"] = tm.QSO frame_fibermap['DESI_TARGET'][sourcetype == "sky"] = tm.SKY frame_fibermap['DESI_TARGET'][sourcetype == "bgs"] = tm.BGS_ANY if targetid is None: targetid = np.arange(nspec).astype(int) # add TARGETID frame_fibermap['TARGETID'] = targetid # spectra fibermap has two extra fields : night and expid # This would be cleaner if desispec would provide the spectra equivalent # of desispec.io.empty_fibermap() spectra_fibermap = desispec.io.empty_fibermap(nspec) spectra_fibermap = desispec.io.util.add_columns( spectra_fibermap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(night), np.int32(expid), np.int32(tileid)], ) for s in range(nspec): for tp in frame_fibermap.dtype.fields: spectra_fibermap[s][tp] = frame_fibermap[s][tp] if ra is not None: spectra_fibermap["RA_TARGET"] = ra spectra_fibermap["RA_OBS"] = ra if dec is not None: spectra_fibermap["DEC_TARGET"] = dec spectra_fibermap["DEC_OBS"] = dec if obsconditions is None: if program in ['dark', 'lrg', 'qso']: obsconditions = desisim.simexp.reference_conditions['DARK'] elif program in ['elg', 'gray', 'grey']: obsconditions = desisim.simexp.reference_conditions['GRAY'] elif program in ['mws', 'bgs', 'bright']: obsconditions = desisim.simexp.reference_conditions['BRIGHT'] else: raise ValueError('unknown program {}'.format(program)) elif isinstance(obsconditions, str): try: obsconditions = desisim.simexp.reference_conditions[ obsconditions.upper()] except KeyError: raise ValueError('obsconditions {} not in {}'.format( obsconditions.upper(), list(desisim.simexp.reference_conditions.keys()))) try: params = desimodel.io.load_desiparams() wavemin = params['ccd']['b']['wavemin'] wavemax = params['ccd']['z']['wavemax'] except KeyError: wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax if wave[0] > wavemin: log.warning( 'Minimum input wavelength {}>{}; padding with zeros'.format( wave[0], wavemin)) dwave = wave[1] - wave[0] npad = int((wave[0] - wavemin) / dwave + 1) wavepad = np.arange(npad) * dwave wavepad += wave[0] - dwave - wavepad[-1] fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wavepad, wave]) flux = np.hstack([fluxpad, flux]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wave[0] <= wavemin if wave[-1] < wavemax: log.warning( 'Maximum input wavelength {}<{}; padding with zeros'.format( wave[-1], wavemax)) dwave = wave[-1] - wave[-2] npad = int((wavemax - wave[-1]) / dwave + 1) wavepad = wave[-1] + dwave + np.arange(npad) * dwave fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wave, wavepad]) flux = np.hstack([flux, fluxpad]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wavemax <= wave[-1] ii = (wavemin <= wave) & (wave <= wavemax) flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2) wave = wave[ii] * u.Angstrom flux = flux[:, ii] * flux_unit sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap, obsconditions=obsconditions, redshift=redshift, seed=seed, psfconvolve=True) random_state = np.random.RandomState(seed) sim.generate_random_noise(random_state) scale = 1e17 specdata = None resolution = {} for camera in sim.instrument.cameras: R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) skyscale = skyerr * random_state.normal(size=sim.num_fibers) for table in sim.camera_output: wave = table['wavelength'].astype(float) flux = (table['observed_flux'] + table['random_noise_electrons'] * table['flux_calibration']).T.astype(float) if np.any(skyscale): flux += ((table['num_sky_electrons'] * skyscale) * table['flux_calibration']).T.astype(float) ivar = table['flux_inverse_variance'].T.astype(float) band = table.meta['name'].strip()[0] flux = flux * scale ivar = ivar / scale**2 mask = np.zeros(flux.shape).astype(int) spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar}, resolution_data={band: resolution[band]}, mask={band: mask}, fibermap=spectra_fibermap, meta=None, single=True) if specdata is None: specdata = spec else: specdata.update(spec) desispec.io.write_spectra(spectra_filename, specdata) log.info('Wrote ' + spectra_filename) # need to clear the simulation buffers that keeps growing otherwise # because of a different number of fibers each time ... desisim.specsim._simulators.clear() desisim.specsim._simdefaults.clear()
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None, expid=0, seed=0, survey='desi'): ''' Simulate spectra from input observer wavelength and (redshifted) flux and writes a .FITS file in the spectra format that can be used as input to the redshift fitter. Args: Wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted) Flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1] = wave.size for multiple input. Note: Flux has to be in units of 1e-17 [ergs/s/cm2/A]. spectra_filename: Path to output FITS file in the Spectra format Optional: obsconditions: Dictionary of observation conditions: {SEEING, EXPTIME, AIRMASS, MOONFRAC, MOONALT, MOONSEP} expid: This expid number will be saved in the spectra fibermap seed: Random seed ''' log = get_logger() if len(flux.shape) == 1: flux = flux.reshape((1, flux.size)) nspec = flux.shape[0] log.info("Starting simulation of {} spectra".format(nspec)) tileid = 0 telera = 0 teledec = 0 night = desisim.obs.get_night(utc=time.gmtime()) frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec) frame_fibermap.meta["FLAVOR"] = "custom" frame_fibermap.meta["NIGHT"] = night frame_fibermap.meta["EXPID"] = expid ## Add DESI_TARGET and TARGETID tm = desitarget.desi_mask ## Contains 'templates' for STD_FSTAR. for spec in range(nspec): frame_fibermap['DESI_TARGET'][spec] = tm.STD_FSTAR frame_fibermap['TARGETID'][spec] = spec ## Spectra fibermap has two extra fields: night and expid. spectra_fibermap = np.zeros(shape=(nspec, ), dtype=spectra_dtype()) for s in range(nspec): for tp in frame_fibermap.dtype.fields: spectra_fibermap[s][tp] = frame_fibermap[s][tp] spectra_fibermap[:]['EXPID'] = expid ## Needed by spectra. spectra_fibermap[:]['NIGHT'] = night ## Needed by spectra. program = program.lower() if obsconditions is None: if program in ['dark', 'lrg', 'qso']: """ E.g. reference_conditions['DARK']['SEEING'] = 1.1 reference_conditions['DARK']['EXPTIME'] = 1000 reference_conditions['DARK']['AIRMASS'] = 1.0 reference_conditions['DARK']['MOONFRAC'] = 0.0 reference_conditions['DARK']['MOONALT'] = -60 reference_conditions['DARK']['MOONSEP'] = 180 """ obsconditions = desisim.simexp.reference_conditions['DARK'] elif program in ['elg', 'gray', 'grey']: obsconditions = desisim.simexp.reference_conditions['GRAY'] elif program in ['mws', 'bgs', 'bright']: obsconditions = desisim.simexp.reference_conditions['BRIGHT'] else: raise ValueError('Unknown program {}'.format(program)) elif isinstance(obsconditions, str): try: obsconditions = desisim.simexp.reference_conditions[ obsconditions.upper()] except KeyError: raise ValueError( 'Input observation conditions {} are not in {}'.format( obsconditions.upper(), list(desisim.simexp.reference_conditions.keys()))) if survey == 'pfs': wavemin = 3796. ## [A] wavemax = 12605. ## [A] elif survey == 'desi': wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax elif survey == 'beast': wavemin = 3796. ## [A] wavemax = 12605. ## [A] else: raise ValueError("\n\nSurvey %s is not available." % survey) log.info("Setting wave limits to survey: {} ... {} to {} [A]".format( survey, wavemin, wavemax)) ii = (wavemin <= wave) & (wave <= wavemax) flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2) wave = wave[ ii] * u.Angstrom ## Dimensionful quantities; only between wavelength limits. flux = flux[:, ii] * flux_unit sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap, obsconditions=obsconditions, survey=survey) ## Add random noise. random_state = np.random.RandomState(seed) sim.generate_random_noise(random_state) specdata = None scale = 1.e17 resolution = {} ## Methods for sim object. for camera in sim.instrument.cameras: R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) for table in sim.camera_output: wave = table['wavelength'].astype(float) flux = (table['observed_flux'] + table['random_noise_electrons'] * table['flux_calibration']).T.astype(float) ivar = table['flux_inverse_variance'].T.astype(float) band = table.meta['name'].strip()[0] flux = flux * scale ivar = ivar / scale**2 mask = np.zeros(flux.shape).astype(int) ## Create spectra object for redrock. spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar}, resolution_data={band: resolution[band]}, mask={band: mask}, fibermap=spectra_fibermap, meta=None, single=True) if specdata is None: specdata = spec else: specdata.update(spec) log.info("Writing to: %s" % spectra_filename) desispec.io.write_spectra(spectra_filename, specdata) log.info('Successfully created %s.' % spectra_filename)
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None, sourcetype=None, targetid=None, redshift=None, expid=0, seed=0, skyerr=0.0, ra=None, dec=None, meta=None, fibermap_columns=None, fullsim=False,use_poisson=True): """ Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can be used as input to the redshift fitter. Args: wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted) flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size flux has to be in units of 10^-17 ergs/s/cm2/A spectra_filename : path to output FITS file in the Spectra format program : dark, lrg, qso, gray, grey, elg, bright, mws, bgs ignored if obsconditions is not None Optional: obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star targetid : list of targetids for each target. default of None has them generated as str(range(nspec)) redshift : list/array with each index being the redshifts for that target expid : this expid number will be saved in the Spectra fibermap seed : random seed skyerr : fractional sky subtraction error ra : numpy array with targets RA (deg) dec : numpy array with targets Dec (deg) meta : dictionnary, saved in primary fits header of the spectra file fibermap_columns : add these columns to the fibermap fullsim : if True, write full simulation data in extra file per camera use_poisson : if False, do not use numpy.random.poisson to simulate the Poisson noise. This is useful to get reproducible random realizations. """ log = get_logger() if len(flux.shape)==1 : flux=flux.reshape((1,flux.size)) nspec=flux.shape[0] log.info("Starting simulation of {} spectra".format(nspec)) if sourcetype is None : sourcetype = np.array(["star" for i in range(nspec)]) log.debug("sourcetype = {}".format(sourcetype)) tileid = 0 telera = 0 teledec = 0 dateobs = time.gmtime() night = desisim.obs.get_night(utc=dateobs) program = program.lower() frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec) frame_fibermap.meta["FLAVOR"]="custom" frame_fibermap.meta["NIGHT"]=night frame_fibermap.meta["EXPID"]=expid # add DESI_TARGET tm = desitarget.targetmask.desi_mask frame_fibermap['DESI_TARGET'][sourcetype=="star"]=tm.STD_FAINT frame_fibermap['DESI_TARGET'][sourcetype=="lrg"]=tm.LRG frame_fibermap['DESI_TARGET'][sourcetype=="elg"]=tm.ELG frame_fibermap['DESI_TARGET'][sourcetype=="qso"]=tm.QSO frame_fibermap['DESI_TARGET'][sourcetype=="sky"]=tm.SKY frame_fibermap['DESI_TARGET'][sourcetype=="bgs"]=tm.BGS_ANY if fibermap_columns is not None : for k in fibermap_columns.keys() : frame_fibermap[k] = fibermap_columns[k] if targetid is None: targetid = np.arange(nspec).astype(int) # add TARGETID frame_fibermap['TARGETID'] = targetid # spectra fibermap has two extra fields : night and expid # This would be cleaner if desispec would provide the spectra equivalent # of desispec.io.empty_fibermap() spectra_fibermap = desispec.io.empty_fibermap(nspec) spectra_fibermap = desispec.io.util.add_columns(spectra_fibermap, ['NIGHT', 'EXPID', 'TILEID'], [np.int32(night), np.int32(expid), np.int32(tileid)], ) for s in range(nspec): for tp in frame_fibermap.dtype.fields: spectra_fibermap[s][tp] = frame_fibermap[s][tp] if ra is not None : spectra_fibermap["TARGET_RA"] = ra spectra_fibermap["FIBER_RA"] = ra if dec is not None : spectra_fibermap["TARGET_DEC"] = dec spectra_fibermap["FIBER_DEC"] = dec if obsconditions is None: if program in ['dark', 'lrg', 'qso']: obsconditions = desisim.simexp.reference_conditions['DARK'] elif program in ['elg', 'gray', 'grey']: obsconditions = desisim.simexp.reference_conditions['GRAY'] elif program in ['mws', 'bgs', 'bright']: obsconditions = desisim.simexp.reference_conditions['BRIGHT'] else: raise ValueError('unknown program {}'.format(program)) elif isinstance(obsconditions, str): try: obsconditions = desisim.simexp.reference_conditions[obsconditions.upper()] except KeyError: raise ValueError('obsconditions {} not in {}'.format( obsconditions.upper(), list(desisim.simexp.reference_conditions.keys()))) try: params = desimodel.io.load_desiparams() wavemin = params['ccd']['b']['wavemin'] wavemax = params['ccd']['z']['wavemax'] except KeyError: wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax if wave[0] > wavemin: log.warning('Minimum input wavelength {}>{}; padding with zeros'.format( wave[0], wavemin)) dwave = wave[1] - wave[0] npad = int((wave[0] - wavemin)/dwave + 1) wavepad = np.arange(npad) * dwave wavepad += wave[0] - dwave - wavepad[-1] fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wavepad, wave]) flux = np.hstack([fluxpad, flux]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wave[0] <= wavemin if wave[-1] < wavemax: log.warning('Maximum input wavelength {}<{}; padding with zeros'.format( wave[-1], wavemax)) dwave = wave[-1] - wave[-2] npad = int( (wavemax - wave[-1])/dwave + 1 ) wavepad = wave[-1] + dwave + np.arange(npad)*dwave fluxpad = np.zeros((flux.shape[0], len(wavepad)), dtype=flux.dtype) wave = np.concatenate([wave, wavepad]) flux = np.hstack([flux, fluxpad]) assert flux.shape[1] == len(wave) assert np.allclose(dwave, np.diff(wave)) assert wavemax <= wave[-1] ii = (wavemin <= wave) & (wave <= wavemax) flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm ** 2 ) wave = wave[ii]*u.Angstrom flux = flux[:,ii]*flux_unit sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap, obsconditions=obsconditions, redshift=redshift, seed=seed, psfconvolve=True) random_state = np.random.RandomState(seed) sim.generate_random_noise(random_state,use_poisson=use_poisson) scale=1e17 specdata = None resolution={} for camera in sim.instrument.cameras: R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) skyscale = skyerr * random_state.normal(size=sim.num_fibers) if fullsim : for table in sim.camera_output : band = table.meta['name'].strip()[0] table_filename=spectra_filename.replace(".fits","-fullsim-{}.fits".format(band)) table.write(table_filename,format="fits",overwrite=True) print("wrote",table_filename) for table in sim.camera_output : wave = table['wavelength'].astype(float) flux = (table['observed_flux']+table['random_noise_electrons']*table['flux_calibration']).T.astype(float) if np.any(skyscale): flux += ((table['num_sky_electrons']*skyscale)*table['flux_calibration']).T.astype(float) ivar = table['flux_inverse_variance'].T.astype(float) band = table.meta['name'].strip()[0] flux = flux * scale ivar = ivar / scale**2 mask = np.zeros(flux.shape).astype(int) spec = Spectra([band], {band : wave}, {band : flux}, {band : ivar}, resolution_data={band : resolution[band]}, mask={band : mask}, fibermap=spectra_fibermap, meta=meta, single=True) if specdata is None : specdata = spec else : specdata.update(spec) desispec.io.write_spectra(spectra_filename, specdata) log.info('Wrote '+spectra_filename) # need to clear the simulation buffers that keeps growing otherwise # because of a different number of fibers each time ... desisim.specsim._simulators.clear() desisim.specsim._simdefaults.clear()
def sim_spectra(wave, flux, program, spectra_filename, obsconditions=None, sourcetype=None, expid=0, seed=0): """ Simulate spectra from an input set of wavelength and flux and writes a FITS file in the Spectra format that can be used as input to the redshift fitter. Args: wave : 1D np.array of wavelength in Angstrom (in vacuum) in observer frame (i.e. redshifted) flux : 1D or 2D np.array. 1D array must have same size as wave, 2D array must have shape[1]=wave.size flux has to be in units of 10^-17 ergs/s/cm2/A spectra_filename : path to output FITS file in the Spectra format Optional: obsconditions : dictionnary of observation conditions with SEEING EXPTIME AIRMASS MOONFRAC MOONALT MOONSEP sourcetype : list of string, allowed values are (sky,elg,lrg,qso,bgs,star), type of sources, used for fiber aperture loss , default is star expid : this expid number will be saved in the Spectra fibermap seed : random seed """ log = get_logger() if len(flux.shape) == 1: flux = flux.reshape((1, flux.size)) nspec = flux.shape[0] log.info("Starting simulation of {} spectra".format(nspec)) if sourcetype is None: sourcetype = np.array(["star" for i in range(nspec)]) log.debug("sourcetype = {}".format(sourcetype)) tileid = 0 telera = 0 teledec = 0 dateobs = time.gmtime() night = desisim.obs.get_night(utc=dateobs) program = program.lower() frame_fibermap = desispec.io.fibermap.empty_fibermap(nspec) frame_fibermap.meta["FLAVOR"] = "custom" frame_fibermap.meta["NIGHT"] = night frame_fibermap.meta["EXPID"] = expid # add DESI_TARGET tm = desitarget.desi_mask frame_fibermap['DESI_TARGET'][sourcetype == "star"] = tm.STD_FSTAR frame_fibermap['DESI_TARGET'][sourcetype == "lrg"] = tm.LRG frame_fibermap['DESI_TARGET'][sourcetype == "elg"] = tm.ELG frame_fibermap['DESI_TARGET'][sourcetype == "qso"] = tm.QSO frame_fibermap['DESI_TARGET'][sourcetype == "sky"] = tm.SKY frame_fibermap['DESI_TARGET'][sourcetype == "bgs"] = tm.BGS_ANY # add dummy TARGETID frame_fibermap['TARGETID'] = np.arange(nspec).astype(int) # spectra fibermap has two extra fields : night and expid spectra_fibermap = np.zeros(shape=(nspec, ), dtype=spectra_dtype()) for s in range(nspec): for tp in frame_fibermap.dtype.fields: spectra_fibermap[s][tp] = frame_fibermap[s][tp] spectra_fibermap[:]['EXPID'] = expid # needed by spectra spectra_fibermap[:]['NIGHT'] = night # needed by spectra if obsconditions is None: if program in ['dark', 'lrg', 'qso']: obsconditions = desisim.simexp.reference_conditions['DARK'] elif program in ['elg', 'gray', 'grey']: obsconditions = desisim.simexp.reference_conditions['GRAY'] elif program in ['mws', 'bgs', 'bright']: obsconditions = desisim.simexp.reference_conditions['BRIGHT'] else: raise ValueError('unknown program {}'.format(program)) elif isinstance(obsconditions, str): try: obsconditions = desisim.simexp.reference_conditions[ obsconditions.upper()] except KeyError: raise ValueError('obsconditions {} not in {}'.format( obsconditions.upper(), list(desisim.simexp.reference_conditions.keys()))) wavemin = desimodel.io.load_throughput('b').wavemin wavemax = desimodel.io.load_throughput('z').wavemax ii = (wavemin <= wave) & (wave <= wavemax) flux_unit = 1e-17 * u.erg / (u.Angstrom * u.s * u.cm**2) wave = wave[ii] * u.Angstrom flux = flux[:, ii] * flux_unit random_state = np.random.RandomState(seed) sim = desisim.simexp.simulate_spectra(wave, flux, fibermap=frame_fibermap, obsconditions=obsconditions) sim.generate_random_noise(random_state) scale = 1e17 specdata = None resolution = {} for camera in sim.instrument.cameras: R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [nspec, 1, 1]) for table in sim.camera_output: wave = table['wavelength'].astype(float) flux = (table['observed_flux'] + table['random_noise_electrons'] * table['flux_calibration']).T.astype(float) ivar = table['flux_inverse_variance'].T.astype(float) band = table.meta['name'].strip()[0] flux = flux * scale ivar = ivar / scale**2 mask = np.zeros(flux.shape).astype(int) spec = Spectra([band], {band: wave}, {band: flux}, {band: ivar}, resolution_data={band: resolution[band]}, mask={band: mask}, fibermap=spectra_fibermap, meta=None, single=True) if specdata is None: specdata = spec else: specdata.update(spec) desispec.io.write_spectra(spectra_filename, specdata) log.info('Wrote ' + spectra_filename)
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR="./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR='specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR=os.environ['SPECPROD'] prod_Dir=specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory"%prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap=read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx=np.where(objtype=='STD') # match STD with STAR mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR bgsindx=np.where(objtype=='BGS') # match BGS with LRG objtype[stdindx]='STAR' objtype[mwsindx]='STAR' objtype[bgsindx]='LRG' NIGHT=fibermap.meta['NIGHT'] EXPID=fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj =='STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave>5800.0-1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32) continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32) for spec in range(args.nspec) : flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line*ref_integrated_flux, CONSTFLUXDENSITY=continuum*ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj,[]) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX'] gray_objects = ['gray','grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax= args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax,3,maxbin)) # object photons nsky = np.zeros((nmax,3,maxbin)) # sky photons nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i,camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: phot = np.vstack(phot) meanspec = resample_flux( waves[channel], camwave, np.average(phot/dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec+args.nstart-1)//500+1): camera = channel+str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start=max(500*kk,args.nstart) end=min(500*(kk+1),nmax) if (args.spectrograph <= kk): log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end)) ff = FiberFlat( waves[channel], fiberflat[start:end,:], ivar[start:end,:], mask[start:end,:], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in( 'Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in( 'Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0] # Get results for our flux-calibrated output file. cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0] cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0] noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] + output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[j, i, :num_pixels] = 25.0 / ( output['variance_electrons'][:,0] - output['num_source_electrons'][:,0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels) armName={"b":0,"r":1,"z":2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave=np.gradient(waves[channel]) nobj[:,armName[channel],:num_pixels]/=dwave frame_rand_noise[:,armName[channel],:num_pixels]/=dwave nivar[:,armName[channel],:num_pixels]*=dwave**2 nsky[:,armName[channel],:num_pixels]/=dwave sky_rand_noise[:,armName[channel],:num_pixels]/=dwave sky_ivar[:,armName[channel],:num_pixels]/=dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec+args.nstart-1)//500+1): start=max(500*ii,args.nstart) # first spectrum for a given spectrograph end=min(500*(ii+1),nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end)) num_pixels = len(waves[channel]) # Write frame file framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar=nivar[start:end,armName[channel],:num_pixels] sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart==start): resol=resolution[channel][:sh1,:,:] else: resol=resolution[channel][-sh1:,:,:] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera) cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels] cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName,cframe) cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar=sky_ivar[start:end,armName[channel],:num_pixels] skymask=np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera) flux = cframe_observedflux[start:end,armName[channel],:num_pixels] phot = nobj[start:end,armName[channel],:num_pixels] calibration = np.zeros_like(phot) jj = (flux>0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels] #mask=(1/calibivar>0).astype(int)?? mask=np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera) log.info("Wrote file {}".format(calibfilePath))