def test_dimensions(self): #- check dimensionality mismatches with self.assertRaises(ValueError): FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec) with self.assertRaises(ValueError): FiberFlat(self.wave, self.wave, self.ivar, self.mask, self.meanspec) with self.assertRaises(ValueError): FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat) with self.assertRaises(ValueError): FiberFlat(self.wave, self.fiberflat[0:2], self.ivar, self.mask, self.meanspec) with self.assertRaises(ValueError): FiberFlat(self.fiberflat, self.fiberflat, self.ivar, self.mask, self.meanspec) with self.assertRaises(ValueError): FiberFlat(self.wave, self.fiberflat, self.wave, self.mask, self.meanspec) with self.assertRaises(ValueError): FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask[0:2, :], self.meanspec) fibers = np.arange(self.nspec) FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers) with self.assertRaises(ValueError): FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec, fibers=fibers[1:])
def test_dimensions(self): #- check dimensionality mismatches self.assertRaises( ValueError, lambda x: FiberFlat(*x), (self.wave, self.wave, self.ivar, self.mask, self.meanspec)) self.assertRaises( ValueError, lambda x: FiberFlat(*x), (self.wave, self.fiberflat, self.ivar, self.mask, self.fiberflat)) self.assertRaises(ValueError, lambda x: FiberFlat(*x), (self.wave, self.fiberflat[0:2], self.ivar, self.mask, self.meanspec))
def read_fiberflat(filename): """Read fiberflat from filename Args: filename (str): Name of fiberflat file, or (night, expid, camera) tuple Returns: FiberFlat object with attributes fiberflat, ivar, mask, meanspec, wave, header Notes: fiberflat, ivar, mask are 2D [nspec, nwave] meanspec and wave are 1D [nwave] """ #- check if outfile is (night, expid, camera) tuple instead if isinstance(filename, (tuple, list)) and len(filename) == 3: night, expid, camera = filename filename = findfile('fiberflat', night, expid, camera) header = fits.getheader(filename, 0) fiberflat = native_endian(fits.getdata(filename, 0)).astype('f8') ivar = native_endian(fits.getdata(filename, "IVAR").astype('f8')) mask = native_endian(fits.getdata(filename, "MASK", uint=True)) meanspec = native_endian(fits.getdata(filename, "MEANSPEC").astype('f8')) wave = native_endian(fits.getdata(filename, "WAVELENGTH").astype('f8')) return FiberFlat(wave, fiberflat, ivar, mask, meanspec, header=header)
def test_fiberflat_rw(self): nspec, nwave, ndiag = 10, 20, 3 flat = np.random.uniform(size=(nspec, nwave)) ivar = np.random.uniform(size=(nspec, nwave)) mask = np.zeros(shape=(nspec, nwave), dtype=int) meanspec = np.random.uniform(size=(nwave, )) wave = np.arange(nwave) ff = FiberFlat(wave, flat, ivar, mask, meanspec) desispec.io.write_fiberflat(self.testfile, ff) xff = desispec.io.read_fiberflat(self.testfile) self.assertTrue( np.all(ff.fiberflat.astype('f4').astype('f8') == xff.fiberflat)) self.assertTrue(np.all(ff.ivar.astype('f4').astype('f8') == xff.ivar)) self.assertTrue(np.all(ff.mask == xff.mask)) self.assertTrue( np.all(ff.meanspec.astype('f4').astype('f8') == xff.meanspec)) self.assertTrue(np.all(ff.wave.astype('f4').astype('f8') == xff.wave)) self.assertTrue(xff.fiberflat.dtype.isnative) self.assertTrue(xff.ivar.dtype.isnative) self.assertTrue(xff.mask.dtype.isnative) self.assertTrue(xff.meanspec.dtype.isnative) self.assertTrue(xff.wave.dtype.isnative)
def run_pa(self, frame, outputfile): from desispec.fiberflat import FiberFlat import desispec.io.fiberflat as ffIO from desispec.linalg import cholesky_solve nwave = frame.nwave nfibers = frame.nspec wave = frame.wave #- this will become part of output too flux = frame.flux sumFlux = np.zeros((nwave)) realFlux = np.zeros(flux.shape) ivar = frame.ivar * (frame.mask == 0) #deconv for fib in range(nfibers): Rf = frame.R[fib].todense() B = flux[fib] realFlux[fib] = cholesky_solve(Rf, B) sumFlux += realFlux[fib] #iflux=nfibers/sumFlux flat = np.zeros(flux.shape) flat_ivar = np.zeros(ivar.shape) avg = sumFlux / nfibers for fib in range(nfibers): Rf = frame.R[fib] # apply and reconvolute M = Rf.dot(avg) M0 = (M == 0) flat[fib] = (~M0) * flux[fib] / (M + M0) + M0 flat_ivar[fib] = ivar[fib] * M**2 fibflat = FiberFlat(frame.wave.copy(), flat, flat_ivar, frame.mask.copy(), avg) #fiberflat=compute_fiberflat(input_frame) ffIO.write_fiberflat(outputfile, fibflat, header=frame.meta) log.info("Wrote fiberflat file {}".format(outputfile))
def _write_fiberflat(self): """Write a fake fiberflat""" fiberflat = np.ones((self.nspec, self.nwave)) ivar = np.ones((self.nspec, self.nwave)) mask = np.zeros((self.nspec, self.nwave), dtype=int) meanspec = np.ones(self.nwave) ff = FiberFlat(self.wave, fiberflat, ivar, mask, meanspec) io.write_fiberflat(self.fiberflatfile, ff)
def get_fiberflat_from_frame(frame): from desispec.fiberflat import FiberFlat flux = frame.flux fiberflat = np.ones_like(flux) ffivar = 2 * np.ones_like(flux) fiberflat[0] *= 0.8 fiberflat[1] *= 1.2 ff = FiberFlat(frame.wave, fiberflat, ffivar) return ff
def setUp(self): self.nspec = 5 self.nwave = 10 self.wave = np.arange(self.nwave) self.fiberflat = np.random.uniform(size=(self.nspec, self.nwave)) self.ivar = np.ones(self.fiberflat.shape) self.mask = np.zeros(self.fiberflat.shape, dtype=np.uint32) self.meanspec = np.random.uniform(size=self.nwave) self.ff = FiberFlat(self.wave, self.fiberflat, self.ivar, self.mask, self.meanspec)
def test_apply_fiberflat_ivar(self): '''test error propagation in apply_fiberflat''' wave = np.arange(5000, 5010) nwave = len(wave) nspec = 3 flux = np.random.uniform(0.9, 1.0, size=(nspec, nwave)) ivar = np.ones_like(flux) origframe = Frame(wave, flux, ivar, spectrograph=0) fiberflat = np.ones_like(flux) ffmask = np.zeros_like(flux) fiberflat[0] *= 0.5 fiberflat[1] *= 1.5 #- ff with essentially no error ffivar = 1e20 * np.ones_like(flux) ff = FiberFlat(wave, fiberflat, ffivar) frame = copy.deepcopy(origframe) apply_fiberflat(frame, ff) self.assertTrue(np.allclose(frame.ivar, fiberflat**2)) #- ff with large error ffivar = np.ones_like(flux) ff = FiberFlat(wave, fiberflat, ffivar) frame = copy.deepcopy(origframe) apply_fiberflat(frame, ff) #- c = a/b #- (sigma_c/c)^2 = (sigma_a/a)^2 + (sigma_b/b)^2 var = frame.flux**2 * (1.0/(origframe.ivar * origframe.flux**2) + \ 1.0/(ff.ivar * ff.fiberflat**2)) self.assertTrue(np.allclose(frame.ivar, 1 / var)) #- ff.ivar=0 should result in frame.ivar=0, even if ff.fiberflat=0 too ffivar = np.ones_like(flux) ffivar[0, 0:5] = 0.0 fiberflat[0, 0:5] = 0.0 ff = FiberFlat(wave, fiberflat, ffivar) frame = copy.deepcopy(origframe) apply_fiberflat(frame, ff) self.assertTrue(np.all(frame.ivar[0, 0:5] == 0.0))
def _write_fiberflat(self, camera=None): """Write a fake fiberflat""" fiberflat = np.ones((self.nspec, self.nwave)) ivar = np.ones((self.nspec, self.nwave)) mask = np.zeros((self.nspec, self.nwave), dtype=int) meanspec = np.ones(self.nwave) ff = FiberFlat(self.wave, fiberflat, ivar, mask, meanspec) if camera is not None: hdr = fits.Header() hdr['CAMERA'] = camera else: hdr = None io.write_fiberflat(self.fiberflatfile, ff, hdr)
def test_apply_fiberflat(self): '''test apply_fiberflat interface and changes to flux and mask''' wave = np.arange(5000, 5050) nwave = len(wave) nspec = 3 flux = np.random.uniform(size=(nspec, nwave)) ivar = np.ones_like(flux) frame = Frame(wave, flux, ivar, spectrograph=0) fiberflat = np.ones_like(flux) ffivar = 2 * np.ones_like(flux) ffmask = np.zeros_like(flux) fiberflat[0] *= 0.8 fiberflat[1] *= 1.2 fiberflat[2, 0:10] = 0 #- bad fiberflat ffivar[2, 10:20] = 0 #- bad fiberflat ffmask[2, 20:30] = 1 #- bad fiberflat ff = FiberFlat(wave, fiberflat, ffivar) origframe = copy.deepcopy(frame) apply_fiberflat(frame, ff) #- was fiberflat applied? self.assertTrue(np.all(frame.flux[0] == origframe.flux[0] / 0.8)) self.assertTrue(np.all(frame.flux[1] == origframe.flux[1] / 1.2)) self.assertTrue(np.all(frame.flux[2] == origframe.flux[2])) #- did mask get set? ii = (ff.fiberflat == 0) self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0)) ii = (ff.ivar == 0) self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0)) ii = (ff.mask != 0) self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0)) #- Should fail if frame and ff don't have a common wavelength grid frame.wave = frame.wave + 0.1 with self.assertRaises(ValueError): apply_fiberflat(frame, ff)
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR = "./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory" % DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR = 'specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR = os.environ['SPECPROD'] prod_Dir = specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory" % prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap = read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx = np.where(objtype == 'STD') # match STD with STAR mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR bgsindx = np.where(objtype == 'BGS') # match BGS with LRG objtype[stdindx] = 'STAR' objtype[mwsindx] = 'STAR' objtype[bgsindx] = 'LRG' NIGHT = fibermap.meta['NIGHT'] EXPID = fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_elg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs, rmagrange=args.rmagrange_bgs, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave > 5800.0 - 1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec) % 2 == 0).astype( np.float32) continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32) for spec in range(args.nspec): flux[spec, indx] = single_line[ spec] * ref_integrated_flux / np.gradient(wavelengths)[ indx] # single line flux[spec] += continuum[ spec] * ref_cst_flux_density # flat continuum meta1 = Table( dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx] * np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line * ref_integrated_flux, CONSTFLUXDENSITY=continuum * ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj, []) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column( Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX'] gray_objects = ['gray', 'grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning( 'Maybe using an outdated simspec file with flavor={}'.format( flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header[ 'MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax = args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty( (args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty( (args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i, camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: phot = np.vstack(phot) meanspec = resample_flux(waves[channel], camwave, np.average(phot / dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec + args.nstart - 1) // 500 + 1): camera = channel + str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start = max(500 * kk, args.nstart) end = min(500 * (kk + 1), nmax) if (args.spectrograph <= kk): log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, kk, start, end)) ff = FiberFlat(waves[channel], fiberflat[start:end, :], ivar[start:end, :], mask[start:end, :], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID, camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in('Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec + args.nstart - 1) // 500 + 1): start = max(500 * ii, args.nstart) # first spectrum for a given spectrograph end = min(500 * (ii + 1), nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, ii, start, end)) num_pixels = len(waves[channel]) # Write frame file framefileName = desispec.io.findfile("frame", NIGHT, EXPID, camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar = nivar[start:end, armName[channel], :num_pixels] sh1 = frame_flux.shape[ 0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart == start): resol = resolution[channel][:sh1, :, :] else: resol = resolution[channel][-sh1:, :, :] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath = desispec.io.findfile("frame", NIGHT, EXPID, camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID, camera) cframeFlux = cframe_observedflux[ start:end, armName[channel], :num_pixels] + cframe_rand_noise[ start:end, armName[channel], :num_pixels] cframeIvar = cframe_ivar[start:end, armName[channel], :num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName, cframe) cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID, camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar = sky_ivar[start:end, armName[channel], :num_pixels] skymask = np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID, camera) flux = cframe_observedflux[start:end, armName[channel], :num_pixels] phot = nobj[start:end, armName[channel], :num_pixels] calibration = np.zeros_like(phot) jj = (flux > 0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar = 10 / cframe_ivar[start:end, armName[channel], :num_pixels] #mask=(1/calibivar>0).astype(int)?? mask = np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID, camera) log.info("Wrote file {}".format(calibfilePath))
def qproc_compute_fiberflat(qframe, niter_meanspec=4, nsig_clipping=3., spline_res_clipping=20., spline_res_flat=5.): """ Fast estimation of fiberflat """ log = get_logger() t0 = time.time() log.info("Starting...") twave = np.mean(qframe.wave, axis=0) tflux = np.zeros(qframe.flux.shape) tivar = np.zeros(qframe.flux.shape) if qframe.mask is not None: qframe.ivar *= (qframe.mask == 0) for i in range(qframe.flux.shape[0]): jj = (qframe.ivar[i] > 0) tflux[i] = np.interp(twave, qframe.wave[i, jj], qframe.flux[i, jj]) tivar[i] = np.interp(twave, qframe.wave[i, jj], qframe.ivar[i, jj], left=0, right=0) # iterative loop to a absorb constant term in fiber (should have more parameters) a = np.ones(tflux.shape[0]) for iter in range(niter_meanspec): mflux = np.median(a[:, np.newaxis] * tflux, axis=0) for i in range(qframe.flux.shape[0]): a[i] = np.median(tflux[i, mflux > 0] / mflux[mflux > 0]) # trivial fiberflat fflat = tflux / (mflux + (mflux == 0)) fivar = tivar * mflux**2 mask = np.zeros((fflat.shape), dtype='uint32') chi2 = 0 # spline fit to reject outliers and smooth the flat for fiber in range(fflat.shape[0]): # iterative spline fit max_rej_it = 5 # not more than 5 pixels at a time max_bad = 1000 nbad_tot = 0 for loop in range(20): good = (fivar[fiber] > 0) splineflat = spline_fit(twave, twave[good], fflat[fiber, good], required_resolution=spline_res_clipping, input_ivar=fivar[fiber, good], max_resolution=3 * spline_res_clipping) fchi2 = fivar[fiber] * (fflat[fiber] - splineflat)**2 bad = np.where(fchi2 > nsig_clipping**2)[0] if bad.size > 0: if bad.size > max_rej_it: # not more than 5 pixels at a time ii = np.argsort(fchi2[bad]) bad = bad[ii[-max_rej_it:]] fivar[fiber, bad] = 0 nbad_tot += len(bad) #log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber)) if nbad_tot >= max_bad: fivar[fiber, :] = 0 log.warning( "1st pass: rejecting fiber {} due to too many (new) bad pixels" .format(fiber)) else: break chi2 += np.sum(fchi2) min_ivar = 0.1 * np.median(fivar[fiber]) med_flat = np.median(fflat[fiber]) good = (fivar[fiber] > 0) splineflat = spline_fit(twave, twave[good], fflat[fiber, good], required_resolution=spline_res_flat, input_ivar=fivar[fiber, good], max_resolution=3 * spline_res_flat) fflat[fiber] = splineflat # replace by spline ii = np.where(fivar[fiber] > min_ivar)[0] if ii.size < 2: fflat[fiber] = 1 fivar[fiber] = 0 # set flat in unkown edges to median value of fiber (and ivar to 0) b = ii[0] e = ii[-1] + 1 fflat[fiber, :b] = med_flat # default fivar[fiber, :b] = 0 mask[fiber, :b] = 1 # need to change this fflat[fiber, e:] = med_flat # default fivar[fiber, e:] = 0 mask[fiber, e:] = 1 # need to change this # internal interpolation bad = (fivar[fiber][b:e] <= min_ivar) good = (fivar[fiber][b:e] > min_ivar) fflat[fiber][b:e][bad] = np.interp(twave[b:e][bad], twave[b:e][good], fflat[fiber][b:e][good]) ndata = np.sum(fivar > 0) if ndata > 0: chi2pdf = chi2 / ndata else: chi2pdf = 0 t1 = time.time() log.info(" done in {:3.1f} sec".format(t1 - t0)) # return a fiberflat object ... return FiberFlat(twave, fflat, fivar, mask, mflux, chi2pdf=chi2pdf)
def qproc_compute_fiberflat(qframe,niter_meanspec=4,nsig_clipping=3.,spline_res_clipping=20.,spline_res_flat=5.) : """ Fast estimation of fiberflat """ log = get_logger() t0=time.time() log.info("Starting...") twave=np.mean(qframe.wave,axis=0) tflux=np.zeros(qframe.flux.shape) tivar=np.zeros(qframe.flux.shape) if qframe.mask is not None : qframe.ivar *= (qframe.mask==0) for i in range(qframe.flux.shape[0]) : jj=(qframe.ivar[i]>0) tflux[i]=np.interp(twave,qframe.wave[i,jj],qframe.flux[i,jj]) tivar[i]=np.interp(twave,qframe.wave[i,jj],qframe.ivar[i,jj],left=0,right=0) # iterative loop to a absorb constant term in fiber if 1 : # simple scaling per fiber a=np.ones(tflux.shape[0]) for iter in range(niter_meanspec) : mflux=np.median(a[:,np.newaxis]*tflux,axis=0) for i in range(qframe.flux.shape[0]) : a[i] = np.median(tflux[i,mflux>0]/mflux[mflux>0]) else : # polynomial fit does not improve much and s more fragile x=np.linspace(-1,1,tflux.shape[1]) pol=np.ones(tflux.shape) for iteration in range(niter_meanspec) : if iteration>0 : for i in range(tflux.shape[0]) : jj=(mflux>0)&(tivar[i]>0) c = np.polyfit(x[jj],tflux[i,jj]/mflux[jj],1,w=mflux[jj]**2*tivar[i,jj]) pol[i] = np.poly1d(c)(x) mflux=np.median(pol*tflux,axis=0) # trivial fiberflat fflat=tflux/(mflux+(mflux==0)) fivar=tivar*mflux**2 mask=np.zeros((fflat.shape), dtype='uint32') chi2=0 # special case with test slit mask_lines = ( qframe.flux.shape[0]<50 ) if mask_lines : log.warning("Will interpolate over absorption lines in input continuum spectrum from illumination bench") # spline fit to reject outliers and smooth the flat for fiber in range(fflat.shape[0]) : # iterative spline fit max_rej_it=5# not more than 5 pixels at a time max_bad=1000 nbad_tot=0 for loop in range(20) : good=(fivar[fiber]>0) splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_clipping,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_clipping) fchi2 = fivar[fiber]*(fflat[fiber]-splineflat)**2 bad=np.where(fchi2>nsig_clipping**2)[0] if bad.size>0 : if bad.size>max_rej_it : # not more than 5 pixels at a time ii=np.argsort(fchi2[bad]) bad=bad[ii[-max_rej_it:]] fivar[fiber,bad] = 0 nbad_tot += len(bad) #log.warning("iteration {} rejecting {} pixels (tot={}) from fiber {}".format(loop,len(bad),nbad_tot,fiber)) if nbad_tot>=max_bad: fivar[fiber,:]=0 log.warning("1st pass: rejecting fiber {} due to too many (new) bad pixels".format(fiber)) else : break chi2 += np.sum(fchi2) min_ivar = 0.1*np.median(fivar[fiber]) med_flat = np.median(fflat[fiber]) good=(fivar[fiber]>0) splineflat = spline_fit(twave,twave[good],fflat[fiber,good],required_resolution=spline_res_flat,input_ivar=fivar[fiber,good],max_resolution=3*spline_res_flat) fflat[fiber] = splineflat # replace by spline ii=np.where(fivar[fiber]>min_ivar)[0] if ii.size<2 : fflat[fiber] = 1 fivar[fiber] = 0 # set flat in unknown edges to median value of fiber (and ivar to 0) b=ii[0] e=ii[-1]+1 fflat[fiber,:b]=med_flat # default fivar[fiber,:b]=0 mask[fiber,:b]=1 # need to change this fflat[fiber,e:]=med_flat # default fivar[fiber,e:]=0 mask[fiber,e:]=1 # need to change this # internal interpolation bad=(fivar[fiber][b:e]<=min_ivar) good=(fivar[fiber][b:e]>min_ivar) fflat[fiber][b:e][bad]=np.interp(twave[b:e][bad],twave[b:e][good],fflat[fiber][b:e][good]) # special case with test slit if mask_lines : if qframe.meta["camera"].upper()[0] == "B" : jj=((twave>3900)&(twave<3960))|((twave>4350)&(twave<4440))|(twave>5800) elif qframe.meta["camera"].upper()[0] == "R" : jj=(twave<5750) else : jj=(twave<7550)|(twave>9800) if np.sum(jj)>0 : njj=np.logical_not(jj) fflat[fiber,jj] = np.interp(twave[jj],twave[njj],fflat[fiber,njj]) ndata=np.sum(fivar>0) if ndata>0 : chi2pdf = chi2/ndata else : chi2pdf = 0 t1=time.time() log.info(" done in {:3.1f} sec".format(t1-t0)) # return a fiberflat object ... return FiberFlat(twave, fflat, fivar, mask, mflux,chi2pdf=chi2pdf)