def __init__(self, specprod_dir=None, **kwargs): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. Notes: Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ if specprod_dir is None: specprod_dir = specprod_root() self.specprod_dir = specprod_dir # Init QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs) # Load up exposures for the full production nights = get_nights(specprod_dir=self.specprod_dir) for night in nights: self.mexp_dict[night] = {} ''' for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) self.mexp_dict[night][exposure] = frames_dict ''' # Output file names self.qaexp_outroot = self.qaprod_dir+'/'+self.prod_name+'_qa' # Nights list self.qa_nights = []
def __init__(self, specprod_dir=None, qaprod_dir=None): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. qaprod_dir(str): Path containing the root path for QA output Notes: Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ # Init if specprod_dir is None: specprod_dir = specprod_root() if qaprod_dir is None: qaprod_dir = qaprod_root() # self.specprod_dir = specprod_dir self.qaprod_dir = qaprod_dir tmp = specprod_dir.split('/') self.prod_name = tmp[-1] if (len(tmp[-1]) > 0) else tmp[-2] # Exposure dict stored as [night][exposure] self.mexp_dict = {} # QA Exposure objects self.qa_exps = [] # dict to hold QA data # Data Model : key1 = Night(s); key2 = Expids self.data = {} # self.qaexp_outroot = None
def __init__(self, specprod_dir=None, **kwargs): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. Notes: Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ if specprod_dir is None: specprod_dir = specprod_root() self.specprod_dir = specprod_dir # Init QA_MultiExp.__init__(self, specprod_dir=specprod_dir, **kwargs) # Load up exposures for the full production nights = get_nights(specprod_dir=self.specprod_dir) for night in nights: self.mexp_dict[night] = {} for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) self.mexp_dict[night][exposure] = frames_dict # Output file names self.qaexp_outroot = self.qaprod_dir + '/' + self.prod_name + '_qa' # Nights list self.qa_nights = []
def main(args, comm=None): t1 = datetime.datetime.now() log = get_logger() rank = 0 nproc = 1 if comm is not None: rank = comm.rank nproc = comm.size # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) if rank == 0: log.info("starting at {}".format(time.asctime())) log.info("using raw dir {}".format(rawdir)) log.info("using spectro production dir {}".format(proddir)) # run it! pipe.run_steps(args.first, args.last, spectrographs=args.spectrographs, nightstr=args.nights, comm=comm) t2 = datetime.datetime.now() if rank == 0: if "STARTTIME" in os.environ: try: t0 = datetime.datetime.strptime(os.getenv("STARTTIME"), "%Y%m%d-%H:%M:%S") dt = t1 - t0 minutes, seconds = dt.seconds // 60, dt.seconds % 60 log.info("Python startup time: {} min {} sec".format( minutes, seconds)) except ValueError: log.error("unable to parse $STARTTIME={}".format( os.getenv("STARTTIME"))) else: log.info("python startup time unknown since $STARTTIME not set") dt = t2 - t1 minutes, seconds = dt.seconds // 60, dt.seconds % 60 log.info("Run time: {} min {} sec".format(minutes, seconds))
def setUpClass(cls): cls.nspec = 6 cls.nwave = 20 id = 1 cls.night = '20160101' cls.expid = 1 # Paths os.environ['DESI_SPECTRO_REDUX'] = os.environ['HOME'] os.environ['SPECPROD'] = 'desi_test_qa' # Files cls.testDir = specprod_root() cls.qafile_b0 = findfile('qa_data', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.qafile_b1 = findfile('qa_data', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.qafile_exp = cls.testDir + '/exposures/' + cls.night + '/{:08d}/qa-{:08d}'.format( id, id) cls.qafile_brick = cls.testDir + '/brick/3582m005/qa-3582m005.yaml' cls.flux_pdf = cls.testDir + '/exposures/' + cls.night + '/{:08d}/qa-flux-{:08d}.pdf'.format( id, id) # Files for exposure fibermap QA figure cls.frame_b0 = findfile('frame', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.frame_b1 = findfile('frame', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.fflat_b0 = findfile('fiberflat', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b0') cls.fflat_b1 = findfile('fiberflat', night=cls.night, expid=cls.expid, specprod_dir=cls.testDir, camera='b1') cls.exp_fmap_plot = cls.testDir + '/test_exp_fibermap_plot.png'
def setUpClass(cls): cls.nspec = 6 cls.nwave = 20 cls.id = 1 # Run cls.nights = ['20160101']*2 + ['20160102']*2 cls.expids = [1,2,3,4] cls.cameras = ['b0','b1'] # Files cls.files_written = [] # Paths os.environ['DESI_SPECTRO_REDUX'] = os.environ['HOME'] os.environ['SPECPROD'] = 'desi_test_qa' cls.testDir = specprod_root() # Files cls.qafile_brick = cls.testDir+'/brick/3582m005/qa-3582m005.yaml' cls.flux_pdf = cls.testDir+'/exposures/'+cls.nights[0]+'/{:08d}/qa-flux-{:08d}.pdf'.format(cls.id,cls.id) cls.frame_pdf = cls.testDir+'/exposures/'+cls.nights[0]+'/{:08d}/qa-frame-{:08d}.pdf'.format(cls.id,cls.id) # Files for exposure fibermap QA figure cls.exp_fmap_plot = cls.testDir+'/test_exp_fibermap_plot.png'
def __init__(self, specprod_dir=None): """ Class to organize and execute QA for a DESI production Args: specprod_dir(str): Path containing the exposures/ directory to use. If the value is None, then the value of :func:`specprod_root` is used instead. Notes: Attributes: qa_exps : list List of QA_Exposure classes, one per exposure in production data : dict """ if specprod_dir is None: specprod_dir = specprod_root() self.specprod_dir = specprod_dir tmp = specprod_dir.split('/') self.prod_name = tmp[-1] if (len(tmp[-1]) > 0) else tmp[-2] self.qa_exps = [] # self.data = {}
def compute_skymag(night, expid, specprod_dir=None): """ Computes the sky magnitude for a given exposure. Uses the sky model and apply a fixed calibration for which the fiber aperture loss is well understood. Args: night: int, YYYYMMDD expid: int, exposure id specprod_dir: str, optional, specify the production directory. default is $DESI_SPECTRO_REDUX/$SPECPROD returns (gmag,rmag,zmag) AB magnitudes per arcsec2, tuple with 3 float values """ log = get_logger() # AR/DK DESI spectra wavelengths wmin, wmax, wdelta = 3600, 9824, 0.8 fullwave = np.round(np.arange(wmin, wmax + wdelta, wdelta), 1) # AR (wmin,wmax) to "stitch" all three cameras wstitch = {"b": (wmin, 5790), "r": (5790, 7570), "z": (7570, 9824)} istitch = {} for camera in ["b", "r", "z"]: ii = np.where((fullwave >= wstitch[camera][0]) & (fullwave < wstitch[camera][1]))[0] istitch[camera] = (ii[0], ii[-1] + 1 ) # begin (included), end (excluded) if specprod_dir is None: specprod_dir = specprod_root() # AR looking for a petal with brz sky and ivar>0 sky_spectra = [] for spec in range(10): sky = np.zeros(fullwave.shape) ok = True for camera in ["b", "r", "z"]: camspec = "{}{}".format(camera, spec) filename = findfile("sky", night=night, expid=expid, camera=camspec, specprod_dir=specprod_dir) if not os.path.isfile(filename): log.warning("skipping {}-{:08d}-{} : missing {}".format( night, expid, spec, filename)) ok = False break fiber = 0 skyivar = fitsio.read(filename, "IVAR")[fiber] if np.all(skyivar == 0): log.warning("skipping {}-{:08d}-{} : ivar=0 for {}".format( night, expid, spec, filename)) ok = False break skyflux = fitsio.read(filename, 0)[fiber] skywave = fitsio.read(filename, "WAVELENGTH") header = fitsio.read_header(filename) exptime = header["EXPTIME"] # for now we use a fixed calibration as used in DESI-6043 for which we know what was the fiber aperture loss cal_filename = "{}/spec/fluxcalib/fluxcalibnight-{}-20201216.fits".format( os.environ["DESI_SPECTRO_CALIB"], camera) # apply the correction from fiber_acceptance_for_point_sources = 0.60 # see DESI-6043 mean_fiber_diameter_arcsec = 1.52 # see DESI-6043 fiber_area_arcsec = np.pi * (mean_fiber_diameter_arcsec / 2)**2 acal = _get_average_calibration(cal_filename) begin, end = istitch[camera] flux = np.interp(fullwave[begin:end], skywave, skyflux) acal_val = np.interp(fullwave[begin:end], acal.wave, acal.value()) sky[begin: end] = flux / exptime / acal_val * fiber_acceptance_for_point_sources / fiber_area_arcsec * 1e-17 # ergs/s/cm2/A/arcsec2 if not ok: continue # to next spectrograph sky_spectra.append(sky) if len(sky_spectra) == 0: return (99., 99., 99.) if len(sky_spectra) == 1: sky = sky_spectra[0] else: sky = np.mean(np.array(sky_spectra), axis=0) # mean over petals/spectrographs # AR integrate over the DECam grz-bands filts = _get_decam_filters() # AR zero-padding spectrum so that it covers the DECam grz passbands # AR looping through filters while waiting issue to be solved (https://github.com/desihub/speclite/issues/64) sky_pad, fullwave_pad = sky.copy(), fullwave.copy() for i in range(len(filts)): sky_pad, fullwave_pad = filts[i].pad_spectrum(sky_pad, fullwave_pad, method="zero") mags = filts.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0] return mags # AB mags for flux per arcsec2
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ import argparse parser = argparse.ArgumentParser(usage = "{prog} [options]") # parser.add_argument("-i", "--input", type=str, help="input data") # parser.add_argument("-o", "--output", type=str, help="output data") parser.add_argument("--skip-psf", action="store_true", help="Skip PSF fitting step") args = parser.parse_args() log = logging.get_logger() # YEARMMDD string, rolls over at noon not midnight if night is None: night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # create production if clobber and os.path.isdir(proddir): shutil.rmtree(proddir) dbfile = io.get_pipe_database() if not os.path.exists(dbfile): com = "desi_pipe create --db-sqlite" log.info('Running {}'.format(com)) sp.check_call(com, shell=True) else: log.info("Using pre-existing production database {}".format(dbfile)) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.prod.yaml_read(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec opts['psf']['specmin'] = 0 opts['psf']['nspec'] = nspec opts['traceshift']['nfibers'] = nspec pipe.prod.yaml_write(optpath, opts) if args.skip_psf: #- Copy desimodel psf into this production instead of fitting psf import shutil for channel in ['b', 'r', 'z']: refpsf = '{}/data/specpsf/psf-{}.fits'.format( os.getenv('DESIMODEL'), channel) nightpsf = io.findfile('psfnight', night, camera=channel+'0') shutil.copy(refpsf, nightpsf) for expid in [0,1,2]: exppsf = io.findfile('psf', night, expid, camera=channel+'0') shutil.copy(refpsf, exppsf) #- Resync database to current state dbpath = io.get_pipe_database() db = pipe.load_db(dbpath, mode="w") db.sync(night) # Run the pipeline tasks in order from desispec.pipeline.tasks.base import default_task_chain for tasktype in default_task_chain: #- if we skip psf/psfnight/traceshift, update state prior to extractions if tasktype == 'traceshift' and args.skip_psf: db.getready() run_pipeline_step(tasktype) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'TRUTH') try: elginfo = fits.getdata(simspec, 'TRUTH_ELG') except: elginfo = None from desimodel.footprint import radec2pix nside=64 pixels = np.unique(radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC'])) num_missing = 0 for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) num_missing += 1 if num_missing > 0: log.critical('{} zbest files missing'.format(num_missing)) sys.exit(1) print() print("--------------------------------------------------") print("Pixel True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) continue zfx = fits.open(zfile, memmap=False) zbest = zfx['ZBEST'].data for i in range(len(zbest['Z'])): objtype = zbest['SPECTYPE'][i] z, zwarn = zbest['Z'][i], zbest['ZWARN'][i] j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0] truetype = siminfo['OBJTYPE'][j] oiiflux = 0.0 if truetype == 'ELG': k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0] oiiflux = elginfo['OIIFLUX'][k] truez = siminfo['REDSHIFT'][j] dv = C_LIGHT*(z-truez)/(1+truez) status = None if truetype == 'SKY' and zwarn > 0: status = 'ok' elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) elif zwarn == 0: if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GALAXY': if abs(dv) < 150: status = 'ok' elif oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) else: status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux) elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype in ('STD', 'FSTD') and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format( pix, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) # YEARMMDD string, rolls over at noon not midnight # TODO: fix usage of night to be something other than today if night is None: #night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # create production # FIXME: someday run PSF estimation too... ### com = "desi_pipe --env env.txt --spectrographs 0 --fakeboot --fakepsf" rawdir = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD')) com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf --raw {}".format(rawdir) sp.check_call(com, shell=True) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.read_options(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec pipe.write_options(optpath, opts) # run the generated shell scripts # FIXME: someday run PSF estimation too... # print("Running bootcalib script...") # com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh") # sp.check_call(["bash", com]) # print("Running specex script...") # com = os.path.join(proddir, "run", "scripts", "specex_all.sh") # sp.check_call(["bash", com]) # print("Running psfcombine script...") # com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh") # sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "extract_all.sh") print("Running extraction script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "fiberflat-procexp_all.sh") print("Running calibration script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "bricks.sh") print("Running makebricks script "+com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "zfind_all.sh") print("Running zfind script "+com) sp.check_call(["bash", com]) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'METADATA') brickdirs = glob.glob(os.path.join(proddir, "bricks", "*")) bricks = [ os.path.basename(x) for x in brickdirs ] print() print("--------------------------------------------------") print("Brick True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for b in bricks: zbest = io.read_zbest(io.findfile('zbest', brickname=b)) for i in range(len(zbest.z)): if zbest.spectype[i] == 'ssp_em_galaxy': objtype = 'GAL' elif zbest.spectype[i] == 'spEigenStar': objtype = 'STAR' else: objtype = zbest.spectype[i] z, zwarn = zbest.z[i], zbest.zwarn[i] j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0] truetype = siminfo['OBJTYPE'][j] truez = siminfo['REDSHIFT'][j] dv = 3e5*(z-truez)/(1+truez) if truetype == 'SKY' and zwarn > 0: status = 'ok' elif zwarn == 0: if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype == 'STD' and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'.format( b, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ import argparse parser = argparse.ArgumentParser(usage="{prog} [options]") # parser.add_argument("-i", "--input", type=str, help="input data") # parser.add_argument("-o", "--output", type=str, help="output data") parser.add_argument("--skip-psf", action="store_true", help="Skip PSF fitting step") args = parser.parse_args() log = logging.get_logger() # YEARMMDD string, rolls over at noon not midnight if night is None: night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # create production if clobber and os.path.isdir(proddir): shutil.rmtree(proddir) dbfile = io.get_pipe_database() if not os.path.exists(dbfile): com = "desi_pipe create --db-sqlite" log.info('Running {}'.format(com)) sp.check_call(com, shell=True) else: log.info("Using pre-existing production database {}".format(dbfile)) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.prod.yaml_read(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec opts['psf']['specmin'] = 0 opts['psf']['nspec'] = nspec opts['traceshift']['nfibers'] = nspec pipe.prod.yaml_write(optpath, opts) if args.skip_psf: #- Copy desimodel psf into this production instead of fitting psf import shutil for channel in ['b', 'r', 'z']: refpsf = '{}/data/specpsf/psf-{}.fits'.format( os.getenv('DESIMODEL'), channel) nightpsf = io.findfile('psfnight', night, camera=channel + '0') shutil.copy(refpsf, nightpsf) for expid in [0, 1, 2]: exppsf = io.findfile('psf', night, expid, camera=channel + '0') shutil.copy(refpsf, exppsf) #- Resync database to current state dbpath = io.get_pipe_database() db = pipe.load_db(dbpath, mode="w") db.sync(night) # Run the pipeline tasks in order from desispec.pipeline.tasks.base import default_task_chain for tasktype in default_task_chain: #- if we skip psf/psfnight/traceshift, update state prior to extractions if tasktype == 'traceshift' and args.skip_psf: db.getready() run_pipeline_step(tasktype) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'TRUTH') try: elginfo = fits.getdata(simspec, 'TRUTH_ELG') except: elginfo = None from desimodel.footprint import radec2pix nside = 64 pixels = np.unique( radec2pix(nside, fibermap['TARGET_RA'], fibermap['TARGET_DEC'])) num_missing = 0 for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) num_missing += 1 if num_missing > 0: log.critical('{} zbest files missing'.format(num_missing)) sys.exit(1) print() print("--------------------------------------------------") print("Pixel True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for pix in pixels: zfile = io.findfile('zbest', groupname=pix) if not os.path.exists(zfile): log.error('Missing {}'.format(zfile)) continue zfx = fits.open(zfile, memmap=False) zbest = zfx['ZBEST'].data for i in range(len(zbest['Z'])): objtype = zbest['SPECTYPE'][i] z, zwarn = zbest['Z'][i], zbest['ZWARN'][i] j = np.where(fibermap['TARGETID'] == zbest['TARGETID'][i])[0][0] truetype = siminfo['OBJTYPE'][j] oiiflux = 0.0 if truetype == 'ELG': k = np.where(elginfo['TARGETID'] == zbest['TARGETID'][i])[0][0] oiiflux = elginfo['OIIFLUX'][k] truez = siminfo['REDSHIFT'][j] dv = C_LIGHT * (z - truez) / (1 + truez) status = None if truetype == 'SKY' and zwarn > 0: status = 'ok' elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) elif zwarn == 0: if truetype == 'LRG' and objtype == 'GALAXY' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GALAXY': if abs(dv) < 150: status = 'ok' elif oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) else: status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux) elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype in ('STD', 'FSTD') and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0:<8d} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'. format(pix, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def integration_test(night=None, nspec=5, clobber=False): """Run an integration test from raw data simulations through redshifts Args: night (str, optional): YEARMMDD, defaults to current night nspec (int, optional): number of spectra to include clobber (bool, optional): rerun steps even if outputs already exist Raises: RuntimeError if any script fails """ log = logging.get_logger() log.setLevel(logging.DEBUG) # YEARMMDD string, rolls over at noon not midnight # TODO: fix usage of night to be something other than today if night is None: #night = time.strftime('%Y%m%d', time.localtime(time.time()-12*3600)) night = "20160726" # check for required environment variables check_env() # simulate inputs sim(night, nspec=nspec, clobber=clobber) # create production # FIXME: someday run PSF estimation too... ### com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf" com = "desi_pipe --spectrographs 0 --fakeboot --fakepsf" sp.check_call(com, shell=True) # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) # Modify options file to restrict the spectral range optpath = os.path.join(proddir, "run", "options.yaml") opts = pipe.yaml_read(optpath) opts['extract']['specmin'] = 0 opts['extract']['nspec'] = nspec pipe.yaml_write(optpath, opts) # run the generated shell scripts # FIXME: someday run PSF estimation too... # print("Running bootcalib script...") # com = os.path.join(proddir, "run", "scripts", "bootcalib_all.sh") # sp.check_call(["bash", com]) # print("Running specex script...") # com = os.path.join(proddir, "run", "scripts", "specex_all.sh") # sp.check_call(["bash", com]) # print("Running psfcombine script...") # com = os.path.join(proddir, "run", "scripts", "psfcombine_all.sh") # sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "run_shell.sh") print("Running extraction through calibration: " + com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "spectra.sh") print("Running spectral regrouping: " + com) sp.check_call(["bash", com]) com = os.path.join(proddir, "run", "scripts", "redshift.sh") print("Running redshift script " + com) sp.check_call(["bash", com]) # #----- # #- Did it work? # #- (this combination of fibermap, simspec, and zbest is a pain) expid = 2 fmfile = io.findfile('fibermap', night=night, expid=expid) fibermap = io.read_fibermap(fmfile) simdir = os.path.dirname(fmfile) simspec = '{}/simspec-{:08d}.fits'.format(simdir, expid) siminfo = fits.getdata(simspec, 'METADATA') brickdirs = glob.glob(os.path.join(proddir, "bricks", "*")) bricks = [os.path.basename(x) for x in brickdirs] print() print("--------------------------------------------------") print("Brick True z -> Class z zwarn") # print("3338p190 SKY 0.00000 -> QSO 1.60853 12 - ok") for b in bricks: zbest = io.read_zbest(io.findfile('zbest', brickname=b)) for i in range(len(zbest.z)): if zbest.spectype[i] == 'ssp_em_galaxy': objtype = 'GAL' elif zbest.spectype[i] == 'spEigenStar': objtype = 'STAR' else: objtype = zbest.spectype[i] z, zwarn = zbest.z[i], zbest.zwarn[i] j = np.where(fibermap['TARGETID'] == zbest.targetid[i])[0][0] truetype = siminfo['OBJTYPE'][j] oiiflux = siminfo['OIIFLUX'][j] truez = siminfo['REDSHIFT'][j] dv = 3e5 * (z - truez) / (1 + truez) if truetype == 'SKY' and zwarn > 0: status = 'ok' elif truetype == 'ELG' and zwarn > 0 and oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) elif zwarn == 0: if truetype == 'LRG' and objtype == 'GAL' and abs(dv) < 150: status = 'ok' elif truetype == 'ELG' and objtype == 'GAL': if abs(dv) < 150 or oiiflux < 8e-17: status = 'ok ([OII] flux {:.2g})'.format(oiiflux) else: status = 'OOPS ([OII] flux {:.2g})'.format(oiiflux) elif truetype == 'QSO' and objtype == 'QSO' and abs(dv) < 750: status = 'ok' elif truetype in ('STD', 'FSTD') and objtype == 'STAR': status = 'ok' else: status = 'OOPS' else: status = 'OOPS' print('{0} {1:4s} {2:8.5f} -> {3:5s} {4:8.5f} {5:4d} - {6}'. format(b, truetype, truez, objtype, z, zwarn, status)) print("--------------------------------------------------")
def main(args, comm=None): t1 = datetime.datetime.now() log = get_logger() rank = 0 nproc = 1 if comm is not None: rank = comm.rank nproc = comm.size # Check start up time. if rank == 0: if "STARTTIME" in os.environ: try: t0 = datetime.datetime.strptime(os.getenv("STARTTIME"), "%Y%m%d-%H%M%S") dt = t1 - t0 minutes, seconds = dt.seconds//60, dt.seconds%60 log.info("Python startup time: {} min {} sec".format(minutes, seconds)) except ValueError: log.error("unable to parse $STARTTIME={}".format(os.getenv("STARTTIME"))) else: log.info("Python startup time unknown since $STARTTIME not set") sys.stdout.flush() # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) if rank == 0: log.info("Starting at {}".format(time.asctime())) log.info(" Using raw dir {}".format(rawdir)) log.info(" Using spectro production dir {}".format(proddir)) sys.stdout.flush() # Get task list from disk or from STDIN tasklist = None if args.taskfile is not None: # One process reads the file and broadcasts if rank == 0: tasklist = pipe.prod.task_read(args.taskfile) if comm is not None: tasklist = comm.bcast(tasklist, root=0) else: # Every process has the same STDIN contents. tasklist = list() for line in sys.stdin: tasklist.append(line.rstrip()) # Do we actually have any tasks? if len(tasklist) == 0: warnings.warn("Task list is empty", RuntimeWarning) # run it! (db, opts) = pipe.load_prod("w") ntask = len(tasklist) ready = None done = None failed = None if args.nodb: ready, done, failed = pipe.run_task_list(args.tasktype, tasklist, opts, comm=comm, db=None) else: ready, done, failed = pipe.run_task_list(args.tasktype, tasklist, opts, comm=comm, db=db) t2 = datetime.datetime.now() if rank == 0: log.info(" {} tasks already done, {} tasks were ready, and {} failed".format(done, ready, failed)) dt = t2 - t1 minutes, seconds = dt.seconds//60, dt.seconds%60 log.info("Run time: {} min {} sec".format(minutes, seconds)) sys.stdout.flush() if comm is not None: comm.barrier() # Did we have any ready tasks that were not already done? # Note: if there were no ready tasks, but some were already # done, then we want to exit with a "0" error code. This will # allow the calling script to continue with other pipeline steps # and / or allow other dependent jobs run. if done == 0: # nothing is done if ready == 0: if rank == 0: warnings.warn("No tasks were ready or done", RuntimeWarning) sys.exit(1) if (failed == ready) and (failed > 1) : # all tasks failed (and there are more than one) if rank == 0: warnings.warn("All tasks that were run failed", RuntimeWarning) sys.exit(1) else: # At least some tasks were done- we return zero so that future # jobs can run. if (ready > 0) and (failed == ready): if rank == 0: warnings.warn("All tasks that were run failed", RuntimeWarning) sys.exit(1) return
def main(args, comm=None): t1 = datetime.datetime.now() log = get_logger() rank = 0 nproc = 1 if comm is not None: rank = comm.rank nproc = comm.size # Check start up time. if rank == 0: if "STARTTIME" in os.environ: try: t0 = datetime.datetime.strptime(os.getenv("STARTTIME"), "%Y%m%d-%H%M%S") dt = t1 - t0 minutes, seconds = dt.seconds // 60, dt.seconds % 60 log.info("Python startup time: {} min {} sec".format( minutes, seconds)) except ValueError: log.error("unable to parse $STARTTIME={}".format( os.getenv("STARTTIME"))) else: log.info("Python startup time unknown since $STARTTIME not set") sys.stdout.flush() # raw and production locations rawdir = os.path.abspath(io.rawdata_root()) proddir = os.path.abspath(io.specprod_root()) if rank == 0: log.info("Starting at {}".format(time.asctime())) log.info(" Using raw dir {}".format(rawdir)) log.info(" Using spectro production dir {}".format(proddir)) sys.stdout.flush() # Get task list from disk or from STDIN tasklist = None if args.taskfile is not None: # One process reads the file and broadcasts if rank == 0: tasklist = pipe.prod.task_read(args.taskfile) if comm is not None: tasklist = comm.bcast(tasklist, root=0) else: # Every process has the same STDIN contents. tasklist = list() for line in sys.stdin: tasklist.append(line.rstrip()) # Do we actually have any tasks? if len(tasklist) == 0: warnings.warn("Task list is empty", RuntimeWarning) # run it! (db, opts) = pipe.load_prod("w") ntask = len(tasklist) ready = None done = None failed = None if args.nodb: ready, done, failed = pipe.run_task_list(args.tasktype, tasklist, opts, comm=comm, db=None) else: ready, done, failed = pipe.run_task_list(args.tasktype, tasklist, opts, comm=comm, db=db) t2 = datetime.datetime.now() if rank == 0: log.info(" {} tasks already done, {} tasks were ready, and {} failed". format(done, ready, failed)) dt = t2 - t1 minutes, seconds = dt.seconds // 60, dt.seconds % 60 log.info("Run time: {} min {} sec".format(minutes, seconds)) sys.stdout.flush() if comm is not None: comm.barrier() # Did we have any ready tasks that were not already done? # Note: if there were no ready tasks, but some were already # done, then we want to exit with a "0" error code. This will # allow the calling script to continue with other pipeline steps # and / or allow other dependent jobs run. if done == 0: # nothing is done if ready == 0: if rank == 0: warnings.warn("No tasks were ready or done", RuntimeWarning) sys.exit(1) if failed == ready: # all tasks failed if rank == 0: warnings.warn("All tasks that were run failed", RuntimeWarning) sys.exit(1) else: # At least some tasks were done- we return zero so that future # jobs can run. if (ready > 0) and (failed == ready): if rank == 0: warnings.warn("All tasks that were run failed", RuntimeWarning) sys.exit(1) return
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR="./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR=os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory"%DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR='specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR=os.environ['SPECPROD'] prod_Dir=specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory"%prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap=read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx=np.where(objtype=='STD') # match STD with STAR mwsindx=np.where(objtype=='MWS_STAR') # match MWS_STAR with STAR bgsindx=np.where(objtype=='BGS') # match BGS with LRG objtype[stdindx]='STAR' objtype[mwsindx]='STAR' objtype[bgsindx]='LRG' NIGHT=fibermap.meta['NIGHT'] EXPID=fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_elg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates(nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs,rmagrange=args.rmagrange_bgs,sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj =='STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave>5800.0-1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec)%2 == 0).astype(np.float32) continuum = (np.arange(args.nspec)%2 == 1).astype(np.float32) for spec in range(args.nspec) : flux[spec,indx] = single_line[spec]*ref_integrated_flux/np.gradient(wavelengths)[indx] # single line flux[spec] += continuum[spec]*ref_cst_flux_density # flat continuum meta1 = Table(dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx]*np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line*ref_integrated_flux, CONSTFLUXDENSITY=continuum*ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj,[]) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column(Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs','mws','bright','BGS','MWS','BRIGHT_MIX'] gray_objects = ['gray','grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning('Maybe using an outdated simspec file with flavor={}'.format(flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header['MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax= args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to(u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax,3,maxbin)) # object photons nsky = np.zeros((nmax,3,maxbin)) # sky photons nivar = np.zeros((nmax,3,maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros((nmax,3,maxbin)) # calibrated object flux cframe_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax,3,maxbin)) # inverse variance of sky sky_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros((nmax,3,maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty((args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty((args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty((args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i,camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError('Unable to find a {} camera in input simspec'.format(camera)) else: phot = np.vstack(phot) meanspec = resample_flux( waves[channel], camwave, np.average(phot/dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec+args.nstart-1)//500+1): camera = channel+str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start=max(500*kk,args.nstart) end=min(500*(kk+1),nmax) if (args.spectrograph <= kk): log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,kk,start,end)) ff = FiberFlat( waves[channel], fiberflat[start:end,:], ivar[start:end,:], mask[start:end,:], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath=desispec.io.findfile("fiberflat",NIGHT,EXPID,camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm ** 2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in( 'Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in( 'Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:,0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:,0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:,0] # Get results for our flux-calibrated output file. cframe_observedflux[j, i, :num_pixels] = 1e17 * output['observed_flux'][:,0] cframe_ivar[j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:,0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:,0] noisyflux[camera][j][:] = 1e17 * (output['observed_flux'][:,0] + output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[j, i, :num_pixels] = output['random_noise_electrons'][:,0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:,0] * output['random_noise_electrons'][:,0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[j, i, :num_pixels] = 25.0 / ( output['variance_electrons'][:,0] - output['num_source_electrons'][:,0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j,i,:num_pixels]),size=num_pixels) armName={"b":0,"r":1,"z":2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave=np.gradient(waves[channel]) nobj[:,armName[channel],:num_pixels]/=dwave frame_rand_noise[:,armName[channel],:num_pixels]/=dwave nivar[:,armName[channel],:num_pixels]*=dwave**2 nsky[:,armName[channel],:num_pixels]/=dwave sky_rand_noise[:,armName[channel],:num_pixels]/=dwave sky_ivar[:,armName[channel],:num_pixels]/=dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec+args.nstart-1)//500+1): start=max(500*ii,args.nstart) # first spectrum for a given spectrograph end=min(500*(ii+1),nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info("Writing files for channel:{}, spectrograph:{}, spectra:{} to {}".format(channel,ii,start,end)) num_pixels = len(waves[channel]) # Write frame file framefileName=desispec.io.findfile("frame",NIGHT,EXPID,camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar=nivar[start:end,armName[channel],:num_pixels] sh1=frame_flux.shape[0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart==start): resol=resolution[channel][:sh1,:,:] else: resol=resolution[channel][-sh1:,:,:] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath=desispec.io.findfile("frame",NIGHT,EXPID,camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName=desispec.io.findfile("cframe",NIGHT,EXPID,camera) cframeFlux=cframe_observedflux[start:end,armName[channel],:num_pixels]+cframe_rand_noise[start:end,armName[channel],:num_pixels] cframeIvar=cframe_ivar[start:end,armName[channel],:num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName,cframe) cframefilePath=desispec.io.findfile("cframe",NIGHT,EXPID,camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName=desispec.io.findfile("sky",NIGHT,EXPID,camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar=sky_ivar[start:end,armName[channel],:num_pixels] skymask=np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath=desispec.io.findfile("sky",NIGHT,EXPID,camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile=desispec.io.findfile("calib",NIGHT,EXPID,camera) flux = cframe_observedflux[start:end,armName[channel],:num_pixels] phot = nobj[start:end,armName[channel],:num_pixels] calibration = np.zeros_like(phot) jj = (flux>0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar=10/cframe_ivar[start:end,armName[channel],:num_pixels] #mask=(1/calibivar>0).astype(int)?? mask=np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath=desispec.io.findfile("calib",NIGHT,EXPID,camera) log.info("Wrote file {}".format(calibfilePath))
def main(args): # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files, get_nights from desispec.io import read_frame from desispec.io import get_reduced_frames from desispec.io.sky import read_sky from desispec.io import specprod_root from desispec.qa import utils as qa_utils import copy import pdb # Log log = get_logger() log.info("starting") # Path if args.reduxdir is not None: specprod_dir = args.reduxdir else: specprod_dir = specprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b', 'r', 'z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Exposure plot? if args.expid is not None: # Nights path_nights = glob.glob(specprod_dir + '/exposures/*') if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres( cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux, 1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series( channel_dict[channel], 'wave', outfile='QA_skyresid_wave_expid_{:d}{:s}.png'. format(args.expid, channel)) skysub_resid_series( channel_dict[channel], 'flux', outfile='QA_skyresid_flux_expid_{:d}{:s}.png'. format(args.expid, channel)) return # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile = 'QA/skyresid_prod_dual_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Full Prod Plot? if args.gauss: from desispec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format( len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes) # Plot log.info("Plotting..") skysub_gauss( sky_wave, sky_flux, sky_res, sky_ivar, outfile='skyresid_prod_gauss_{:s}.png'.format(channel)) return
def main(args): # Set up the logger if args.verbose: log = get_logger(DEBUG) else: log = get_logger() # Make sure all necessary environment variables are set DESI_SPECTRO_REDUX_DIR = "./quickGen" if 'DESI_SPECTRO_REDUX' not in os.environ: log.info('DESI_SPECTRO_REDUX environment is not set.') else: DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX'] if os.path.exists(DESI_SPECTRO_REDUX_DIR): if not os.path.isdir(DESI_SPECTRO_REDUX_DIR): raise RuntimeError("Path %s Not a directory" % DESI_SPECTRO_REDUX_DIR) else: try: os.makedirs(DESI_SPECTRO_REDUX_DIR) except: raise SPECPROD_DIR = 'specprod' if 'SPECPROD' not in os.environ: log.info('SPECPROD environment is not set.') else: SPECPROD_DIR = os.environ['SPECPROD'] prod_Dir = specprod_root() if os.path.exists(prod_Dir): if not os.path.isdir(prod_Dir): raise RuntimeError("Path %s Not a directory" % prod_Dir) else: try: os.makedirs(prod_Dir) except: raise # Initialize random number generator to use. np.random.seed(args.seed) random_state = np.random.RandomState(args.seed) # Derive spectrograph number from nstart if needed if args.spectrograph is None: args.spectrograph = args.nstart / 500 # Read fibermapfile to get object type, night and expid if args.fibermap: log.info("Reading fibermap file {}".format(args.fibermap)) fibermap = read_fibermap(args.fibermap) objtype = get_source_types(fibermap) stdindx = np.where(objtype == 'STD') # match STD with STAR mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR bgsindx = np.where(objtype == 'BGS') # match BGS with LRG objtype[stdindx] = 'STAR' objtype[mwsindx] = 'STAR' objtype[bgsindx] = 'LRG' NIGHT = fibermap.meta['NIGHT'] EXPID = fibermap.meta['EXPID'] else: # Create a blank fake fibermap fibermap = empty_fibermap(args.nspec) targetids = random_state.randint(2**62, size=args.nspec) fibermap['TARGETID'] = targetids night = get_night() expid = 0 log.info("Initializing SpecSim with config {}".format(args.config)) desiparams = load_desiparams() qsim = get_simulator(args.config, num_fibers=1) if args.simspec: # Read the input file log.info('Reading input file {}'.format(args.simspec)) simspec = desisim.io.read_simspec(args.simspec) nspec = simspec.nspec if simspec.flavor == 'arc': log.warning("quickgen doesn't generate flavor=arc outputs") return else: wavelengths = simspec.wave spectra = simspec.flux if nspec < args.nspec: log.info("Only {} spectra in input file".format(nspec)) args.nspec = nspec else: # Initialize the output truth table. spectra = [] wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value npix = len(wavelengths) truth = dict() meta = Table() truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10)) truth['FLUX'] = np.zeros((args.nspec, npix)) truth['WAVE'] = wavelengths jj = list() for thisobj in set(true_objtype): ii = np.where(true_objtype == thisobj)[0] nobj = len(ii) truth['OBJTYPE'][ii] = thisobj log.info('Generating {} template'.format(thisobj)) # Generate the templates if thisobj == 'ELG': elg = desisim.templates.ELG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = elg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_elg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'LRG': lrg = desisim.templates.LRG(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = lrg.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_lrg, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'QSO': qso = desisim.templates.QSO(wave=wavelengths) flux, tmpwave, meta1 = qso.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_qso) elif thisobj == 'BGS': bgs = desisim.templates.BGS(wave=wavelengths, add_SNeIa=args.add_SNeIa) flux, tmpwave, meta1 = bgs.make_templates( nmodel=nobj, seed=args.seed, zrange=args.zrange_bgs, rmagrange=args.rmagrange_bgs, sne_rfluxratiorange=args.sne_rfluxratiorange) elif thisobj == 'STD': std = desisim.templates.STD(wave=wavelengths) flux, tmpwave, meta1 = std.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'QSO_BAD': # use STAR template no color cuts star = desisim.templates.STAR(wave=wavelengths) flux, tmpwave, meta1 = star.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'MWS_STAR' or thisobj == 'MWS': mwsstar = desisim.templates.MWS_STAR(wave=wavelengths) flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'WD': wd = desisim.templates.WD(wave=wavelengths) flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj, seed=args.seed) elif thisobj == 'SKY': flux = np.zeros((nobj, npix)) meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32))) elif thisobj == 'TEST': flux = np.zeros((args.nspec, npix)) indx = np.where(wave > 5800.0 - 1E-6)[0][0] ref_integrated_flux = 1E-10 ref_cst_flux_density = 1E-17 single_line = (np.arange(args.nspec) % 2 == 0).astype( np.float32) continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32) for spec in range(args.nspec): flux[spec, indx] = single_line[ spec] * ref_integrated_flux / np.gradient(wavelengths)[ indx] # single line flux[spec] += continuum[ spec] * ref_cst_flux_density # flat continuum meta1 = Table( dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32), LINE=wave[indx] * np.ones(args.nspec, dtype=np.float32), LINEFLUX=single_line * ref_integrated_flux, CONSTFLUXDENSITY=continuum * ref_cst_flux_density)) else: log.fatal('Unknown object type {}'.format(thisobj)) sys.exit(1) # Pack it in. truth['FLUX'][ii] = flux meta = vstack([meta, meta1]) jj.append(ii.tolist()) # Sanity check on units; templates currently return ergs, not 1e-17 ergs... # assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6) # Sort the metadata table. jj = sum(jj, []) meta_new = Table() for k in range(args.nspec): index = int(np.where(np.array(jj) == k)[0]) meta_new = vstack([meta_new, meta[index]]) meta = meta_new # Add TARGETID and the true OBJTYPE to the metadata table. meta.add_column( Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE')) meta.add_column(Column(targetids, name='TARGETID')) # Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z meta.rename_column('REDSHIFT', 'TRUEZ') # explicitly set location on focal plane if needed to support airmass # variations when using specsim v0.5 if qsim.source.focal_xy is None: qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm')) # Set simulation parameters from the simspec header or desiparams bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX'] gray_objects = ['gray', 'grey'] if args.simspec is None: object_type = objtype flavor = None elif simspec.flavor == 'science': object_type = None flavor = simspec.header['PROGRAM'] else: object_type = None flavor = simspec.flavor log.warning( 'Maybe using an outdated simspec file with flavor={}'.format( flavor)) # Set airmass if args.airmass is not None: qsim.atmosphere.airmass = args.airmass elif args.simspec and 'AIRMASS' in simspec.header: qsim.atmosphere.airmass = simspec.header['AIRMASS'] else: qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2 # Set exptime if args.exptime is not None: qsim.observation.exposure_time = args.exptime * u.s elif args.simspec and 'EXPTIME' in simspec.header: qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s elif objtype in bright_objects: qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s else: qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s # Set Moon Phase if args.moon_phase is not None: qsim.atmosphere.moon.moon_phase = args.moon_phase elif args.simspec and 'MOONFRAC' in simspec.header: qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC'] elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_phase = 0.7 elif flavor in gray_objects: qsim.atmosphere.moon.moon_phase = 0.1 else: qsim.atmosphere.moon.moon_phase = 0.5 # Set Moon Zenith if args.moon_zenith is not None: qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg elif args.simspec and 'MOONALT' in simspec.header: qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.moon_zenith = 30 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.moon_zenith = 80 * u.deg else: qsim.atmosphere.moon.moon_zenith = 100 * u.deg # Set Moon - Object Angle if args.moon_angle is not None: qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg elif args.simspec and 'MOONSEP' in simspec.header: qsim.atmosphere.moon.separation_angle = simspec.header[ 'MOONSEP'] * u.deg elif flavor in bright_objects or object_type in bright_objects: qsim.atmosphere.moon.separation_angle = 50 * u.deg elif flavor in gray_objects: qsim.atmosphere.moon.separation_angle = 60 * u.deg else: qsim.atmosphere.moon.separation_angle = 60 * u.deg # Initialize per-camera output arrays that will be saved waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {} maxbin = 0 nmax = args.nspec for camera in qsim.instrument.cameras: # Lookup this camera's resolution matrix and convert to the sparse # format used in desispec. R = Resolution(camera.get_output_resolution_matrix()) resolution[camera.name] = np.tile(R.to_fits_array(), [args.nspec, 1, 1]) waves[camera.name] = (camera.output_wavelength.to( u.Angstrom).value.astype(np.float32)) nwave = len(waves[camera.name]) maxbin = max(maxbin, len(waves[camera.name])) nobj = np.zeros((nmax, 3, maxbin)) # object photons nsky = np.zeros((nmax, 3, maxbin)) # sky photons nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky) cframe_observedflux = np.zeros( (nmax, 3, maxbin)) # calibrated object flux cframe_ivar = np.zeros( (nmax, 3, maxbin)) # inverse variance of calibrated object flux cframe_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to calibrated flux sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky sky_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to sky only frame_rand_noise = np.zeros( (nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky trueflux[camera.name] = np.empty( (args.nspec, nwave)) # calibrated flux noisyflux[camera.name] = np.empty( (args.nspec, nwave)) # observed flux with noise obsivar[camera.name] = np.empty( (args.nspec, nwave)) # inverse variance of flux if args.simspec: for i in range(10): cn = camera.name + str(i) if cn in simspec.cameras: dw = np.gradient(simspec.cameras[cn].wave) break else: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: sflux = np.empty((args.nspec, npix)) #- Check if input simspec is for a continuum flat lamp instead of science #- This does not convolve to per-fiber resolution if args.simspec: if simspec.flavor == 'flat': log.info("Simulating flat lamp exposure") for i, camera in enumerate(qsim.instrument.cameras): channel = camera.name #- from simspec, b/r/z not b0/r1/z9 assert camera.output_wavelength.unit == u.Angstrom num_pixels = len(waves[channel]) phot = list() for j in range(10): cn = camera.name + str(j) if cn in simspec.cameras: camwave = simspec.cameras[cn].wave dw = np.gradient(camwave) phot.append(simspec.cameras[cn].phot) if len(phot) == 0: raise RuntimeError( 'Unable to find a {} camera in input simspec'.format( camera)) else: phot = np.vstack(phot) meanspec = resample_flux(waves[channel], camwave, np.average(phot / dw, axis=0)) fiberflat = random_state.normal(loc=1.0, scale=1.0 / np.sqrt(meanspec), size=(nspec, num_pixels)) ivar = np.tile(meanspec, [nspec, 1]) mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32) for kk in range((args.nspec + args.nstart - 1) // 500 + 1): camera = channel + str(kk) outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID, camera) start = max(500 * kk, args.nstart) end = min(500 * (kk + 1), nmax) if (args.spectrograph <= kk): log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, kk, start, end)) ff = FiberFlat(waves[channel], fiberflat[start:end, :], ivar[start:end, :], mask[start:end, :], meanspec, header=dict(CAMERA=camera)) write_fiberflat(outfile, ff) filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID, camera) log.info("Wrote file {}".format(filePath)) sys.exit(0) # Repeat the simulation for all spectra fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom) for j in range(args.nspec): thisobjtype = objtype[j] sys.stdout.flush() if flavor == 'arc': qsim.source.update_in('Quickgen source {0}'.format, 'perfect', wavelengths * u.Angstrom, spectra * fluxunits) else: qsim.source.update_in('Quickgen source {0}'.format(j), thisobjtype.lower(), wavelengths * u.Angstrom, spectra[j, :] * fluxunits) qsim.source.update_out() qsim.simulate() qsim.generate_random_noise(random_state) for i, output in enumerate(qsim.camera_output): assert output['observed_flux'].unit == 1e17 * fluxunits # Extract the simulation results needed to create our uncalibrated # frame output file. num_pixels = len(output) nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0] nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0] nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0] # Get results for our flux-calibrated output file. cframe_observedflux[ j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0] cframe_ivar[ j, i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0] # Fill brick arrays from the results. camera = output.meta['name'] trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0] noisyflux[camera][j][:] = 1e17 * ( output['observed_flux'][:, 0] + output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:, 0] # Use the same noise realization in the cframe and frame, without any # additional noise from sky subtraction for now. frame_rand_noise[ j, i, :num_pixels] = output['random_noise_electrons'][:, 0] cframe_rand_noise[j, i, :num_pixels] = 1e17 * ( output['flux_calibration'][:, 0] * output['random_noise_electrons'][:, 0]) # The sky output file represents a model fit to ~40 sky fibers. # We reduce the variance by a factor of 25 to account for this and # give the sky an independent (Gaussian) noise realization. sky_ivar[ j, i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] - output['num_source_electrons'][:, 0]) sky_rand_noise[j, i, :num_pixels] = random_state.normal( scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]), size=num_pixels) armName = {"b": 0, "r": 1, "z": 2} for channel in 'brz': #Before writing, convert from counts/bin to counts/A (as in Pixsim output) #Quicksim Default: #FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang] #COUNTS_OBJ - object counts in 0.5 Ang bin #COUNTS_SKY - sky counts in 0.5 Ang bin num_pixels = len(waves[channel]) dwave = np.gradient(waves[channel]) nobj[:, armName[channel], :num_pixels] /= dwave frame_rand_noise[:, armName[channel], :num_pixels] /= dwave nivar[:, armName[channel], :num_pixels] *= dwave**2 nsky[:, armName[channel], :num_pixels] /= dwave sky_rand_noise[:, armName[channel], :num_pixels] /= dwave sky_ivar[:, armName[channel], :num_pixels] /= dwave**2 # Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra # Looping over spectrograph for ii in range((args.nspec + args.nstart - 1) // 500 + 1): start = max(500 * ii, args.nstart) # first spectrum for a given spectrograph end = min(500 * (ii + 1), nmax) # last spectrum for the spectrograph if (args.spectrograph <= ii): camera = "{}{}".format(channel, ii) log.info( "Writing files for channel:{}, spectrograph:{}, spectra:{} to {}" .format(channel, ii, start, end)) num_pixels = len(waves[channel]) # Write frame file framefileName = desispec.io.findfile("frame", NIGHT, EXPID, camera) frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \ nsky[start:end,armName[channel],:num_pixels] + \ frame_rand_noise[start:end,armName[channel],:num_pixels] frame_ivar = nivar[start:end, armName[channel], :num_pixels] sh1 = frame_flux.shape[ 0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave) # for example if nstart =400, nspec=150: two spectrographs: # 400-499=> 0 spectrograph, 500-549 => 1 if (args.nstart == start): resol = resolution[channel][:sh1, :, :] else: resol = resolution[channel][-sh1:, :, :] # must create desispec.Frame object frame=Frame(waves[channel], frame_flux, frame_ivar,\ resolution_data=resol, spectrograph=ii, \ fibermap=fibermap[start:end], \ meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.write_frame(framefileName, frame) framefilePath = desispec.io.findfile("frame", NIGHT, EXPID, camera) log.info("Wrote file {}".format(framefilePath)) if args.frameonly or simspec.flavor == 'arc': continue # Write cframe file cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID, camera) cframeFlux = cframe_observedflux[ start:end, armName[channel], :num_pixels] + cframe_rand_noise[ start:end, armName[channel], :num_pixels] cframeIvar = cframe_ivar[start:end, armName[channel], :num_pixels] # must create desispec.Frame object cframe = Frame(waves[channel], cframeFlux, cframeIvar, \ resolution_data=resol, spectrograph=ii, fibermap=fibermap[start:end], meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) ) desispec.io.frame.write_frame(cframeFileName, cframe) cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID, camera) log.info("Wrote file {}".format(cframefilePath)) # Write sky file skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera) skyflux=nsky[start:end,armName[channel],:num_pixels] + \ sky_rand_noise[start:end,armName[channel],:num_pixels] skyivar = sky_ivar[start:end, armName[channel], :num_pixels] skymask = np.zeros(skyflux.shape, dtype=np.uint32) # must create desispec.Sky object skymodel = SkyModel(waves[channel], skyflux, skyivar, skymask, header=dict(CAMERA=camera)) desispec.io.sky.write_sky(skyfileName, skymodel) skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera) log.info("Wrote file {}".format(skyfilePath)) # Write calib file calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID, camera) flux = cframe_observedflux[start:end, armName[channel], :num_pixels] phot = nobj[start:end, armName[channel], :num_pixels] calibration = np.zeros_like(phot) jj = (flux > 0) calibration[jj] = phot[jj] / flux[jj] #- TODO: what should calibivar be? #- For now, model it as the noise of combining ~10 spectra calibivar = 10 / cframe_ivar[start:end, armName[channel], :num_pixels] #mask=(1/calibivar>0).astype(int)?? mask = np.zeros(calibration.shape, dtype=np.uint32) # write flux calibration fluxcalib = FluxCalib(waves[channel], calibration, calibivar, mask) write_flux_calibration(calibVectorFile, fluxcalib) calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID, camera) log.info("Wrote file {}".format(calibfilePath))
def main(args) : # imports import glob from desispec.io import findfile, makepath from desispec.io import get_exposures from desispec.io import get_files, get_nights from desispec.io import get_reduced_frames from desispec.io import specprod_root from desispec.io import qaprod_root from desispec.qa import utils as qa_utils import copy import pdb # Init specprod_dir = specprod_root() # Log log=get_logger() log.info("starting") # Path if args.qaprod_dir is not None: qaprod_dir = args.qaprod_dir else: qaprod_dir = qaprod_root() # Channels if args.channels is not None: channels = [iarg for iarg in args.channels.split(',')] else: channels = ['b','r','z'] # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Nights if args.nights is not None: nights = [iarg for iarg in args.nights.split(',')] else: nights = None # Exposure plot? if args.expid is not None: # Nights if nights is None: nights = get_nights() nights.sort() # Find the exposure for night in nights: if args.expid in get_exposures(night, specprod_dir=specprod_dir): frames_dict = get_files(filetype=str('cframe'), night=night, expid=args.expid, specprod_dir=specprod_dir) # Loop on channel #for channel in ['b','r','z']: for channel in ['z']: channel_dict[channel]['cameras'] = [] for camera, cframe_fil in frames_dict.items(): if channel in camera: sky_file = findfile(str('sky'), night=night, camera=camera, expid=args.expid, specprod_dir=specprod_dir) wave, flux, res, _ = qa_utils.get_skyres(cframe_fil) # Append channel_dict[channel]['wave'].append(wave) channel_dict[channel]['skyflux'].append(np.log10(np.maximum(flux,1e-1))) channel_dict[channel]['res'].append(res) channel_dict[channel]['cameras'].append(camera) channel_dict[channel]['count'] += 1 if channel_dict[channel]['count'] > 0: from desispec.qa.qa_plots import skysub_resid_series # Hidden to help with debugging skysub_resid_series(channel_dict[channel], 'wave', outfile=qaprod_dir+'/QA_skyresid_wave_expid_{:d}{:s}.png'.format(args.expid, channel)) skysub_resid_series(channel_dict[channel], 'flux', outfile=qaprod_dir+'/QA_skyresid_flux_expid_{:d}{:s}.png'.format(args.expid, channel)) return # Skyline if args.skyline: from desispec.qa.qa_plots import skyline_resid # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) if len(cframes) == 1: log.error('len(cframes)==1; starting debugging') pdb.set_trace() # Need to call differently else: sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres( cframes, flatten=False) # Plot outfile=args.outdir+'/skyline_{:s}.png'.format(channel) log.info("Plotting to {:s}".format(outfile)) skyline_resid(channel, sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, _ = qa_utils.get_skyres(cframes) # Plot outfile=qaprod_dir+'/skyresid_prod_dual_{:s}.png'.format(channel) makepath(outfile) log.info("Plotting to {:s}".format(outfile)) skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile=outfile) return # Test sky noise for Gaussianity if args.gauss: from desispec.qa.qa_plots import skysub_gauss # Loop on channel for channel in channels: cframes = get_reduced_frames(nights=nights, channels=[channel]) if len(cframes) > 0: # Cut down for debugging #cframes = [cframes[ii] for ii in range(15)] # log.info("Loading sky residuals for {:d} cframes".format(len(cframes))) sky_wave, sky_flux, sky_res, sky_ivar = qa_utils.get_skyres(cframes) # Plot log.info("Plotting..") outfile=qaprod_dir+'/skyresid_prod_gauss_{:s}.png'.format(channel) makepath(outfile) skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar, outfile=outfile) return