def bgs_write_simdata(sim, obs_conds, simdir, obsrand, overwrite=False): """Build and write a metadata table with the simulation inputs. Currently, the only quantities that can be varied are moonfrac, moonsep, and exptime, but more choices can be added as needed. """ from desispec.io.util import makepath simdatafile = os.path.join(simdir, sim['suffix'], 'bgs-{}-simdata.fits'.format(sim['suffix'])) makepath(simdatafile) cols = [('SEED', 'S20'), ('NSPEC', 'i4'), ('EXPTIME', 'f4'), ('AIRMASS', 'f4'), ('SEEING', 'f4'), ('MOONFRAC', 'f4'), ('MOONSEP', 'f4'), ('MOONALT', 'f4')] simdata = Table(np.zeros(sim['nsim'], dtype=cols)) simdata['EXPTIME'].unit = 's' simdata['SEEING'].unit = 'arcsec' simdata['MOONSEP'].unit = 'deg' simdata['MOONALT'].unit = 'deg' simdata['SEED'] = sim['seed'] simdata['NSPEC'] = sim['nspec'] simdata['AIRMASS'] = obs_conds['AIRMASS'] simdata['SEEING'] = obs_conds['SEEING'] simdata['MOONALT'] = obs_conds['MOONALT'] if 'moonfracmin' in obs_conds.keys(): simdata['MOONFRAC'] = obsrand.uniform(obs_conds['moonfracmin'], obs_conds['moonfracmax'], sim['nsim']) else: simdata['MOONFRAC'] = obs_conds['MOONFRAC'] if 'moonsepmin' in obs_conds.keys(): simdata['MOONSEP'] = obsrand.uniform(obs_conds['moonsepmin'], obs_conds['moonsepmax'], sim['nsim']) else: simdata['MOONSEP'] = obs_conds['MOONSEP'] if 'exptimemin' in obs_conds.keys(): simdata['EXPTIME'] = obsrand.uniform(obs_conds['exptimemin'], obs_conds['exptimemax'], sim['nsim']) else: simdata['EXPTIME'] = obs_conds['EXPTIME'] if overwrite or not os.path.isfile(simdatafile): print('Writing {}'.format(simdatafile)) write_bintable(simdatafile, simdata, extname='SIMDATA', clobber=overwrite) return simdata
def write_fiberflat(outfile, fiberflat, header=None): """Write fiberflat object to outfile Args: outfile: filepath string or (night, expid, camera) tuple fiberflat: FiberFlat object header: (optional) dict or fits.Header object to use as HDU 0 header Returns: filepath of file that was written """ outfile = makepath(outfile, 'fiberflat') if header is None: hdr = fitsheader(fiberflat.header) else: hdr = fitsheader(header) if fiberflat.chi2pdf is not None: hdr['chi2pdf'] = float(fiberflat.chi2pdf) add_dependencies(hdr) ff = fiberflat #- shorthand hdus = fits.HDUList() hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr)) hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR')) hdus.append(fits.CompImageHDU(ff.mask, name='MASK')) hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC')) hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH')) hdus.writeto(outfile + '.tmp', clobber=True, checksum=True) os.rename(outfile + '.tmp', outfile) return outfile
def write_templates(outfile, flux, wave, meta, objtype=None, comments=None, units=None): """Write out simulated galaxy templates. (Incomplete documentation...) Args: outfile (str): Output file name. """ from astropy.io import fits from desispec.io.util import fitsheader, write_bintable, makepath # Create the path to OUTFILE if necessary. outfile = makepath(outfile) header = dict(OBJTYPE=(objtype, 'Object type'), CUNIT=('Angstrom', 'units of wavelength array'), CRPIX1=(1, 'reference pixel number'), CRVAL1=(wave[0], 'Starting wavelength [Angstrom]'), CDELT1=(wave[1] - wave[0], 'Wavelength step [Angstrom]'), LOGLAM=(0, 'linear wavelength steps, not log10'), AIRORVAC=('vac', 'wavelengths in vacuum (vac) or air'), BUNIT=('1e-17 erg/(s cm2 Angstrom)', 'spectrum flux units')) hdr = fitsheader(header) fits.writeto(outfile, flux.astype(np.float32), header=hdr, clobber=True) write_bintable(outfile, meta, header=hdr, comments=comments, units=units, extname='METADATA')
def write_qa_frame(outfile, qaframe, verbose=False): """Write QA for a given frame Args: outfile : str filename qa_exp : QA_Frame object, with the following attributes qa_data: dict of QA info """ log = get_logger() outfile = makepath(outfile, 'qa') # Generate the dict odict = { qaframe.night: { qaframe.expid: { qaframe.camera: {}, 'flavor': qaframe.flavor } } } odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data ydict = yamlify(odict) # Simple yaml with open(outfile, 'w') as yamlf: yamlf.write(yaml.dump(ydict)) if verbose: log.info("Wrote QA frame file: {:s}".format(outfile)) return outfile
def write_qa_prod(outroot, qaprod, indent=True): """Write QA for a given production Args: outroot : str filename without format extension qa_prod : QA_Prod object Returns: outfile: str output filename """ from desiutil.io import combine_dicts log = get_logger() outfile = outroot + '.json' outfile = makepath(outfile, 'qa') # Loop on exposures odict = {} for qaexp in qaprod.qa_exps: # Get the exposure dict idict = write_qa_exposure('foo', qaexp, ret_dict=True) odict = combine_dicts(odict, idict) ydict = yamlify(odict) # This works well for JSON too # Simple json with open(outfile, 'wt') as fh: json.dump(ydict, fh, indent=indent) log.info('Wrote QA_Prod file: {:s}'.format(outfile)) return outfile
def write_qa_frame(outfile, qaframe): """Write QA for a given frame Args: outfile : str filename qa_exp : QA_Frame object, with the following attributes qa_data: dict of QA info """ outfile = makepath(outfile, 'qa') # Generate the dict odict = { qaframe.night: { qaframe.expid: { qaframe.camera: {}, 'flavor': qaframe.flavor } } } odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data ydict = yamlify(odict) # Simple yaml with open(outfile, 'w') as yamlf: yamlf.write(yaml.dump(ydict)) #, default_flow_style=True) ) return outfile
def write_qa_multiexp(outroot, qa_mexp, indent=True, skip_rebuild=False): """Write QA for a given production Args: outroot : str filename without format extension qa_prod : QA_Prod object skip_rebuild : bool, optional Do not rebuild the data dict Returns: outfile: str output filename """ log = get_logger() outfile = outroot + '.json' outfile = makepath(outfile, 'qa') if not skip_rebuild: qa_mexp.build_data() ydict = yamlify(qa_mexp.data) # This works well for JSON too # Simple json with open(outfile, 'wt') as fh: json.dump(ydict, fh, indent=indent) log.info('Wrote QA_Prod file: {:s}'.format(outfile)) return outfile
def write_fibermap(outfile, fibermap, header=None, clobber=True, extname='FIBERMAP'): """Write fibermap binary table to outfile. Args: outfile (str): output filename fibermap: astropy Table of fibermap data header: header data to include in same HDU as fibermap clobber (bool, optional): overwrite outfile if it exists extname (str, optional): set the extension name. Returns: write_fibermap (str): full path to filename of fibermap file written. """ outfile = makepath(outfile) #- astropy.io.fits incorrectly generates warning about 2D arrays of strings #- Temporarily turn off warnings to avoid this; desispec.test.test_io will #- catch it if the arrays actually are written incorrectly. if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(fibermap.meta) add_dependencies(hdr) with warnings.catch_warnings(): warnings.simplefilter("ignore") write_bintable(outfile, fibermap, hdr, comments=fibermap_comments, extname=extname, clobber=clobber) return outfile
def write_fibermap(outfile, fibermap, header=None): """Write fibermap binary table to outfile. Args: outfile (str): output filename fibermap: astropy Table of fibermap data header: header data to include in same HDU as fibermap Returns: write_fibermap (str): full path to filename of fibermap file written. """ outfile = makepath(outfile) #- astropy.io.fits incorrectly generates warning about 2D arrays of strings #- Temporarily turn off warnings to avoid this; desispec.test.test_io will #- catch it if the arrays actually are written incorrectly. if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(fibermap.meta) add_dependencies(hdr) with warnings.catch_warnings(): warnings.simplefilter("ignore") write_bintable(outfile, fibermap, hdr, comments=fibermap_comments, extname="FIBERMAP", clobber=True) return outfile
def write_fiberflat(outfile,fiberflat,header=None): """Write fiberflat object to outfile Args: outfile: filepath string or (night, expid, camera) tuple fiberflat: FiberFlat object header: (optional) dict or fits.Header object to use as HDU 0 header Returns: filepath of file that was written """ outfile = makepath(outfile, 'fiberflat') if header is None: hdr = fitsheader(fiberflat.header) else: hdr = fitsheader(header) if fiberflat.chi2pdf is not None: hdr['chi2pdf'] = float(fiberflat.chi2pdf) add_dependencies(hdr) ff = fiberflat #- shorthand hdus = fits.HDUList() hdus.append(fits.PrimaryHDU(ff.fiberflat.astype('f4'), header=hdr)) hdus.append(fits.ImageHDU(ff.ivar.astype('f4'), name='IVAR')) hdus.append(fits.CompImageHDU(ff.mask, name='MASK')) hdus.append(fits.ImageHDU(ff.meanspec.astype('f4'), name='MEANSPEC')) hdus.append(fits.ImageHDU(ff.wave.astype('f4'), name='WAVELENGTH')) hdus.writeto(outfile+'.tmp', clobber=True, checksum=True) os.rename(outfile+'.tmp', outfile) return outfile
def write_sky(outfile, skymodel, header=None): """Write sky model. Args: outfile : filename or (night, expid, camera) tuple skymodel : SkyModel object, with the following attributes wave : 1D wavelength in vacuum Angstroms flux : 2D[nspec, nwave] sky flux ivar : 2D inverse variance of sky flux mask : 2D mask for sky flux header : optional fits header data (fits.Header, dict, or list) """ outfile = makepath(outfile, 'sky') #- Convert header to fits.Header if needed if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(skymodel.header) add_dependencies(hdr) hx = fits.HDUList() hdr['EXTNAME'] = ('SKY', 'no dimension') hx.append( fits.PrimaryHDU(skymodel.flux.astype('f4'), header=hdr) ) hx.append( fits.ImageHDU(skymodel.ivar.astype('f4'), name='IVAR') ) hx.append( fits.CompImageHDU(skymodel.mask, name='MASK') ) hx.append( fits.ImageHDU(skymodel.wave.astype('f4'), name='WAVELENGTH') ) hx.writeto(outfile+'.tmp', clobber=True, checksum=True) os.rename(outfile+'.tmp', outfile) return outfile
def write_fibermap(outfile, fibermap, header=None): """Write fibermap binary table to outfile. Args: outfile (str): output filename fibermap: ndarray with named columns of fibermap data header: header data to include in same HDU as fibermap Returns: write_fibermap (str): full path to filename of fibermap file written. """ outfile = makepath(outfile) #- astropy.io.fits incorrectly generates warning about 2D arrays of strings #- Temporarily turn off warnings to avoid this; desispec.test.test_io will #- catch it if the arrays actually are written incorrectly. hdr = fitsheader(header) add_dependencies(hdr) with warnings.catch_warnings(): warnings.simplefilter("ignore") write_bintable(outfile, fibermap, hdr, comments=fibermap_comments, extname="FIBERMAP", clobber=True) return outfile
def write_qa_prod(outroot, qaprod): """Write QA for a given production Args: outroot : str filename without format extension qa_prod : QA_Prod object Returns: outfile or odict : str or dict """ from desiutil.io import combine_dicts outfile = outroot+'.yaml' outfile = makepath(outfile, 'qa') # Loop on exposures odict = {} for qaexp in qaprod.qa_exps: # Get the exposure dict idict = write_qa_exposure('foo', qaexp, ret_dict=True) odict = combine_dicts(odict, idict) ydict = yamlify(odict) # Simple yaml with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) log.info('Wrote QA_Prod file: {:s}'.format(outfile)) return outfile
def write_fiberflat(outfile, fiberflat, header=None): """Write fiberflat object to outfile Args: outfile: filepath string or (night, expid, camera) tuple header: (optional) dict or fits.Header object to use as HDU 0 header Returns: filepath of file that was written """ outfile = makepath(outfile, 'fiberflat') if header is None: hdr = fitsheader(fiberflat.header) else: hdr = fitsheader(header) ff = fiberflat #- shorthand hdus = fits.HDUList() hdus.append(fits.PrimaryHDU(ff.fiberflat, header=hdr)) hdus.append(fits.ImageHDU(ff.ivar, name='IVAR')) hdus.append(fits.ImageHDU(ff.mask, name='MASK')) hdus.append(fits.ImageHDU(ff.meanspec, name='MEANSPEC')) hdus.append(fits.ImageHDU(ff.wave, name='WAVELENGTH')) hdus.writeto(outfile, clobber=True) return outfile
def write_qa_exposure(outroot, qaexp, ret_dict=False): """Write QA for a given exposure Args: outroot : str filename without format extension qa_exp : QA_Exposure object ret_dict : bool, optional Return dict only? [for qa_prod, mainly] Returns: outfile or odict : str or dict """ # Generate the dict odict = {qaexp.night: {qaexp.expid: {}}} odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor odict[qaexp.night][qaexp.expid]['meta'] = qaexp.meta cameras = list(qaexp.data['frames'].keys()) for camera in cameras: odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera] # Return dict only? if ret_dict: return odict # Simple yaml ydict = yamlify(odict) outfile = outroot+'.yaml' outfile = makepath(outfile, 'qa') with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) return outfile
def write_qa_exposure(outroot, qaexp, ret_dict=False): """Write QA for a given exposure Args: outroot : str filename without format extension qa_exp : QA_Exposure object ret_dict : bool, optional Return dict only? [for qa_prod, mainly] Returns: outfile or odict : str or dict """ # Generate the dict odict = {qaexp.night: {qaexp.expid: {}}} odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor cameras = qaexp.data['frames'].keys() for camera in cameras: odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera] # Return dict only? if ret_dict: return odict # Simple yaml ydict = yamlify(odict) outfile = outroot+'.yaml' outfile = makepath(outfile, 'qa') with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) return outfile
def write_templates(outfile, flux, wave, meta, objtype=None, comments=None, units=None): """Write out simulated galaxy templates. (Incomplete documentation...) Args: outfile (str): Output file name. Returns: Raises """ from astropy.io import fits from desispec.io.util import fitsheader, write_bintable, makepath # Create the path to OUTFILE if necessary. outfile = makepath(outfile) header = dict( OBJTYPE = (objtype, 'Object type'), CUNIT = ('Angstrom', 'units of wavelength array'), CRPIX1 = (1, 'reference pixel number'), CRVAL1 = (wave[0], 'Starting wavelength [Angstrom]'), CDELT1 = (wave[1]-wave[0], 'Wavelength step [Angstrom]'), LOGLAM = (0, 'linear wavelength steps, not log10'), AIRORVAC = ('vac', 'wavelengths in vacuum (vac) or air'), BUNIT = ('erg/s/cm2/A', 'spectrum flux units') ) hdr = fitsheader(header) fits.writeto(outfile,flux.astype(np.float32),header=hdr,clobber=True) write_bintable(outfile, meta, header=hdr, comments=comments, units=units, extname='METADATA')
def bgs_write_simdata(sim, overwrite=False): """Create a metadata table with simulation inputs. Parameters ---------- sim : dict Simulation parameters from command line. overwrite : bool Overwrite simulation data file. Returns ------- simdata : Table Data table written to disk. """ from desispec.io.util import makepath from desispec.io.util import write_bintable simdatafile = os.path.join(sim.simdir, 'bgs_{}_simdata.fits'.format(sim.simid)) makepath(simdatafile) cols = [('SEED', 'S20'), ('NSPEC', 'i4'), ('EXPTIME', 'f4'), ('AIRMASS', 'f4'), ('SEEING', 'f4'), ('MOONFRAC', 'f4'), ('MOONSEP', 'f4'), ('MOONALT', 'f4')] simdata = Table(np.zeros(sim.nsim, dtype=cols)) simdata['EXPTIME'].unit = 's' simdata['SEEING'].unit = 'arcsec' simdata['MOONSEP'].unit = 'deg' simdata['MOONALT'].unit = 'deg' simdata['SEED'] = sim.seed simdata['NSPEC'] = sim.nspec simdata['AIRMASS'] = sim.airmass simdata['SEEING'] = sim.seeing simdata['MOONALT'] = sim.moonalt simdata['MOONSEP'] = sim.moonsep simdata['MOONFRAC'] = sim.moonfrac simdata['EXPTIME'] = sim.exptime if overwrite or not os.path.isfile(simdatafile): print('Writing {}'.format(simdatafile)) write_bintable(simdatafile, simdata, extname='SIMDATA', clobber=True) return simdata
def calib(qaprod_dir=None, specprod_dir=None): """ Generate HTML to orgainze calib HTML """ # Organized HTML html_file = meta.findfile('qa_calib_html', qaprod_dir=qaprod_dir) html_path, _ = os.path.split(html_file) makepath(html_file) # Open f = open(html_file, 'w') init(f, 'Calibration QA') # Loop on Nights nights = get_nights(sub_folder='calibnight', specprod_dir=specprod_dir) nights.sort() links = '' body = '' for night in nights: all_png = glob.glob(html_path + '/' + night + '/qa*.png') if len(all_png) == 0: continue # Find expid expids = [] for png in all_png: expids.append(int(png[-12:-4])) # A bit risky expids = np.unique(expids) expids.sort() f.write('<h2> Night -- {:s} </h2>\n'.format(night)) f.write('<h3><ul>\n') for expid in expids: # Link f.write( '<li><a href="{:s}/qa-{:08d}.html">Exposure {:08d}</a></li>\n'. format(night, expid, expid)) # Generate Exposure html calib_exp(night, expid, qaprod_dir=qaprod_dir) f.write('</ul></h3>\n') # Finish finish(f, body) # Return return links, body
def write_frame(outfile, frame, header=None, fibermap=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) frame: desispec.frame.Frame object with wave, flux, ivar... Optional: header: astropy.io.fits.Header or dict to override frame.header fibermap: table to store as FIBERMAP HDU Returns: full filepath of output file that was written Note: to create a Frame object to pass into write_frame, frame = Frame(wave, flux, ivar, resolution_data) """ outfile = makepath(outfile, 'frame') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(frame.meta) add_dependencies(hdr) hdus = fits.HDUList() x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr) x.header['EXTNAME'] = 'FLUX' hdus.append(x) hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR')) hdus.append(fits.CompImageHDU(frame.mask, name='MASK')) hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH')) hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION')) if fibermap is not None: hdus.append(fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP')) elif frame.fibermap is not None: hdus.append( fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP')) elif frame.spectrograph is not None: x.header[ 'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame) else: log.error( "You are likely writing a frame without sufficient fiber info") hdus.writeto(outfile + '.tmp', clobber=True, checksum=True) os.rename(outfile + '.tmp', outfile) return outfile
def write_frame(outfile, frame, header=None, fibermap=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) frame: desispec.frame.Frame object with wave, flux, ivar... Optional: header: astropy.io.fits.Header or dict to override frame.header fibermap: table to store as FIBERMAP HDU Returns: full filepath of output file that was written Note: to create a Frame object to pass into write_frame, frame = Frame(wave, flux, ivar, resolution_data) """ outfile = makepath(outfile, 'frame') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(frame.meta) add_dependencies(hdr) hdus = fits.HDUList() x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr) x.header['EXTNAME'] = 'FLUX' hdus.append(x) hdus.append( fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR') ) hdus.append( fits.CompImageHDU(frame.mask, name='MASK') ) hdus.append( fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH') ) hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION' ) ) if fibermap is not None: hdus.append( fits.BinTableHDU(np.asarray(fibermap), name='FIBERMAP' ) ) elif frame.fibermap is not None: hdus.append( fits.BinTableHDU(np.asarray(frame.fibermap), name='FIBERMAP' ) ) elif frame.spectrograph is not None: x.header['FIBERMIN'] = 500*frame.spectrograph # Hard-coded (as in desispec.frame) else: log.error("You are likely writing a frame without sufficient fiber info") if frame.chi2pix is not None: hdus.append( fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX' ) ) hdus.writeto(outfile+'.tmp', clobber=True, checksum=True) os.rename(outfile+'.tmp', outfile) return outfile
def write_qa_brick(outfile, qabrick): """Write QA for a given exposure Args: outfile : filename qabrick : QA_Brick object _data: dict of QA info """ outfile = makepath(outfile, 'qa') # Simple yaml ydict = yamlify(qabrick.data) with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) return outfile
def write_qa_brick(outfile, qabrick): """Write QA for a given exposure Args: outfile : filename qabrick : QA_Brick object _data: dict of QA info """ outfile = makepath(outfile, 'qa') # Simple yaml ydict = yamlify(qabrick.data) with open(outfile, 'w') as yamlf: yamlf.write(yaml.dump(ydict))#, default_flow_style=True) ) return outfile
def write_fibermap(outfile, fibermap, header=None, clobber=True, extname='FIBERMAP'): """Write fibermap binary table to outfile. Args: outfile (str): output filename fibermap: astropy Table of fibermap data header: header data to include in same HDU as fibermap clobber (bool, optional): overwrite outfile if it exists extname (str, optional): set the extension name. Returns: write_fibermap (str): full path to filename of fibermap file written. """ log = get_logger() outfile = makepath(outfile) #- astropy.io.fits incorrectly generates warning about 2D arrays of strings #- Temporarily turn off warnings to avoid this; desispec.test.test_io will #- catch it if the arrays actually are written incorrectly. if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(fibermap.meta) add_dependencies(hdr) with warnings.catch_warnings(): warnings.simplefilter("ignore") t0 = time.time() write_bintable(outfile, fibermap, hdr, comments=fibermap_comments, extname=extname, clobber=clobber) duration = time.time() - t0 log.info(iotime.format('write', outfile, duration)) return outfile
def write_qa_frame(outfile, qaframe): """Write QA for a given frame Args: outfile : str filename qa_exp : QA_Frame object, with the following attributes qa_data: dict of QA info """ outfile = makepath(outfile, 'qa') # Generate the dict odict = {qaframe.night: {qaframe.expid: {qaframe.camera: {}, 'flavor': qaframe.flavor}}} odict[qaframe.night][qaframe.expid][qaframe.camera] = qaframe.qa_data ydict = yamlify(odict) # Simple yaml with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) return outfile
def write_frame(outfile, frame, header=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) frame: desispec.frame.Frame object with wave, flux, ivar... header: optional astropy.io.fits.Header or dict to override frame.header Returns: full filepath of output file that was written Note: to create a Frame object to pass into write_frame, frame = Frame(wave, flux, ivar, resolution_data) """ outfile = makepath(outfile, 'frame') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(frame.header) if 'SPECMIN' not in hdr: hdr['SPECMIN'] = 0 if 'SPECMAX' not in hdr: hdr['SPECMAX'] = hdr['SPECMIN'] + frame.nspec hdus = fits.HDUList() x = fits.PrimaryHDU(frame.flux, header=hdr) x.header['EXTNAME'] = 'FLUX' hdus.append(x) hdus.append( fits.ImageHDU(frame.ivar, name='IVAR') ) hdus.append( fits.ImageHDU(frame.mask, name='MASK') ) hdus.append( fits.ImageHDU(frame.wave, name='WAVELENGTH') ) hdus.append( fits.ImageHDU(frame.resolution_data, name='RESOLUTION' ) ) hdus.writeto(outfile, clobber=True) return outfile
def write_qa_multiexp(outroot, mdict, indent=True): """Write QA for a given production Args: outroot : str filename without format extension mdict : dict Returns: outfile: str output filename """ log=get_logger() outfile = outroot+'.json' outfile = makepath(outfile, 'qa') ydict = yamlify(mdict) # This works well for JSON too # Simple json with open(outfile, 'wt') as fh: json.dump(ydict, fh, indent=indent) log.info('Wrote QA Multi-Exposure file: {:s}'.format(outfile)) return outfile
def write_qa_multiexp(outroot, mdict, indent=True): """Write QA for a given production Args: outroot : str filename without format extension mdict : dict Returns: outfile: str output filename """ log = get_logger() outfile = outroot + '.json' outfile = makepath(outfile, 'qa') ydict = yamlify(mdict) # This works well for JSON too # Simple json with open(outfile, 'wt') as fh: json.dump(ydict, fh, indent=indent) log.info('Wrote QA Multi-Exposure file: {:s}'.format(outfile)) return outfile
def write_templates(outfile, flux, wave, meta): """Write out simulated galaxy templates. Args: outfile (str): Output file name. flux (numpy.ndarray): Flux vector (1e-17 erg/s/cm2/A) wave (numpy.ndarray): Wavelength vector (Angstrom). meta (astropy.table.Table): metadata table. """ from astropy.io import fits from desispec.io.util import makepath # Create the path to OUTFILE if necessary. outfile = makepath(outfile) hx = fits.HDUList() hdu_wave = fits.PrimaryHDU(wave) hdu_wave.header['EXTNAME'] = 'WAVE' hdu_wave.header['BUNIT'] = 'Angstrom' hdu_wave.header['AIRORVAC'] = ('vac', 'Vacuum wavelengths') hx.append(hdu_wave) hdu_flux = fits.ImageHDU(flux) hdu_flux.header['EXTNAME'] = 'FLUX' hdu_flux.header['BUNIT'] = str(fluxunits) hx.append(hdu_flux) hdu_meta = fits.table_to_hdu(meta) hdu_meta.header['EXTNAME'] = 'METADATA' hx.append(hdu_meta) log.info('Writing {}'.format(outfile)) try: hx.writeto(outfile, overwrite=True) except: hx.writeto(outfile, clobber=True)
def write_sky(outfile, skymodel, header=None): """Write sky model. Args: outfile : filename or (night, expid, camera) tuple skymodel : SkyModel object, with the following attributes wave : 1D wavelength in vacuum Angstroms flux : 2D[nspec, nwave] sky flux ivar : 2D inverse variance of sky flux mask : 2D mask for sky flux header : optional fits header data (fits.Header, dict, or list) """ outfile = makepath(outfile, 'sky') #- Convert header to fits.Header if needed if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(skymodel.header) hdr['EXTNAME'] = ('SKY', 'no dimension') fits.writeto(outfile, skymodel.flux, header=hdr, clobber=True) hdr['EXTNAME'] = ('IVAR', 'no dimension') hdu = fits.ImageHDU(skymodel.ivar, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) hdr['EXTNAME'] = ('MASK', 'no dimension') hdu = fits.ImageHDU(skymodel.mask, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]') hdu = fits.ImageHDU(skymodel.wave, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) return outfile
def write_sky(outfile, skymodel, header=None): """Write sky model. Args: outfile : filename or (night, expid, camera) tuple skymodel : SkyModel object, with the following attributes wave : 1D wavelength in vacuum Angstroms flux : 2D[nspec, nwave] sky flux ivar : 2D inverse variance of sky flux mask : 2D mask for sky flux header : optional fits header data (fits.Header, dict, or list) """ outfile = makepath(outfile, 'sky') #- Convert header to fits.Header if needed if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(skymodel.header) hdr['EXTNAME'] = ('SKY', 'no dimension') fits.writeto(outfile, skymodel.flux,header=hdr, clobber=True) hdr['EXTNAME'] = ('IVAR', 'no dimension') hdu = fits.ImageHDU(skymodel.ivar, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) hdr['EXTNAME'] = ('MASK', 'no dimension') hdu = fits.ImageHDU(skymodel.mask, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) hdr['EXTNAME'] = ('WAVELENGTH', '[Angstroms]') hdu = fits.ImageHDU(skymodel.wave, header=hdr) fits.append(outfile, hdu.data, header=hdu.header) return outfile
def main(): """Runs the process """ # Check environmental variables are set assert "DESI_SPECTRO_DATA" in os.environ, "Missing $DESI_SPECTRO_DATA environment variable" assert "PRODNAME" in os.environ, "Missing $PRODNAME environment variable" assert "DESI_SPECTRO_REDUX" in os.environ, "Missing $DESI_SPECTRO_REDUX environment variable" # Grab nights in data redux nights = glob.glob(dio_meta.specprod_root() + "/exposures/*") if len(nights) == 0: raise ValueError("No nights in exposures!") # Build up list of fibermap files fibermap_files = [] for night in nights: onight = night[night.rfind("/") :] files = glob.glob(dio_meta.data_root() + "/" + onight + "/fibermap*") # fibermap_files += files # Get list of zbest files zbest_files = glob.glob(dio_meta.specprod_root() + "/bricks/*/zbest*") if len(zbest_files) == 0: raise ValueError("No redshifts?!") # Meta meta = get_meta() # Load+write table simtab_fil = dio_meta.specprod_root() + "/QA/sim_z_table.fits" dio_util.makepath(simtab_fil) simz_tab = dsqa_z.load_z(fibermap_files, zbest_files) simz_tab.meta = meta simz_tab.write(simtab_fil, overwrite=True) # Summary stats summ_file = dio_meta.specprod_root() + "/QA/sim_z_summ.yaml" dio_util.makepath(summ_file) summ_dict = dsqa_z.summ_stats(simz_tab) # Write with open(summ_file, "w") as outfile: outfile.write(yaml.dump(meta)) # , default_flow_style=True) ) outfile.write(yaml.dump(summ_dict, default_flow_style=False)) # import pdb # pdb.set_trace() """ # Reorder + cut summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL'] # Write summ_tab.meta = meta summ_tab.write(summ_file,format='ascii.ecsv', formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True) """ # QA Figures fig_file = dio_meta.specprod_root() + "/QA/sim_z.pdf" dio_util.makepath(fig_file) pp = PdfPages(fig_file) # Summ dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp) for objtype in ["ELG", "LRG", "QSO_T", "QSO_L"]: dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp) # All done pp.close()
def write_frame(outfile, frame, header=None, fibermap=None, units=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) frame: desispec.frame.Frame object with wave, flux, ivar... Optional: header: astropy.io.fits.Header or dict to override frame.header fibermap: table to store as FIBERMAP HDU Returns: full filepath of output file that was written Note: to create a Frame object to pass into write_frame, frame = Frame(wave, flux, ivar, resolution_data) """ outfile = makepath(outfile, 'frame') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(frame.meta) add_dependencies(hdr) hdus = fits.HDUList() x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr) x.header['EXTNAME'] = 'FLUX' if units is not None: units = str(units) if 'BUNIT' in hdr and hdr['BUNIT'] != units: log.warn('BUNIT {bunit} != units {units}; using {units}'.format( bunit=hdr['BUNIT'], units=units)) x.header['BUNIT'] = units hdus.append(x) hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR')) hdus.append(fits.CompImageHDU(frame.mask, name='MASK')) hdus.append(fits.ImageHDU(frame.wave.astype('f4'), name='WAVELENGTH')) hdus[-1].header['BUNIT'] = 'Angstrom' hdus.append( fits.ImageHDU(frame.resolution_data.astype('f4'), name='RESOLUTION')) if fibermap is not None: fibermap = desiutil.io.encode_table(fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append(fits.convenience.table_to_hdu(fibermap)) elif frame.fibermap is not None: fibermap = desiutil.io.encode_table( frame.fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append(fits.convenience.table_to_hdu(fibermap)) elif frame.spectrograph is not None: x.header[ 'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame) else: log.error( "You are likely writing a frame without sufficient fiber info") if frame.chi2pix is not None: hdus.append(fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX')) hdus.writeto(outfile + '.tmp', clobber=True, checksum=True) os.rename(outfile + '.tmp', outfile) return outfile
def main(): '''Runs the process ''' # Check environmental variables are set assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable' assert 'SPECPROD' in os.environ, 'Missing $SPECPROD environment variable' assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable' # Grab nights in data redux nights = glob.glob(dio_meta.specprod_root()+'/exposures/*') if len(nights) == 0: raise ValueError('No nights in exposures!') # Build up list of fibermap files fibermap_files = [] for night in nights: onight = night[night.rfind('/'):] files = glob.glob(dio_meta.rawdata_root()+'/'+onight+'/fibermap*') # fibermap_files += files # Get list of zbest files zbest_files = glob.glob(dio_meta.specprod_root()+'/bricks/*/zbest*') if len(zbest_files) == 0: raise ValueError('No redshifts?!') # Meta meta = get_meta() # Load+write table simtab_fil = dio_meta.specprod_root()+'/QA/sim_z_table.fits' dio_util.makepath(simtab_fil) simz_tab = dsqa_z.load_z(fibermap_files, zbest_files) simz_tab.meta = meta simz_tab.write(simtab_fil,overwrite=True) # Summary stats summ_file = dio_meta.specprod_root()+'/QA/sim_z_summ.yaml' dio_util.makepath(summ_file) summ_dict = dsqa_z.summ_stats(simz_tab) # Write with open(summ_file, 'w') as outfile: outfile.write( yaml.dump(meta))#, default_flow_style=True) ) outfile.write( yaml.dump(summ_dict, default_flow_style=False) ) #import pdb #pdb.set_trace() ''' # Reorder + cut summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL'] # Write summ_tab.meta = meta summ_tab.write(summ_file,format='ascii.ecsv', formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,overwrite=True) ''' # QA Figures fig_file = dio_meta.specprod_root()+'/QA/sim_z.pdf' dio_util.makepath(fig_file) pp = PdfPages(fig_file) # Summ dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp) for objtype in ['ELG','LRG', 'QSO_T', 'QSO_L']: dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp) # All done pp.close()
def main(): log = get_logger() key = 'DESI_ROOT' if key not in os.environ: log.fatal('Required ${} environment variable not set'.format(key)) return 0 desidir = os.getenv(key) simsdir = os.path.join(desidir, 'spectro', 'sim', 'bgs-sims1.0') brickdir = os.path.join(simsdir, 'bricks') parser = argparse.ArgumentParser() parser.add_argument('--sim', type=int, default=None, help='Simulation number (see documentation)') parser.add_argument('--nproc', type=int, default=1, help='Number of processors to use.') parser.add_argument('--simsdir', default=simsdir, help='Top-level simulation directory') parser.add_argument('--bricks', action='store_true', help='Generate the brick files.') parser.add_argument('--zfind', action='store_true', help='Fit for the redshifts.') parser.add_argument('--results', action='store_true', help='Merge all the relevant results.') parser.add_argument('--qaplots', action='store_true', help='Generate QAplots.') args = parser.parse_args() if args.sim is None: parser.print_help() sys.exit(1) # -------------------------------------------------- # Initialize the parameters of each simulation here. if args.sim == 1: seed = 678245 brickname = 'sim01' nbrick = 50 nspec = 20 phase = 0.25 rmag = (19.5, 19.5) redshift = (0.1, 0.3) zenith = 30 exptime = 300 angle = (0, 150) simoptions = [ '--exptime-range', '{}'.format(exptime), '{}'.format(exptime), '--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]), '--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]), '--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith), '--moon-phase-range', '{}'.format(phase), '{}'.format(phase), '--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1]) ] elif args.sim == 2: seed = 991274 brickname = 'sim02' nbrick = 50 nspec = 20 phase = (0.0, 1.0) rmag = (19.5, 19.5) redshift = (0.1, 0.3) zenith = 30 exptime = 300 angle = 60 simoptions = [ '--exptime-range', '{}'.format(exptime), '{}'.format(exptime), '--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]), '--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]), '--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith), '--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]), '--moon-angle-range', '{}'.format(angle), '{}'.format(angle) ] elif args.sim == 3: seed = 471934 brickname = 'sim03' nbrick = 50 nspec = 20 phase = (0.0, 1.0) rmag = (19.5, 19.5) redshift = (0.1, 0.3) zenith = 30 exptime = 300 angle = (0, 150) simoptions = [ '--exptime-range', '{}'.format(exptime), '{}'.format(exptime), '--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]), '--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]), '--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith), '--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]), '--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1]) ] elif args.sim == 4: seed = 971234 brickname = 'sim04' nbrick = 100 nspec = 50 phase = (0.0, 1.0) rmag = (17.5, 20.0) redshift = (0.1, 0.3) zenith = 30 exptime = 300 angle = (0, 150) simoptions = [ '--exptime-range', '{}'.format(exptime), '{}'.format(exptime), '--rmagrange-bgs', '{}'.format(rmag[0]), '{}'.format(rmag[1]), '--zrange-bgs', '{}'.format(redshift[0]), '{}'.format(redshift[1]), '--moon-zenith-range', '{}'.format(zenith), '{}'.format(zenith), '--moon-phase-range', '{}'.format(phase[0]), '{}'.format(phase[1]), '--moon-angle-range', '{}'.format(angle[0]), '{}'.format(angle[1]) ] nobj = nbrick*nspec rand = np.random.RandomState(seed) # -------------------------------------------------- # Generate the brick and truth files. if args.bricks: brightoptions = [ '--brickname', '{}'.format(brickname), '--nbrick', '{}'.format(nbrick), '--nspec', '{}'.format(nspec), '--outdir', '{}'.format(simsdir), '--brickdir', '{}'.format(brickdir), '--seed', '{}'.format(seed), '--objtype', 'BGS'] brightargs = brightsims.parse(np.hstack((brightoptions, simoptions))) brightargs.verbose = True brightsims.main(brightargs) # -------------------------------------------------- # Fit the redshifts if args.zfind: inputfile = os.path.join(simsdir, brickname+'-input.fits') log.info('Reading {}'.format(inputfile)) cat = fits.getdata(inputfile, 1) log.info('Testing with just one brick!') #for ib in range(10, 11): for ib in range(nbrick): thisbrick = cat['BRICKNAME'][ib] brickfiles = [os.path.join(brickdir, 'brick-{}-{}.fits'.format(ch, thisbrick)) for ch in ['b', 'r', 'z']] redoptions = [ '--brick', thisbrick, '--nproc', '{}'.format(args.nproc), '--specprod_dir', simsdir, '--zrange-galaxy', '{}'.format(redshift[0]), '{}'.format(redshift[1]), '--outfile', os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick)), '--objtype', 'ELG,LRG'] redargs = zfind.parse(redoptions) zfind.main(redargs) # -------------------------------------------------- # Parse and write out the simulation inputs, brick spectra, and redshifts if args.results: inputfile = os.path.join(simsdir, brickname+'-input.fits') log.info('Reading {}'.format(inputfile)) cat = fits.getdata(inputfile, 1) # Build a results table. resultfile = makepath(os.path.join(simsdir, '{}-results.fits'.format(brickname))) resultcols = [ ('EXPTIME', 'f4'), ('AIRMASS', 'f4'), ('MOONPHASE', 'f4'), ('MOONANGLE', 'f4'), ('MOONZENITH', 'f4'), ('SNR_B', 'f4'), ('SNR_R', 'f4'), ('SNR_Z', 'f4'), ('TARGETID', 'i8'), ('RMAG', 'f4'), ('D4000', 'f4'), ('EWHBETA', 'f4'), ('ZTRUE', 'f4'), ('Z', 'f4'), ('ZERR', 'f4'), ('ZWARNING', 'f4')] result = Table(np.zeros(nobj, dtype=resultcols)) result['EXPTIME'].unit = 's' result['MOONANGLE'].unit = 'deg' result['MOONZENITH'].unit = 'deg' for ib in range(nbrick): # Copy over some data. thisbrick = cat['BRICKNAME'][ib] result['EXPTIME'][nspec*ib:nspec*(ib+1)] = cat['EXPTIME'][ib] result['AIRMASS'][nspec*ib:nspec*(ib+1)] = cat['AIRMASS'][ib] result['MOONPHASE'][nspec*ib:nspec*(ib+1)] = cat['MOONPHASE'][ib] result['MOONANGLE'][nspec*ib:nspec*(ib+1)] = cat['MOONANGLE'][ib] result['MOONZENITH'][nspec*ib:nspec*(ib+1)] = cat['MOONZENITH'][ib] # Read the truth file of the first channel to get the metadata. truthfile = os.path.join(brickdir, thisbrick, 'truth-brick-{}-{}.fits'.format('b', thisbrick)) log.info('Reading {}'.format(truthfile)) truth = io.Brick(truthfile).hdu_list[4].data result['TARGETID'][nspec*ib:nspec*(ib+1)] = truth['TARGETID'] result['RMAG'][nspec*ib:nspec*(ib+1)] = 22.5-2.5*np.log10(truth['DECAM_FLUX'][:,2]) result['D4000'][nspec*ib:nspec*(ib+1)] = truth['D4000'] result['EWHBETA'][nspec*ib:nspec*(ib+1)] = truth['EWHBETA'] result['ZTRUE'][nspec*ib:nspec*(ib+1)] = truth['TRUEZ'] # Finally read the zbest file. zbestfile = os.path.join(brickdir, thisbrick, 'zbest-{}.fits'.format(thisbrick)) if os.path.isfile(zbestfile): log.info('Reading {}'.format(zbestfile)) zbest = read_zbest(zbestfile) # There's gotta be a better way than looping here! for ii in range(nspec): this = np.where(zbest.targetid[ii] == result['TARGETID'])[0] result['Z'][this] = zbest.z[ii] result['ZERR'][this] = zbest.zerr[ii] result['ZWARNING'][this] = zbest.zwarn[ii] #pdb.set_trace() # Finally, read the spectra and truth tables, one per channel. for channel in ('b','r','z'): brickfile = os.path.join(brickdir, thisbrick, 'brick-{}-{}.fits'.format(channel, thisbrick)) log.info('Reading {}'.format(brickfile)) brick = io.Brick(brickfile) wave = brick.get_wavelength_grid() for iobj in range(nspec): flux = brick.hdu_list[0].data[iobj,:] ivar = brick.hdu_list[1].data[iobj,:] these = np.where((wave>np.mean(wave)-50)*(wave<np.mean(wave)+50)*(flux>0))[0] result['SNR_'+channel.upper()][nspec*ib+iobj] = \ np.median(np.sqrt(flux[these]*ivar[these])) log.info('Writing {}'.format(resultfile)) write_bintable(resultfile, result, extname='RESULTS', clobber=True) # -------------------------------------------------- # Build QAplots if args.qaplots: phaserange = (-0.05, 1.05) snrrange = (0, 3) rmagrange = (17.5, 20) anglerange = (-5, 155) d4000range = (0.9, 2.2) snrinterval = 1.0 cmap = mpl.colors.ListedColormap(sns.color_palette('muted')) resultfile = os.path.join(simsdir, '{}-results.fits'.format(brickname)) log.info('Reading {}'.format(resultfile)) res = fits.getdata(resultfile, 1) qafile = os.path.join(simsdir, 'qa-{}.pdf'.format(brickname)) log.info('Writing {}'.format(qafile)) # ------------------------------ # Simulation 1 if args.sim == 1: fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5)) ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_B'], label='b channel', c=col[1]) ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_R'], label='r channel', c=col[2]) ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.2, nobj), res['SNR_Z'], label='z channel', c=col[0]) ax0.set_xlabel('Object-Moon Angle (deg)') ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)') ax0.set_xlim(anglerange) ax0.set_ylim(snrrange) plt.legend(loc='upper left', labelspacing=0.25) plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag, redshift[0], redshift[1])+\ '\nLunar Zenith Angle = {:g} deg\nLunar Phase = {:g}'.format(zenith, phase), horizontalalignment='right', transform=ax0.transAxes, fontsize=11) plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15) plt.savefig(qafile) plt.close() #pdb.set_trace() # ------------------------------ # Simulation 2 if args.sim == 2: fig, ax0 = plt.subplots(1, 1, figsize=(6, 4.5)) ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_B'], label='b channel', c=col[1]) ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_R'], label='r channel', c=col[2]) ax0.scatter(res['MOONPHASE']+rand.normal(0, 0.02, nobj), res['SNR_Z'], label='z channel', c=col[0]) ax0.set_xlabel('Lunar Phase (0=Full, 1=New)') ax0.set_ylabel(r'Signal-to-Noise Ratio (pixel$^{-1}$)') ax0.set_xlim(phaserange) ax0.set_ylim(snrrange) plt.legend(loc='upper left', labelspacing=0.25) plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag, redshift[0], redshift[1])+\ '\nLunar Zenith Angle = {:g} deg\nObject-Moon Angle = {:g} deg'.format(zenith, angle), horizontalalignment='right', transform=ax0.transAxes, fontsize=11) plt.subplots_adjust(bottom=0.2, right=0.95, left=0.15) plt.savefig(qafile) plt.close() # ------------------------------ # Simulation 3 if args.sim == 3: fig, ax0 = plt.subplots(1, 1, figsize=(6, 4)) im = ax0.scatter(res['MOONANGLE']+rand.normal(0, 0.3, nobj), res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange, cmap=cmap) ax0.set_xlabel('Object-Moon Angle (deg)') ax0.set_ylabel(r'r channel Signal-to-Noise Ratio (pixel$^{-1}$)') ax0.set_xlim(anglerange) ax0.set_ylim(snrrange) ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval)) plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag, redshift[0], redshift[1])+\ '\nLunar Zenith Angle = {:g} deg'.format(zenith), horizontalalignment='right', transform=ax0.transAxes, fontsize=11) cbar = fig.colorbar(im) cbar.set_ticks([0, 0.5, 1]) cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90) cbar.ax.set_ylabel('Lunar Phase') plt.subplots_adjust(bottom=0.2, right=1.0) plt.savefig(qafile) plt.close() # ------------------------------ # Simulation 4 if args.sim == 4: bins = 30 zgood = (np.abs(res['Z']-res['ZTRUE'])<5E-5)*(res['ZWARNING']==0)*1 H, xedges, yedges = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins, weights=zgood) H2, _, _ = np.histogram2d(res['RMAG'], res['MOONPHASE'], bins=bins) extent = np.array((rmagrange, phaserange)).flatten() #pdb.set_trace() fig, ax0 = plt.subplots(1, 1, figsize=(6, 4)) im = ax0.imshow(H/H2, extent=extent, interpolation='nearest')#, cmap=cmap) ax0.set_xlabel('r (AB mag)') ax0.set_ylabel('Lunar Phase (0=Full, 1=New)') ax0.set_xlim(rmagrange) ax0.set_ylim(phaserange) #plt.text(0.95, 0.7, 't = {:g} s\nr = {:g} mag\nRedshift = {:.2f}-{:.2f}'.format(exptime, rmag, # redshift[0], redshift[1])+\ # '\nLunar Zenith Angle = {:g} deg'.format(zenith), # horizontalalignment='right', transform=ax0.transAxes, # fontsize=11) cbar = fig.colorbar(im) #cbar.set_ticks([0, 0.5, 1]) #cbar.ax.set_yticklabels(['Full','Quarter','New'], rotation=90) #cbar.ax.set_ylabel('Lunar Phase') plt.subplots_adjust(bottom=0.2, right=1.0) plt.savefig(qafile) plt.close() # ------------------------------ # Simulation 10 if args.sim == 10: zgood = (np.abs(res['Z']-res['ZTRUE'])<0.001)*(res['ZWARNING']==0)*1 #pdb.set_trace() fig, (ax0, ax1, ax2) = plt.subplots(3, 1, figsize=(6,6)) # moon phase vs S/N(r) im = ax0.scatter(res['RMAG'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange, cmap=cmap) ax0.set_xlabel('r (AB mag)') ax0.set_ylabel('S/N (r channel)') ax0.set_xlim(rmagrange) ax0.set_ylim(snrrange) ax0.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval)) # object-moon angle vs S/N(r) im = ax1.scatter(res['MOONANGLE'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange, cmap=cmap) ax1.set_xlabel('Object-Moon Angle (deg)') ax1.set_ylabel('S/N (r channel)') ax1.set_xlim(anglerange) ax1.set_ylim(snrrange) ax1.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval)) # D(4000) vs S/N(r) im = ax2.scatter(res['D4000'], res['SNR_R'], c=res['MOONPHASE'], vmin=phaserange, cmap=cmap) ax2.set_xlabel('$D_{n}(4000)$') ax2.set_ylabel('S/N (r channel)') ax2.set_xlim(d4000range) ax2.set_ylim(snrrange) ax2.yaxis.set_major_locator(mpl.ticker.MultipleLocator(snrinterval)) # Shared colorbar cbarax = fig.add_axes([0.83, 0.15, 0.03, 0.8]) cbar = fig.colorbar(im, cax=cbarax) ticks = ['Full','Quarter','New'] cbar.set_ticks([0, 0.5, 1]) cbar.ax.set_yticklabels(ticks, rotation=-45) plt.tight_layout(pad=0.5)#, h_pad=0.2, w_pad=0.3) plt.subplots_adjust(right=0.78) plt.savefig(qafile) plt.close()
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) qframe: desispec.qproc.QFrame object with wave, flux, ivar... Optional: header: astropy.io.fits.Header or dict to override frame.header fibermap: table to store as FIBERMAP HDU Returns: full filepath of output file that was written Note: to create a QFrame object to pass into write_qframe, qframe = QFrame(wave, flux, ivar) """ log = get_logger() outfile = makepath(outfile, 'qframe') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(qframe.meta) add_dependencies(hdr) hdus = fits.HDUList() x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr) x.header['EXTNAME'] = 'FLUX' if units is not None: units = str(units) if 'BUNIT' in hdr and hdr['BUNIT'] != units: log.warning('BUNIT {bunit} != units {units}; using {units}'.format( bunit=hdr['BUNIT'], units=units)) x.header['BUNIT'] = units hdus.append(x) hdus.append( fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR') ) if qframe.mask is None : qframe.mask=np.zeros(qframe.flux.shape,dtype=np.uint32) # hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') ) hdus.append( fits.ImageHDU(qframe.mask, name='MASK') ) if qframe.sigma is None : qframe.sigma=np.zeros(qframe.flux.shape,dtype=np.float) hdus.append( fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA') ) hdus.append( fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH') ) hdus[-1].header['BUNIT'] = 'Angstrom' if fibermap is not None: fibermap = encode_table(fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append( fits.convenience.table_to_hdu(fibermap) ) elif qframe.fibermap is not None: fibermap = encode_table(qframe.fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append( fits.convenience.table_to_hdu(fibermap) ) elif qframe.spectrograph is not None: x.header['FIBERMIN'] = 500*qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe) else: log.error("You are likely writing a qframe without sufficient fiber info") hdus.writeto(outfile+'.tmp', clobber=True, checksum=True) os.rename(outfile+'.tmp', outfile) return outfile
def main(args): # Set up the logger. if args.verbose: log = get_logger(DEBUG) else: log = get_logger() objtype = args.objtype.upper() log.debug('Using OBJTYPE {}'.format(objtype)) log.debug('Simulating {:g} bricks each with {:g} spectra'.format(args.nbrick, args.nspec)) # Draw priors uniformly given the input ranges. rand = np.random.RandomState(args.seed) exptime = rand.uniform(args.exptime_range[0], args.exptime_range[1], args.nbrick) airmass = rand.uniform(args.airmass_range[0], args.airmass_range[1], args.nbrick) moonphase = rand.uniform(args.moon_phase_range[0], args.moon_phase_range[1], args.nbrick) moonangle = rand.uniform(args.moon_angle_range[0], args.moon_angle_range[1], args.nbrick) moonzenith = rand.uniform(args.moon_zenith_range[0], args.moon_zenith_range[1], args.nbrick) # Build a metadata table with the simulation inputs. metafile = makepath(os.path.join(args.outdir, '{}-input.fits'.format(args.brickname))) metacols = [ ('BRICKNAME', 'S20'), ('SEED', 'S20'), ('EXPTIME', 'f4'), ('AIRMASS', 'f4'), ('MOONPHASE', 'f4'), ('MOONANGLE', 'f4'), ('MOONZENITH', 'f4')] meta = Table(np.zeros(args.nbrick, dtype=metacols)) meta['EXPTIME'].unit = 's' meta['MOONANGLE'].unit = 'deg' meta['MOONZENITH'].unit = 'deg' meta['BRICKNAME'] = ['{}-{:03d}'.format(args.brickname, ii) for ii in range(args.nbrick)] meta['EXPTIME'] = exptime meta['AIRMASS'] = airmass meta['MOONPHASE'] = moonphase meta['MOONANGLE'] = moonangle meta['MOONZENITH'] = moonzenith log.debug('Writing {}'.format(metafile)) write_bintable(metafile, meta, extname='METADATA', clobber=True) # Generate each brick in turn. for ii in range(args.nbrick): thisbrick = meta['BRICKNAME'][ii] log.debug('Building brick {}'.format(thisbrick)) brickargs = ['--brickname', thisbrick, '--objtype', args.objtype, '--nspec', '{}'.format(args.nspec), '--outdir', os.path.join(args.brickdir, thisbrick), '--outdir-truth', os.path.join(args.brickdir, thisbrick), '--exptime', '{}'.format(exptime[ii]), '--airmass', '{}'.format(airmass[ii]), '--moon-phase', '{}'.format(moonphase[ii]), '--moon-angle', '{}'.format(moonangle[ii]), '--moon-zenith', '{}'.format(moonzenith[ii]), '--zrange-bgs', '{}'.format(args.zrange_bgs[0]), '{}'.format(args.zrange_bgs[1]), '--rmagrange-bgs', '{}'.format(args.rmagrange_bgs[0]), '{}'.format(args.rmagrange_bgs[1])] if args.seed is not None: brickargs.append('--seed') brickargs.append('{}'.format(args.seed)) quickargs = quickbrick.parse(brickargs) if args.verbose: quickargs.verbose = True quickbrick.main(quickargs)
def main(args): # Set up the logger. if args.verbose: log = get_logger(DEBUG) else: log = get_logger() objtype = args.objtype.upper() log.debug('Using OBJTYPE {}'.format(objtype)) log.debug('Simulating {:g} bricks each with {:g} spectra'.format( args.nexp, args.nspec)) # Draw priors uniformly given the input ranges. rand = np.random.RandomState(args.seed) exptime = rand.uniform(args.exptime_range[0], args.exptime_range[1], args.nexp) airmass = rand.uniform(args.airmass_range[0], args.airmass_range[1], args.nexp) moonphase = rand.uniform(args.moon_phase_range[0], args.moon_phase_range[1], args.nexp) moonangle = rand.uniform(args.moon_angle_range[0], args.moon_angle_range[1], args.nexp) moonzenith = rand.uniform(args.moon_zenith_range[0], args.moon_zenith_range[1], args.nexp) maglimits = args.rmagrange - bgs zrange = args.zrange - bgs # Build a metadata table with the simulation inputs. metafile = makepath( os.path.join(args.outdir, '{}-input.fits'.format(args.brickname))) metacols = [('BRICKNAME', 'S20'), ('SEED', 'S20'), ('EXPTIME', 'f4'), ('AIRMASS', 'f4'), ('MOONPHASE', 'f4'), ('MOONANGLE', 'f4'), ('MOONZENITH', 'f4')] meta = Table(np.zeros(args.nexp, dtype=metacols)) meta['EXPTIME'].unit = 's' meta['MOONANGLE'].unit = 'deg' meta['MOONZENITH'].unit = 'deg' meta['BRICKNAME'] = [ '{}-{:03d}'.format(args.nexp, ii) for ii in range(args.nexp) ] meta['EXPTIME'] = exptime meta['AIRMASS'] = airmass meta['MOONPHASE'] = moonphase meta['MOONANGLE'] = moonangle meta['MOONZENITH'] = moonzenith log.debug('Writing {}'.format(metafile)) write_bintable(metafile, meta, extname='METADATA', clobber=True) # Generate each brick in turn. for ii in range(args.nexp): thisbrick = meta['BRICKNAME'][ii] log.debug('Building brick {}'.format(thisbrick)) brickargs = [ '--brickname', thisbrick, '--objtype', args.objtype, '--nspec', '{}'.format(args.nspec), '--outdir', os.path.join(args.brickdir, thisbrick), '--outdir-truth', os.path.join(args.brickdir, thisbrick), '--exptime', '{}'.format(exptime[ii]), '--airmass', '{}'.format(airmass[ii]), '--moon-phase', '{}'.format( moonphase[ii]), '--moon-angle', '{}'.format(moonangle[ii]), '--moon-zenith', '{}'.format(moonzenith[ii]), '--zrange-bgs', '{}'.format(args.zrange_bgs[0]), '{}'.format(args.zrange_bgs[1]), '--rmagrange-bgs', '{}'.format(args.rmagrange_bgs[0]), '{}'.format(args.rmagrange_bgs[1]) ] if args.seed is not None: brickargs.append('--seed') brickargs.append('{}'.format(args.seed)) quickargs = quickbrick.parse(brickargs) if args.verbose: quickargs.verbose = True quickbrick.main(quickargs)
def main(): '''Runs the process ''' # Check environmental variables are set assert 'DESI_SPECTRO_DATA' in os.environ, 'Missing $DESI_SPECTRO_DATA environment variable' assert 'PRODNAME' in os.environ, 'Missing $PRODNAME environment variable' assert 'DESI_SPECTRO_REDUX' in os.environ, 'Missing $DESI_SPECTRO_REDUX environment variable' # Grab nights in data redux nights = glob.glob(dio_meta.specprod_root() + '/exposures/*') if len(nights) == 0: raise ValueError('No nights in exposures!') # Build up list of fibermap files fibermap_files = [] for night in nights: onight = night[night.rfind('/'):] files = glob.glob(dio_meta.rawdata_root() + '/' + onight + '/fibermap*') # fibermap_files += files # Get list of zbest files zbest_files = glob.glob(dio_meta.specprod_root() + '/bricks/*/zbest*') if len(zbest_files) == 0: raise ValueError('No redshifts?!') # Meta meta = get_meta() # Load+write table simtab_fil = dio_meta.specprod_root() + '/QA/sim_z_table.fits' dio_util.makepath(simtab_fil) simz_tab = dsqa_z.load_z(fibermap_files, zbest_files) simz_tab.meta = meta simz_tab.write(simtab_fil, overwrite=True) # Summary stats summ_file = dio_meta.specprod_root() + '/QA/sim_z_summ.yaml' dio_util.makepath(summ_file) summ_dict = dsqa_z.summ_stats(simz_tab) # Write with open(summ_file, 'w') as outfile: outfile.write(yaml.dump(meta)) #, default_flow_style=True) ) outfile.write(yaml.dump(summ_dict, default_flow_style=False)) #import pdb #pdb.set_trace() ''' # Reorder + cut summ_tab=full_summ_tab['OBJTYPE', 'NTARG', 'N_SURVEY', 'EFF', 'MED_DZ', 'CAT_RATE', 'REQ_FINAL'] # Write summ_tab.meta = meta summ_tab.write(summ_file,format='ascii.ecsv', formats=dict(MED_DZ='%8.6f',EFF='%5.3f',CAT_RATE='%6.4f'))#,clobber=True) ''' # QA Figures fig_file = dio_meta.specprod_root() + '/QA/sim_z.pdf' dio_util.makepath(fig_file) pp = PdfPages(fig_file) # Summ dsqa_z.summ_fig(simz_tab, summ_dict, meta, pp=pp) for objtype in ['ELG', 'LRG', 'QSO_T', 'QSO_L']: dsqa_z.obj_fig(simz_tab, objtype, summ_dict, pp=pp) # All done pp.close()
def write_qframe(outfile, qframe, header=None, fibermap=None, units=None): """Write a frame fits file and returns path to file written. Args: outfile: full path to output file, or tuple (night, expid, channel) qframe: desispec.qproc.QFrame object with wave, flux, ivar... Optional: header: astropy.io.fits.Header or dict to override frame.header fibermap: table to store as FIBERMAP HDU Returns: full filepath of output file that was written Note: to create a QFrame object to pass into write_qframe, qframe = QFrame(wave, flux, ivar) """ log = get_logger() outfile = makepath(outfile, 'qframe') if header is not None: hdr = fitsheader(header) else: hdr = fitsheader(qframe.meta) add_dependencies(hdr) hdus = fits.HDUList() x = fits.PrimaryHDU(qframe.flux.astype('f4'), header=hdr) x.header['EXTNAME'] = 'FLUX' if units is not None: units = str(units) if 'BUNIT' in hdr and hdr['BUNIT'] != units: log.warning('BUNIT {bunit} != units {units}; using {units}'.format( bunit=hdr['BUNIT'], units=units)) x.header['BUNIT'] = units hdus.append(x) hdus.append(fits.ImageHDU(qframe.ivar.astype('f4'), name='IVAR')) if qframe.mask is None: qframe.mask = np.zeros(qframe.flux.shape, dtype=np.uint32) # hdus.append( fits.CompImageHDU(qframe.mask, name='MASK') ) hdus.append(fits.ImageHDU(qframe.mask, name='MASK')) if qframe.sigma is None: qframe.sigma = np.zeros(qframe.flux.shape, dtype=np.float) hdus.append(fits.ImageHDU(qframe.sigma.astype('f4'), name='YSIGMA')) hdus.append(fits.ImageHDU(qframe.wave.astype('f8'), name='WAVELENGTH')) hdus[-1].header['BUNIT'] = 'Angstrom' if fibermap is not None: fibermap = encode_table(fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append(fits.convenience.table_to_hdu(fibermap)) elif qframe.fibermap is not None: fibermap = encode_table(qframe.fibermap) #- unicode -> bytes fibermap.meta['EXTNAME'] = 'FIBERMAP' hdus.append(fits.convenience.table_to_hdu(fibermap)) elif qframe.spectrograph is not None: x.header[ 'FIBERMIN'] = 500 * qframe.spectrograph # Hard-coded (as in desispec.qproc.qframe) else: log.error( "You are likely writing a qframe without sufficient fiber info") raise ValueError('no fibermap') hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True) os.rename(outfile + '.tmp', outfile) return outfile