Ejemplo n.º 1
0
def main(args) :

    log=get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers=read_stdstar_models(args.models)

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap
    model_fibers = model_fibers%500
    if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
        for i in model_fibers:
            log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 2
0
def main() :

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat', type = str, default = None,
                        help = 'path of DESI fiberflat fits file')
    parser.add_argument('--sky', type = str, default = None,
                        help = 'path of DESI sky fits file')
    parser.add_argument('--calib', type = str, default = None,
                        help = 'path of DESI calibration fits file')
    parser.add_argument('--outfile', type = str, default = None, required=True,
                        help = 'path of DESI sky fits file')
    # add calibration here when exists

    args = parser.parse_args()
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)
    
    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky!=None :
        log.info("subtract sky")
        # read sky
        skymodel=read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)


    # save output
    write_frame(args.outfile, frame)

    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 3
0
def main(args):
    log = get_logger()

    frame=read_frame(args.infile, skip_resolution=True)
    fibermap=read_fibermap(args.infile)
    fiberflat=read_fiberflat(args.fiberflat)
    skymodel=read_sky(args.sky)
    fluxcalib=read_flux_calibration(args.calib)

    cam=args.infile.split('/')[-1].split('-')[1]
    band=cam[0]
    bands=[band]

    # Indices of sky fibers. 
    sky_indx = np.where(fibermap['OBJTYPE'] == 'SKY')[0]
    
    rd_var, sky_var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=True)   
    var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=False)
    
    nsky = 4 
    fig, axes = plt.subplots(1, nsky, figsize=(5 * nsky, 5))

    for i in range(nsky):
        def calc_alphavar(alpha):
            return alpha * rd_var[sky_indx,:] + sky_var[sky_indx,:]
        
        def alpha_fit(alpha):
            _var = calc_alphavar(alpha)
            ivar =  1. / _var
            X2 = (frame.ivar[sky_indx,:] - ivar)**2.
            return np.sum(X2)
        
        res = minimize(alpha_fit, x0=[1.])
        alpha = res.x[0]

        indx = sky_indx[i]
        
        axes[i].plot(skymodel.wave, median_filter(frame.ivar[indx,:], 10), lw=0.4, label='Sky frame IVAR', alpha=0.4)
        axes[i].plot(skymodel.wave, 1./rd_var[indx,:], lw=0.4, label='Model rd. IVAR', alpha=0.4)
        # axes[i].plot(skymodel.wave, 1./sky_var[indx,:], lw=0.4, label='Model Sky IVAR', alpha=0.4)
        # axes[i].plot(skymodel.wave, 1./var[indx,:], lw=0.4, label=r'Model IVAR', alpha=0.4)
        axes[i].plot(skymodel.wave, median_filter(1./calc_alphavar(alpha)[i,:], 10), lw=0.4, label=r'$\alpha$ Model IVAR', alpha=0.4)
        axes[i].set_title(r'Fiber {:d} ($\alpha$ = {:.6f})'.format(indx, alpha))
        axes[i].set_xlabel(r'Wavelength [$AA$]')
        axes[i].set_yscale('log')
        axes[i].set_ylim(bottom=5.e-4, top=3.e-2)
        axes[i].legend(frameon=False, loc=2)
        
    axes[0].set_ylabel('e/A')
        
    pl.show()
Ejemplo n.º 4
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky!=None :
        log.info("subtract sky")
        # read sky
        skymodel=read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)


    # save output
    write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')

    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 5
0
def cmx_skies():
    ''' sky surface brightnesses from sky fibers from the CMX exposures. The
    sky surface brightnesses are generated from the flat fielded sky data
    that's throughput corrected. 

    Return
    ------
    skies : dictionary
        dictionary that contains data on sky fibers for a given (tile, date,
        exposure, and spectrograph). The following keys are available in the
        output dictionary: 
        - `tileid`, `date`, `mjd`, `expid`, `spectrograph`
        - observing conditions: `airmass`, `moon_ill`, `moon_alt`, `moon_sep`,
          `sun_alt`, `sun_sep`, `transparency`, `transp_min`, `transp_max`,
          `fwhm`, `exptime`
        - surface brightnesses: `sky_sb_b`, `sky_sb_r`, `sky_sb_z`,
    '''
    f_cmx_skies = '/global/cfs/cdirs/desi/users/chahah/bgs_exp_coadd/sky_fibers.cmx.v1.hdf5'

    if not os.path.isfile(f_cmx_skies):
        # read observing conditions compiled from GFAs
        gfa = fitsio.read(
            os.path.join(
                '/global/cfs/cdirs/desi/users/ameisner/GFA/conditions',
                'offline_all_guide_ccds_thru_20200315.fits'))

        # CMX BGS tiles
        bgs_minisv_tiles = [70500, 70502, 70510]
        bgs_sv0_tiles = [66000, 66014, 66003]
        bgs_tiles = bgs_minisv_tiles + bgs_sv0_tiles

        # gather all (tile, date, expid, spectrograph) combinations
        cmx = []
        for tile in bgs_tiles:
            dates = _get_dates(tile)
            for date in dates:
                exps = _get_exposures(tile, date)
                for exp in exps:
                    spectographs = _get_spectograph(tile, date, exp)
                    for spec in spectographs:
                        cmx.append([tile, date, exp, spec])

        #  all the keys
        sky_data = {}
        for k in [
                'airmass', 'moon_ill', 'moon_alt', 'moon_sep', 'sun_alt',
                'sun_sep', 'sky_sb_b', 'sky_sb_r', 'sky_sb_z', 'tileid',
                'date', 'expid', 'spectrograph', 'mjd', 'transparency',
                'transp_min', 'transp_max', 'fwhm', 'exptime'
        ]:
            sky_data[k] = []

        # loop through each (tile, date, expid, spectrograph)
        for i, _cmx in enumerate(cmx):
            _tileid, _date, _exp, _spec = _cmx
            print('--- %i, %i, %i, %i ---' % (_tileid, _date, _exp, _spec))

            # coadd file
            f_coadd = os.path.join(
                dir_coadd, 'coadd-%i-%i-%i-%s.fits' %
                (_tileid, _date, _spec, str(_exp).zfill(8)))

            dir_exp = os.path.join(dir_redux, 'exposures', str(_date),
                                   str(_exp).zfill(8))
            f_sky = lambda band: os.path.join(
                dir_exp, 'sky-%s%i-%s.fits' %
                (band, _spec, str(_exp).zfill(8)))

            if not os.path.isfile(f_coadd):
                print('... no coadd: %s' % os.path.basename(f_coadd))
                continue

            # read coadd and sky data. coadd data is used to determine good sky
            # fibers
            coadd = fitsio.read(f_coadd)

            sky_b = read_sky(f_sky('b'))
            sky_r = read_sky(f_sky('r'))
            sky_z = read_sky(f_sky('z'))

            exptime = sky_b.header['EXPTIME']

            if i == 0:
                wave_b = sky_b.wave
                wave_r = sky_r.wave
                wave_z = sky_z.wave

            # good sky fibers only
            is_good = (coadd['FIBERSTATUS'] == 0)
            is_sky = (coadd['CMX_TARGET']
                      & cmx_targetmask.cmx_mask.mask('SKY')) != 0
            good_sky = is_good & is_sky

            # match to GFA obs condition using NIGHT and EXPID
            m_gfa = ((gfa['NIGHT'] == int(_date)) &
                     (gfa['EXPID'] == int(_exp)))
            if np.sum(m_gfa) == 0:
                print('... no match to GFA conditions')
                continue
            assert (gfa['MJD'][m_gfa].max() -
                    gfa['MJD'][m_gfa].min()) < 0.08333333333

            # median MJD of GFA data
            mjd_mid = np.median(gfa['MJD'][m_gfa])
            # median transparency from GFA data
            transp = gfa['TRANSPARENCY'][m_gfa]
            not_nan = np.isfinite(transp)
            if np.sum(not_nan) > 0:
                transp_min = np.min(transp[not_nan])
                transp_max = np.max(transp[not_nan])
                transp_mid = np.median(transp[not_nan])
            else:
                transp_min = 0.
                transp_max = 1.
                transp_mid = 1.
            # FWHM
            _fwhm = gfa['FWHM_ASEC'][m_gfa]
            fwhm = np.median(_fwhm[np.isfinite(_fwhm)])

            # get observing conditions (these values more or less agree with
            # the GFA values)
            _airmass, _moon_ill, _moon_alt, _moon_sep, _sun_alt, _sun_sep = \
                    _get_obs_param(coadd['TARGET_RA'][good_sky],
                            coadd['TARGET_DEC'][good_sky], mjd_mid)

            print('%.f exptime' % exptime)
            print('%i sky fibers' % np.sum(is_sky))
            print('%i good sky fibers' % np.sum(good_sky))
            print('%.2f < transp < %.2f' % (transp_min, transp_max))

            sky_data['tileid'].append(np.repeat(_tileid, len(_airmass)))
            sky_data['date'].append(np.repeat(_date, len(_airmass)))
            sky_data['expid'].append(np.repeat(_exp, len(_airmass)))
            sky_data['spectrograph'].append(np.repeat(_spec, len(_airmass)))
            sky_data['mjd'].append(np.repeat(mjd_mid, len(_airmass)))
            sky_data['transparency'].append(
                np.repeat(transp_mid, len(_airmass)))
            sky_data['transp_min'].append(np.repeat(transp_min, len(_airmass)))
            sky_data['transp_max'].append(np.repeat(transp_max, len(_airmass)))
            sky_data['fwhm'].append(np.repeat(fwhm, len(_airmass)))
            sky_data['exptime'].append(np.repeat(exptime, len(_airmass)))

            # sky surface brightness
            sky_data['sky_sb_b'].append(
                _convert_sky_rate(sky_b, good_sky, exptime, 'b'))
            sky_data['sky_sb_r'].append(
                _convert_sky_rate(sky_r, good_sky, exptime, 'r'))
            sky_data['sky_sb_z'].append(
                _convert_sky_rate(sky_z, good_sky, exptime, 'z'))

            # observing conditions
            sky_data['airmass'].append(_airmass)
            sky_data['moon_ill'].append(np.repeat(_moon_ill, len(_airmass)))
            sky_data['moon_alt'].append(np.repeat(_moon_alt, len(_airmass)))
            sky_data['moon_sep'].append(_moon_sep)
            sky_data['sun_alt'].append(np.repeat(_sun_alt, len(_airmass)))
            sky_data['sun_sep'].append(_sun_sep)

        # save to file
        f = h5py.File(os.path.join(dir_coadd, 'sky_fibers.cmx.v1.hdf5'), 'w')
        for k in sky_data.keys():
            f.create_dataset(k, data=np.concatenate(sky_data[k], axis=0))
        f.create_dataset('wave_b', data=wave_b)
        f.create_dataset('wave_r', data=wave_r)
        f.create_dataset('wave_z', data=wave_z)
        f.close()

    # read in CMX sky data
    fskies = h5py.File(f_cmx_skies, 'r')
    skies = {}
    for k in fskies.keys():
        skies[k] = fskies[k][...]
    return skies
Ejemplo n.º 6
0
def main(args) :
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)"%(args.color,args.delta_color))

    frames={}
    flats={}
    skies={}

    spectrograph=None
    starfibers=None
    starindices=None
    fibermap=None

    # READ DATA
    ############################################

    for filename in args.frames :

        log.info("reading %s"%filename)
        frame=io.read_frame(filename)
        header=fits.getheader(filename, 0)
        frame_fibermap = frame.fibermap
        frame_starindices = np.where(isStdStar(frame_fibermap['DESI_TARGET']))[0]
        
        #- Confirm that all fluxes have entries but trust targeting bits
        #- to get basic magnitude range correct
        keep = np.ones(len(frame_starindices), dtype=bool)

        for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']:  #- and W1 and W2?
            keep &= frame_fibermap[colname][frame_starindices] > 10**((22.5-30)/2.5)
            keep &= frame_fibermap[colname][frame_starindices] < 10**((22.5-0)/2.5)

        frame_starindices = frame_starindices[keep]
        
        camera=safe_read_key(header,"CAMERA").strip().lower()

        if spectrograph is None :
            spectrograph = frame.spectrograph
            fibermap = frame_fibermap
            starindices=frame_starindices
            starfibers=fibermap["FIBER"][starindices]

        elif spectrograph != frame.spectrograph :
            log.error("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
            raise ValueError("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
        elif starindices.size != frame_starindices.size or np.sum(starindices!=frame_starindices)>0 :
            log.error("incompatible fibermap")
            raise ValueError("incompatible fibermap")

        if not camera in frames :
            frames[camera]=[]
        frames[camera].append(frame)
 
    for filename in args.skymodels :
        log.info("reading %s"%filename)
        sky=io.read_sky(filename)
        header=fits.getheader(filename, 0)
        camera=safe_read_key(header,"CAMERA").strip().lower()
        if not camera in skies :
            skies[camera]=[]
        skies[camera].append(sky)
        
    for filename in args.fiberflats :
        log.info("reading %s"%filename)
        header=fits.getheader(filename, 0)
        flat=io.read_fiberflat(filename)
        camera=safe_read_key(header,"CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if camera in flats:
            log.warning("cannot handle several flats of same camera (%s), will use only the first one"%camera)
            #raise ValueError("cannot handle several flats of same camera (%s)"%camera)
        else :
            flats[camera]=flat
    

    if starindices.size == 0 :
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")

    log.info("found %d STD stars"%starindices.size)

    log.warning("Not using flux errors for Standard Star fits!")
    
    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    for cam in frames :

        if not cam in skies:
            log.warning("Missing sky for %s"%cam)
            frames.pop(cam)
            continue
        if not cam in flats:
            log.warning("Missing flat for %s"%cam)
            frames.pop(cam)
            continue
        

        flat=flats[cam]
        for frame,sky in zip(frames[cam],skies[cam]) :
            frame.flux = frame.flux[starindices]
            frame.ivar = frame.ivar[starindices]
            frame.ivar *= (frame.mask[starindices] == 0)
            frame.ivar *= (sky.ivar[starindices] != 0)
            frame.ivar *= (sky.mask[starindices] == 0)
            frame.ivar *= (flat.ivar[starindices] != 0)
            frame.ivar *= (flat.mask[starindices] == 0)
            frame.flux *= ( frame.ivar > 0) # just for clean plots
            for star in range(frame.flux.shape[0]) :
                ok=np.where((frame.ivar[star]>0)&(flat.fiberflat[star]!=0))[0]
                if ok.size > 0 :
                    frame.flux[star] = frame.flux[star]/flat.fiberflat[star] - sky.flux[star]
            frame.resolution_data = frame.resolution_data[starindices]

    nstars = starindices.size
    fibermap = Table(fibermap[starindices])

    # READ MODELS
    ############################################
    log.info("reading star models in %s"%args.starmodels)
    stdwave,stdflux,templateid,teff,logg,feh=io.read_stdstar_templates(args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################

    #- Support older fibermaps
    if 'PHOTSYS' not in fibermap.colnames:
        log.warning('Old fibermap format; using defaults for missing columns')
        log.warning("    PHOTSYS = 'S'")
        log.warning("    MW_TRANSMISSION_G/R/Z = 1.0")
        log.warning("    EBV = 0.0")
        fibermap['PHOTSYS'] = 'S'
        fibermap['MW_TRANSMISSION_G'] = 1.0
        fibermap['MW_TRANSMISSION_R'] = 1.0
        fibermap['MW_TRANSMISSION_Z'] = 1.0
        fibermap['EBV'] = 0.0

    model_filters = dict()
    if 'S' in fibermap['PHOTSYS']:
        for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
            model_filters[filter_name] = load_filter(filter_name)

    if 'N' in fibermap['PHOTSYS']:
        for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
            model_filters[filter_name] = load_filter(filter_name)

    if len(model_filters) == 0:
        raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")

    log.info("computing model mags for %s"%sorted(model_filters.keys()))
    model_mags = dict()
    fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
    for filter_name, filter_response in model_filters.items():
        model_mags[filter_name] = filter_response.get_ab_magnitude(stdflux*fluxunits,stdwave)
    log.info("done computing model mags")

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    linear_coefficients=np.zeros((nstars,stdflux.shape[0]))
    chi2dof=np.zeros((nstars))
    redshift=np.zeros((nstars))
    normflux=[]

    star_mags = dict()
    star_unextincted_mags = dict()
    for band in ['G', 'R', 'Z']:
        star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band])
        star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band] / fibermap['MW_TRANSMISSION_'+band])

    star_colors = dict()
    star_colors['G-R'] = star_mags['G'] - star_mags['R']
    star_colors['R-Z'] = star_mags['R'] - star_mags['Z']

    star_unextincted_colors = dict()
    star_unextincted_colors['G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
    star_unextincted_colors['R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']

    fitted_model_colors = np.zeros(nstars)

    for star in range(nstars) :

        log.info("finding best model for observed star #%d"%star)

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames :
            for i,frame in enumerate(frames[camera]) :
                identifier="%s-%d"%(camera,i)
                wave[identifier]=frame.wave
                flux[identifier]=frame.flux[star]
                ivar[identifier]=frame.ivar[star]
                resolution_data[identifier]=frame.resolution_data[star]

        # preselect models based on magnitudes
        if fibermap['PHOTSYS'][star] == 'N':
            if args.color == 'G-R':
                model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
            elif args.color == 'R-Z':
                model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
            else:
                raise ValueError('Unknown color {}'.format(args.color))
        else:
            if args.color == 'G-R':
                model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
            elif args.color == 'R-Z':
                model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
            else:
                raise ValueError('Unknown color {}'.format(args.color))

        color_diff = model_colors - star_unextincted_colors[args.color][star]
        selection = np.abs(color_diff) < args.delta_color

        # smallest cube in parameter space including this selection (needed for interpolation)
        new_selection = (teff>=np.min(teff[selection]))&(teff<=np.max(teff[selection]))
        new_selection &= (logg>=np.min(logg[selection]))&(logg<=np.max(logg[selection]))
        new_selection &= (feh>=np.min(feh[selection]))&(feh<=np.max(feh[selection]))
        selection = np.where(new_selection)[0]

        log.info("star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"%(
            star, starfibers[star], args.color, star_unextincted_colors[args.color][star],
            selection.size, stdflux.shape[0]))
        
        # Match unextincted standard stars to data
        coefficients, redshift[star], chi2dof[star] = match_templates(
            wave, flux, ivar, resolution_data,
            stdwave, stdflux[selection],
            teff[selection], logg[selection], feh[selection],
            ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res,
            template_error=args.template_error
            )
        
        linear_coefficients[star,selection] = coefficients
        
        log.info('Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'.format(
            starfibers[star],
            np.inner(teff,linear_coefficients[star]),
            np.inner(logg,linear_coefficients[star]),
            np.inner(feh,linear_coefficients[star]),
            redshift[star],
            chi2dof[star])
            )
        
        # Apply redshift to original spectrum at full resolution
        model=np.zeros(stdwave.size)
        redshifted_stdwave = stdwave*(1+redshift[star])
        for i,c in enumerate(linear_coefficients[star]) :
            if c != 0 :
                model += c*np.interp(stdwave,redshifted_stdwave,stdflux[i])

        # Apply dust extinction to the model
        model *= dust_transmission(stdwave, fibermap['EBV'][star])

        # Compute final color of dust-extincted model
        if fibermap['PHOTSYS'][star] == 'N':
            if args.color == 'G-R':
                model_mag1 = model_filters['BASS_G'].get_ab_magnitude(model*fluxunits, stdwave)
                model_mag2 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
                model_magr = model_mag2
            elif args.color == 'R-Z':
                model_mag1 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
                model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(model*fluxunits, stdwave)
                model_magr = model_mag1
            else:
                raise ValueError('Unknown color {}'.format(args.color))
        else:
            if args.color == 'G-R':
                model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(model*fluxunits, stdwave)
                model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
                model_magr = model_mag2
            elif args.color == 'R-Z':
                model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
                model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(model*fluxunits, stdwave)
                model_magr = model_mag1
            else:
                raise ValueError('Unknown color {}'.format(args.color))

        fitted_model_colors[star] = model_mag1 - model_mag2
        
        #- TODO: move this back into normalize_templates, at the cost of
        #- recalculating a model magnitude?

        # Normalize the best model using reported magnitude
        scalefac=10**((model_magr - star_mags['R'][star])/2.5)

        log.info('scaling R mag {} to {} using scale {}'.format(model_magr, star_mags['R'][star], scalefac))
        normflux.append(model*scalefac)

    # Now write the normalized flux for all best models to a file
    normflux=np.array(normflux)
    data={}
    data['LOGG']=linear_coefficients.dot(logg)
    data['TEFF']= linear_coefficients.dot(teff)
    data['FEH']= linear_coefficients.dot(feh)
    data['CHI2DOF']=chi2dof
    data['REDSHIFT']=redshift
    data['COEFF']=linear_coefficients
    data['DATA_%s'%args.color]=star_colors[args.color]
    data['MODEL_%s'%args.color]=fitted_model_colors
    io.write_stdstar_models(args.outfile,normflux,stdwave,starfibers,data)
Ejemplo n.º 7
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame,suffix="RAW")

    if args.cosmics_nsig>0 and args.sky==None : # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame,args.cosmics_nsig)

    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame,suffix="FFLAT")

    if args.sky!=None :

        # read sky
        skymodel=read_sky(args.sky)

        if args.cosmics_nsig>0 :

            # use a copy the frame (not elegant but robust)
            copied_frame = copy.deepcopy(frame)
            
            # first subtract sky without throughput correction
            subtract_sky(copied_frame, skymodel, apply_throughput_correction = False)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(copied_frame,args.cosmics_nsig)

            # copy mask
            frame.mask = copied_frame.mask
            
            # and (re-)subtract sky, but just the correction term
            subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )

        else :
            # subtract sky
            subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )

        compute_and_append_frame_scores(frame,suffix="SKYSUB")

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

        # Ensure that ivars are set to 0 for all values if any designated
        # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
        frame = get_fiberbitmasked_frame(frame,bitmask="flux",ivar_framemask=True)
        compute_and_append_frame_scores(frame,suffix="CALIB")


    # save output
    write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 8
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat',
                        type=str,
                        default=None,
                        help='path of DESI fiberflat fits file')
    parser.add_argument('--sky',
                        type=str,
                        default=None,
                        help='path of DESI sky fits file')
    parser.add_argument('--calib',
                        type=str,
                        default=None,
                        help='path of DESI calibration fits file')
    parser.add_argument('--outfile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI sky fits file')
    # add calibration here when exists

    args = parser.parse_args()
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky != None:
        log.info("subtract sky")
        # read sky
        skymodel = read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

    # save output
    write_frame(args.outfile, frame)

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 9
0
def main(args):
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
             (args.color, args.delta_color))

    frames = {}
    flats = {}
    skies = {}

    spectrograph = None
    starfibers = None
    starindices = None
    fibermap = None

    # READ DATA
    ############################################

    for filename in args.frames:

        log.info("reading %s" % filename)
        frame = io.read_frame(filename)
        header = fits.getheader(filename, 0)
        frame_fibermap = frame.fibermap
        frame_starindices = np.where(isStdStar(frame_fibermap))[0]

        #- Confirm that all fluxes have entries but trust targeting bits
        #- to get basic magnitude range correct
        keep = np.ones(len(frame_starindices), dtype=bool)

        for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']:  #- and W1 and W2?
            keep &= frame_fibermap[colname][frame_starindices] > 10**(
                (22.5 - 30) / 2.5)
            keep &= frame_fibermap[colname][frame_starindices] < 10**(
                (22.5 - 0) / 2.5)

        frame_starindices = frame_starindices[keep]

        camera = safe_read_key(header, "CAMERA").strip().lower()

        if spectrograph is None:
            spectrograph = frame.spectrograph
            fibermap = frame_fibermap
            starindices = frame_starindices
            starfibers = fibermap["FIBER"][starindices]

        elif spectrograph != frame.spectrograph:
            log.error("incompatible spectrographs %d != %d" %
                      (spectrograph, frame.spectrograph))
            raise ValueError("incompatible spectrographs %d != %d" %
                             (spectrograph, frame.spectrograph))
        elif starindices.size != frame_starindices.size or np.sum(
                starindices != frame_starindices) > 0:
            log.error("incompatible fibermap")
            raise ValueError("incompatible fibermap")

        if not camera in frames:
            frames[camera] = []
        frames[camera].append(frame)

    for filename in args.skymodels:
        log.info("reading %s" % filename)
        sky = io.read_sky(filename)
        header = fits.getheader(filename, 0)
        camera = safe_read_key(header, "CAMERA").strip().lower()
        if not camera in skies:
            skies[camera] = []
        skies[camera].append(sky)

    for filename in args.fiberflats:
        log.info("reading %s" % filename)
        header = fits.getheader(filename, 0)
        flat = io.read_fiberflat(filename)
        camera = safe_read_key(header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if camera in flats:
            log.warning(
                "cannot handle several flats of same camera (%s), will use only the first one"
                % camera)
            #raise ValueError("cannot handle several flats of same camera (%s)"%camera)
        else:
            flats[camera] = flat

    if starindices.size == 0:
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")

    log.info("found %d STD stars" % starindices.size)

    log.warning("Not using flux errors for Standard Star fits!")

    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    for cam in frames:

        if not cam in skies:
            log.warning("Missing sky for %s" % cam)
            frames.pop(cam)
            continue
        if not cam in flats:
            log.warning("Missing flat for %s" % cam)
            frames.pop(cam)
            continue

        flat = flats[cam]
        for frame, sky in zip(frames[cam], skies[cam]):
            frame.flux = frame.flux[starindices]
            frame.ivar = frame.ivar[starindices]
            frame.ivar *= (frame.mask[starindices] == 0)
            frame.ivar *= (sky.ivar[starindices] != 0)
            frame.ivar *= (sky.mask[starindices] == 0)
            frame.ivar *= (flat.ivar[starindices] != 0)
            frame.ivar *= (flat.mask[starindices] == 0)
            frame.flux *= (frame.ivar > 0)  # just for clean plots
            for star in range(frame.flux.shape[0]):
                ok = np.where((frame.ivar[star] > 0)
                              & (flat.fiberflat[star] != 0))[0]
                if ok.size > 0:
                    frame.flux[star] = frame.flux[star] / flat.fiberflat[
                        star] - sky.flux[star]
            frame.resolution_data = frame.resolution_data[starindices]

    nstars = starindices.size
    fibermap = Table(fibermap[starindices])

    # READ MODELS
    ############################################
    log.info("reading star models in %s" % args.starmodels)
    stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
        args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################

    #- Support older fibermaps
    if 'PHOTSYS' not in fibermap.colnames:
        log.warning('Old fibermap format; using defaults for missing columns')
        log.warning("    PHOTSYS = 'S'")
        log.warning("    MW_TRANSMISSION_G/R/Z = 1.0")
        log.warning("    EBV = 0.0")
        fibermap['PHOTSYS'] = 'S'
        fibermap['MW_TRANSMISSION_G'] = 1.0
        fibermap['MW_TRANSMISSION_R'] = 1.0
        fibermap['MW_TRANSMISSION_Z'] = 1.0
        fibermap['EBV'] = 0.0

    model_filters = dict()
    if 'S' in fibermap['PHOTSYS']:
        for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
            model_filters[filter_name] = load_filter(filter_name)

    if 'N' in fibermap['PHOTSYS']:
        for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
            model_filters[filter_name] = load_filter(filter_name)

    if len(model_filters) == 0:
        raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")

    log.info("computing model mags for %s" % sorted(model_filters.keys()))
    model_mags = dict()
    fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
    for filter_name, filter_response in model_filters.items():
        model_mags[filter_name] = filter_response.get_ab_magnitude(
            stdflux * fluxunits, stdwave)
    log.info("done computing model mags")

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
    chi2dof = np.zeros((nstars))
    redshift = np.zeros((nstars))
    normflux = []

    star_mags = dict()
    star_unextincted_mags = dict()
    for band in ['G', 'R', 'Z']:
        star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
        star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
            fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])

    star_colors = dict()
    star_colors['G-R'] = star_mags['G'] - star_mags['R']
    star_colors['R-Z'] = star_mags['R'] - star_mags['Z']

    star_unextincted_colors = dict()
    star_unextincted_colors[
        'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
    star_unextincted_colors[
        'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']

    fitted_model_colors = np.zeros(nstars)

    for star in range(nstars):

        log.info("finding best model for observed star #%d" % star)

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames:
            for i, frame in enumerate(frames[camera]):
                identifier = "%s-%d" % (camera, i)
                wave[identifier] = frame.wave
                flux[identifier] = frame.flux[star]
                ivar[identifier] = frame.ivar[star]
                resolution_data[identifier] = frame.resolution_data[star]

        # preselect models based on magnitudes
        if fibermap['PHOTSYS'][star] == 'N':
            if args.color == 'G-R':
                model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
            elif args.color == 'R-Z':
                model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
            else:
                raise ValueError('Unknown color {}'.format(args.color))
        else:
            if args.color == 'G-R':
                model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
            elif args.color == 'R-Z':
                model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
            else:
                raise ValueError('Unknown color {}'.format(args.color))

        color_diff = model_colors - star_unextincted_colors[args.color][star]
        selection = np.abs(color_diff) < args.delta_color

        # smallest cube in parameter space including this selection (needed for interpolation)
        new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
            teff[selection]))
        new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
            logg[selection]))
        new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
            feh[selection]))
        selection = np.where(new_selection)[0]

        log.info(
            "star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
            % (star, starfibers[star], args.color,
               star_unextincted_colors[args.color][star], selection.size,
               stdflux.shape[0]))

        # Match unextincted standard stars to data
        coefficients, redshift[star], chi2dof[star] = match_templates(
            wave,
            flux,
            ivar,
            resolution_data,
            stdwave,
            stdflux[selection],
            teff[selection],
            logg[selection],
            feh[selection],
            ncpu=args.ncpu,
            z_max=args.z_max,
            z_res=args.z_res,
            template_error=args.template_error)

        linear_coefficients[star, selection] = coefficients

        log.info(
            'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
            .format(starfibers[star], np.inner(teff,
                                               linear_coefficients[star]),
                    np.inner(logg, linear_coefficients[star]),
                    np.inner(feh, linear_coefficients[star]), redshift[star],
                    chi2dof[star]))

        # Apply redshift to original spectrum at full resolution
        model = np.zeros(stdwave.size)
        redshifted_stdwave = stdwave * (1 + redshift[star])
        for i, c in enumerate(linear_coefficients[star]):
            if c != 0:
                model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])

        # Apply dust extinction to the model
        model *= dust_transmission(stdwave, fibermap['EBV'][star])

        # Compute final color of dust-extincted model
        if fibermap['PHOTSYS'][star] == 'N':
            if args.color == 'G-R':
                model_mag1 = model_filters['BASS_G'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_mag2 = model_filters['BASS_R'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_magr = model_mag2
            elif args.color == 'R-Z':
                model_mag1 = model_filters['BASS_R'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_magr = model_mag1
            else:
                raise ValueError('Unknown color {}'.format(args.color))
        else:
            if args.color == 'G-R':
                model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_magr = model_mag2
            elif args.color == 'R-Z':
                model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(
                    model * fluxunits, stdwave)
                model_magr = model_mag1
            else:
                raise ValueError('Unknown color {}'.format(args.color))

        fitted_model_colors[star] = model_mag1 - model_mag2

        #- TODO: move this back into normalize_templates, at the cost of
        #- recalculating a model magnitude?

        # Normalize the best model using reported magnitude
        scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)

        log.info('scaling R mag {} to {} using scale {}'.format(
            model_magr, star_mags['R'][star], scalefac))
        normflux.append(model * scalefac)

    # Now write the normalized flux for all best models to a file
    normflux = np.array(normflux)
    data = {}
    data['LOGG'] = linear_coefficients.dot(logg)
    data['TEFF'] = linear_coefficients.dot(teff)
    data['FEH'] = linear_coefficients.dot(feh)
    data['CHI2DOF'] = chi2dof
    data['REDSHIFT'] = redshift
    data['COEFF'] = linear_coefficients
    data['DATA_%s' % args.color] = star_colors[args.color]
    data['MODEL_%s' % args.color] = fitted_model_colors
    io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
Ejemplo n.º 10
0
def main() :
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID')
    parser.add_argument('--fibermap', type = str, help = 'path of fibermap file')
    parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits')
    parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9')
    parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux')

    args = parser.parse_args()
    log = get_logger()
    # Call necessary environment variables. No need if add argument to give full file path.
    if 'DESI_SPECTRO_REDUX' not in os.environ:
        raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles')

    DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX']
    PRODNAME=os.environ['PRODNAME']
    if 'DESISIM' not in os.environ:
        raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration')

    DESISIM=os.environ['DESISIM']   # to read the filter transmission files

    if args.fibermap is None or args.models is None or \
       args.spectrograph is None or args.outfile is None or \
       args.fiberflatexpid is None:
        log.critical('Missing a required argument')
        parser.print_help()
        sys.exit(12)

    # read Standard Stars from the fibermap file
    # returns the Fiber id, filter names and mags for the standard stars

    fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True)

    #- Trim to just fibers on this spectrograph
    ii =  (500*args.spectrograph <= fiber_tbdata["FIBER"])
    ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1))
    fiber_tbdata = fiber_tbdata[ii]

    #- Get info for the standard stars
    refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD")
    refFibers=fiber_tbdata["FIBER"][refStarIdx]
    refFilters=fiber_tbdata["FILTER"][refStarIdx]
    refMags=fiber_tbdata["MAG"]

    fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags}

    NIGHT=fiber_header['NIGHT']
    EXPID=fiber_header['EXPID']
    filters=fibers["FILTER"]
    if 'DESISIM' not in os.environ:
        raise RuntimeError('Set environment DESISIM. Can not find filter response files')
    basepath=DESISIM+"/data/"

    #now load all the skyfiles, framefiles, fiberflatfiles etc
    # all three channels files are simultaneously treated for model fitting
    skyfile={}
    framefile={}
    fiberflatfile={}
    for i in ["b","r","z"]:
        camera = i+str(args.spectrograph)
        skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera)
        framefile[i] = io.findfile('frame', NIGHT, EXPID, camera)
        fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera)

    #Read Frames, Flats and Sky files
    frameFlux={}
    frameIvar={}
    frameWave={}
    frameResolution={}
    framehdr={}
    fiberFlat={}
    ivarFlat={}
    maskFlat={}
    meanspecFlat={}
    waveFlat={}
    headerFlat={}
    sky={}
    skyivar={}
    skymask={}
    skywave={}
    skyhdr={}

    for i in ["b","r","z"]:
       #arg=(night,expid,'%s%s'%(i,spectrograph))
       #- minimal code change for refactored I/O, while not taking advantage of simplified structure
       frame = io.read_frame(framefile[i])
       frameFlux[i] = frame.flux
       frameIvar[i] = frame.ivar
       frameWave[i] = frame.wave
       frameResolution[i] = frame.resolution_data
       framehdr[i] = frame.header

       ff = io.read_fiberflat(fiberflatfile[i])
       fiberFlat[i] = ff.fiberflat
       ivarFlat[i] = ff.ivar
       maskFlat[i] = ff.mask
       meanspecFlat[i] = ff.meanspec
       waveFlat[i] = ff.wave
       headerFlat[i] = ff.header

       skymodel = io.read_sky(skyfile[i])
       sky[i] = skymodel.flux
       skyivar[i] = skymodel.ivar
       skymask[i] = skymodel.mask
       skywave[i] = skymodel.wave
       skyhdr[i] = skymodel.header

    # Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract

    convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]}

    # Read the standard Star data and divide by flat and subtract sky

    stars=[]
    ivars=[]
    for i in fibers["FIBER"]:
        #flat and sky should have same wavelength binning as data, otherwise should be rebinned.

        stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]],
                         "r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]],
                         "z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i]))
        ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]}))


    stdwave,stdflux,templateid=io.read_stdstar_templates(args.models)

    #- Trim standard star wavelengths to just the range we need
    minwave = min([min(w) for w in frameWave.values()])
    maxwave = max([max(w) for w in frameWave.values()])
    ii = (minwave-10 < stdwave) & (stdwave < maxwave+10)
    stdwave = stdwave[ii]
    stdflux = stdflux[:, ii]

    log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars)))
    if len(stars) == 0:
        log.critical("No standard stars!  Exiting")
        sys.exit(1)

    # Now for each star, find the best model and normalize.

    normflux=[]
    bestModelIndex=np.arange(len(stars))
    templateID=np.arange(len(stars))
    chi2dof=np.zeros(len(stars))

    #- TODO: don't use 'l' as a variable name.  Can look like a '1'
    for k,l in enumerate(stars):
        log.info("checking best model for star {0}".format(l[0]))

        starindex=l[0]
        mags=l[2]
        filters=fibers["FILTER"][k]
        rflux=stars[k][1]["r"][0]
        bflux=stars[k][1]["b"][0]
        zflux=stars[k][1]["z"][0]
        flux={"b":bflux,"r":rflux,"z":zflux}

        #print ivars
        rivar=ivars[k][1]["r"][0]
        bivar=ivars[k][1]["b"][0]
        zivar=ivars[k][1]["z"][0]
        ivar={"b":bivar,"r":rivar,"z":zivar}

        resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]}

        # Now find the best Model

        bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux)

        log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k]))
        # Normalize the best model using reported magnitude
        modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath)
        normflux.append(normalizedflux)

    # Now write the normalized flux for all best models to a file
    normflux=np.array(normflux)
    stdfibers=fibers["FIBER"]
    data={}
    data['BESTMODEL']=bestModelIndex
    data['CHI2DOF']=chi2dof
    data['TEMPLATEID']=templateid[bestModelIndex]
    norm_model_file=args.outfile
    io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
Ejemplo n.º 11
0
def main() :

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fibermap', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat', type = str, default = None, required=True,
                        help = 'path of DESI fiberflat fits file')
    parser.add_argument('--sky', type = str, default = None, required=True,
                        help = 'path of DESI sky fits file')
    parser.add_argument('--models', type = str, default = None, required=True,
                        help = 'path of spetro-photometric stellar spectra fits file')
    parser.add_argument('--outfile', type = str, default = None, required=True,
                        help = 'path of DESI flux calbration fits file')


    args = parser.parse_args()
    log=get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)
    
    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers=read_stdstar_models(args.models)

    # select fibers
    SPECMIN=frame.header["SPECMIN"]
    SPECMAX=frame.header["SPECMAX"]
    selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0]
    if selec.size == 0 :
        log.error("no stellar models for this spectro")
        sys.exit(12)
    fibers=model_fibers[selec]-frame.header["SPECMIN"]
    log.info("star fibers= %s"%str(fibers))

    table = read_fibermap(args.fibermap)
    bad=np.where(table["OBJTYPE"][fibers]!="STD")[0]
    if bad.size > 0 :
        for fiber in fibers[bad] :
            log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.header)


    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 12
0
def main(args):

    log = get_logger()

    cmd = [
        'desi_compute_fluxcalibration',
    ]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--' + key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    # Set fibermask flagged spectra to have 0 flux and variance
    frame = get_fiberbitmasked_frame(frame,
                                     bitmask='flux',
                                     ivar_framemask=True)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    ok = np.ones(len(model_metadata), dtype=bool)

    if args.chi2cut > 0:
        log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
        ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
    if args.delta_color_cut > 0:
        log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
        ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
               < args.delta_color_cut)
    if args.min_color is not None:
        log.info("Apply cut DATA_G-R>{}".format(args.min_color))
        ok &= (model_metadata["DATA_G-R"] > args.min_color)
    if args.chi2cut_nsig > 0:
        # automatically reject stars that ar chi2 outliers
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
            maxchi2, args.chi2cut_nsig))
        ok &= (model_metadata["CHI2DOF"] <= maxchi2)

    ok = np.where(ok)[0]
    if ok.size == 0:
        log.error("cuts discarded all stars")
        sys.exit(12)
    nstars = model_flux.shape[0]
    nbad = nstars - ok.size
    if nbad > 0:
        log.warning("discarding %d star(s) out of %d because of cuts" %
                    (nbad, nstars))
        model_flux = model_flux[ok]
        model_fibers = model_fibers[ok]
        model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap))[0]
    if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
        target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
        colname = target_colnames[0]
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
                .format(i, fibermap["OBJTYPE"][i], colname,
                        fibermap[colname][i]))
        sys.exit(12)

    # Make sure the fibers of interest aren't entirely masked.
    if np.sum(
            np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
            frame.nwave) == len(model_fibers):
        log.warning('All standard-star spectra are masked!')
        return

    fluxcalib = compute_flux_calibration(
        frame,
        model_wave,
        model_flux,
        model_fibers % 500,
        highest_throughput_nstars=args.highest_throughput)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame_meta=frame.meta,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 13
0
def main(args):
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
             (args.color, args.delta_color))

    frames = {}
    flats = {}
    skies = {}

    spectrograph = None
    starfibers = None
    starindices = None
    fibermap = None

    # READ DATA
    ############################################

    for filename in args.frames:

        log.info("reading %s" % filename)
        frame = io.read_frame(filename)
        header = fits.getheader(filename, 0)
        frame_fibermap = frame.fibermap
        frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]

        # check magnitude are well defined or discard stars
        tmp = []
        for i in frame_starindices:
            mags = frame_fibermap["MAG"][i]
            ok = np.sum((mags > 0) & (mags < 30))
            if np.sum((mags > 0) & (mags < 30)) == mags.size:
                tmp.append(i)
        frame_starindices = np.array(tmp).astype(int)

        camera = safe_read_key(header, "CAMERA").strip().lower()

        if spectrograph is None:
            spectrograph = frame.spectrograph
            fibermap = frame_fibermap
            starindices = frame_starindices
            starfibers = fibermap["FIBER"][starindices]

        elif spectrograph != frame.spectrograph:
            log.error("incompatible spectrographs %d != %d" %
                      (spectrograph, frame.spectrograph))
            raise ValueError("incompatible spectrographs %d != %d" %
                             (spectrograph, frame.spectrograph))
        elif starindices.size != frame_starindices.size or np.sum(
                starindices != frame_starindices) > 0:
            log.error("incompatible fibermap")
            raise ValueError("incompatible fibermap")

        if not camera in frames:
            frames[camera] = []
        frames[camera].append(frame)

    for filename in args.skymodels:
        log.info("reading %s" % filename)
        sky = io.read_sky(filename)
        header = fits.getheader(filename, 0)
        camera = safe_read_key(header, "CAMERA").strip().lower()
        if not camera in skies:
            skies[camera] = []
        skies[camera].append(sky)

    for filename in args.fiberflats:
        log.info("reading %s" % filename)
        header = fits.getheader(filename, 0)
        flat = io.read_fiberflat(filename)
        camera = safe_read_key(header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if camera in flats:

            log.warning(
                "cannot handle several flats of same camera (%s), will use only the first one"
                % camera)
            #raise ValueError("cannot handle several flats of same camera (%s)"%camera)
        else:
            flats[camera] = flat

    if starindices.size == 0:
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")

    log.info("found %d STD stars" % starindices.size)

    imaging_filters = fibermap["FILTER"][starindices]
    imaging_mags = fibermap["MAG"][starindices]

    log.warning(
        "NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")

    ebv = np.zeros(starindices.size)
    if "SFD_EBV" in fibermap.columns.names:
        log.info("Using 'SFD_EBV' from fibermap")
        ebv = fibermap["SFD_EBV"][starindices]
    else:
        log.warning("NO EXTINCTION VALUES IN FIBERMAP!!")

    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    for cam in frames:

        if not cam in skies:
            log.warning("Missing sky for %s" % cam)
            frames.pop(cam)
            continue
        if not cam in flats:
            log.warning("Missing flat for %s" % cam)
            frames.pop(cam)
            continue

        flat = flats[cam]
        for frame, sky in zip(frames[cam], skies[cam]):
            frame.flux = frame.flux[starindices]
            frame.ivar = frame.ivar[starindices]
            frame.ivar *= (frame.mask[starindices] == 0)
            frame.ivar *= (sky.ivar[starindices] != 0)
            frame.ivar *= (sky.mask[starindices] == 0)
            frame.ivar *= (flat.ivar[starindices] != 0)
            frame.ivar *= (flat.mask[starindices] == 0)
            frame.flux *= (frame.ivar > 0)  # just for clean plots
            for star in range(frame.flux.shape[0]):
                ok = np.where((frame.ivar[star] > 0)
                              & (flat.fiberflat[star] != 0))[0]
                if ok.size > 0:
                    frame.flux[star] = frame.flux[star] / flat.fiberflat[
                        star] - sky.flux[star]
            frame.resolution_data = frame.resolution_data[starindices]

    nstars = starindices.size
    starindices = None  # we don't need this anymore

    # READ MODELS
    ############################################
    log.info("reading star models in %s" % args.starmodels)
    stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
        args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################
    model_filters = []
    for tmp in np.unique(imaging_filters):
        if len(tmp) > 0:  # can be one empty entry
            model_filters.append(tmp)

    log.info("computing model mags %s" % model_filters)
    model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
    fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom

    for index in range(len(model_filters)):
        if model_filters[index].startswith('WISE'):
            log.warning('not computing stdstar {} mags'.format(
                model_filters[index]))
            continue

        filter_response = load_filter(model_filters[index])
        for m in range(stdflux.shape[0]):
            model_mags[m, index] = filter_response.get_ab_magnitude(
                stdflux[m] * fluxunits, stdwave)
    log.info("done computing model mags")

    mean_extinction_delta_mags = None
    mean_ebv = np.mean(ebv)
    if mean_ebv > 0:
        log.info(
            "Compute a mean delta_color from average E(B-V) = %3.2f based on canonial model star"
            % mean_ebv)
        # compute a mean delta_color from mean_ebv based on canonial model star
        #######################################################################
        # will then use this color offset in the model pre-selection
        # find canonical f-type model: Teff=6000, logg=4, Fe/H=-1.5
        canonical_model = np.argmin((teff - 6000.0)**2 + (logg - 4.0)**2 +
                                    (feh + 1.5)**2)
        canonical_model_mags_without_extinction = model_mags[canonical_model]
        canonical_model_mags_with_extinction = np.zeros(
            canonical_model_mags_without_extinction.shape)

        canonical_model_reddened_flux = stdflux[
            canonical_model] * dust_transmission(stdwave, mean_ebv)
        for index in range(len(model_filters)):
            if model_filters[index].startswith('WISE'):
                log.warning('not computing stdstar {} mags'.format(
                    model_filters[index]))
                continue
            filter_response = load_filter(model_filters[index])
            canonical_model_mags_with_extinction[
                index] = filter_response.get_ab_magnitude(
                    canonical_model_reddened_flux * fluxunits, stdwave)

        mean_extinction_delta_mags = canonical_model_mags_with_extinction - canonical_model_mags_without_extinction

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
    chi2dof = np.zeros((nstars))
    redshift = np.zeros((nstars))
    normflux = []

    star_colors_array = np.zeros((nstars))
    model_colors_array = np.zeros((nstars))

    for star in range(nstars):

        log.info("finding best model for observed star #%d" % star)

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames:

            for i, frame in enumerate(frames[camera]):
                identifier = "%s-%d" % (camera, i)
                wave[identifier] = frame.wave
                flux[identifier] = frame.flux[star]
                ivar[identifier] = frame.ivar[star]
                resolution_data[identifier] = frame.resolution_data[star]

        # preselec models based on magnitudes

        # compute star color
        index1, index2 = get_color_filter_indices(imaging_filters[star],
                                                  args.color)
        if index1 < 0 or index2 < 0:
            log.error("cannot compute '%s' color from %s" %
                      (color_name, filters))
        filter1 = imaging_filters[star][index1]
        filter2 = imaging_filters[star][index2]
        star_color = imaging_mags[star][index1] - imaging_mags[star][index2]
        star_colors_array[star] = star_color

        # compute models color
        model_index1 = -1
        model_index2 = -1
        for i, fname in enumerate(model_filters):
            if fname == filter1:
                model_index1 = i
            elif fname == filter2:
                model_index2 = i

        if model_index1 < 0 or model_index2 < 0:
            log.error("cannot compute '%s' model color from %s" %
                      (color_name, filters))
        model_colors = model_mags[:, model_index1] - model_mags[:,
                                                                model_index2]

        # apply extinction here
        # use the colors derived from the cannonical model with the mean ebv of the stars
        # and simply apply a scaling factor based on the ebv of this star
        # this is sufficiently precise for the broad model pre-selection we are doing here
        # the exact reddening of the star to each pre-selected model is
        # apply afterwards
        if mean_extinction_delta_mags is not None and mean_ebv != 0:
            delta_color = (mean_extinction_delta_mags[model_index1] -
                           mean_extinction_delta_mags[model_index2]
                           ) * ebv[star] / mean_ebv
            model_colors += delta_color
            log.info(
                "Apply a %s-%s color offset = %4.3f to the models for star with E(B-V)=%4.3f"
                % (model_filters[model_index1], model_filters[model_index2],
                   delta_color, ebv[star]))
        # selection

        selection = np.abs(model_colors - star_color) < args.delta_color
        # smallest cube in parameter space including this selection (needed for interpolation)
        new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
            teff[selection]))
        new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
            logg[selection]))
        new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
            feh[selection]))
        selection = np.where(new_selection)[0]

        log.info(
            "star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
            % (star, starfibers[star], args.color, filter1, filter2,
               star_color, selection.size, stdflux.shape[0]))

        # apply extinction to selected_models
        dust_transmission_of_this_star = dust_transmission(stdwave, ebv[star])
        selected_reddened_stdflux = stdflux[
            selection] * dust_transmission_of_this_star

        coefficients, redshift[star], chi2dof[star] = match_templates(
            wave,
            flux,
            ivar,
            resolution_data,
            stdwave,
            selected_reddened_stdflux,
            teff[selection],
            logg[selection],
            feh[selection],
            ncpu=args.ncpu,
            z_max=args.z_max,
            z_res=args.z_res,
            template_error=args.template_error)

        linear_coefficients[star, selection] = coefficients

        log.info(
            'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
            .format(starfibers[star], np.inner(teff,
                                               linear_coefficients[star]),
                    np.inner(logg, linear_coefficients[star]),
                    np.inner(feh, linear_coefficients[star]), redshift[star],
                    chi2dof[star]))

        # Apply redshift to original spectrum at full resolution
        model = np.zeros(stdwave.size)
        for i, c in enumerate(linear_coefficients[star]):
            if c != 0:
                model += c * np.interp(stdwave, stdwave *
                                       (1 + redshift[star]), stdflux[i])

        # Apply dust extinction
        model *= dust_transmission_of_this_star

        # Compute final model color
        mag1 = load_filter(model_filters[model_index1]).get_ab_magnitude(
            model * fluxunits, stdwave)
        mag2 = load_filter(model_filters[model_index2]).get_ab_magnitude(
            model * fluxunits, stdwave)
        model_colors_array[star] = mag1 - mag2

        # Normalize the best model using reported magnitude
        normalizedflux = normalize_templates(stdwave, model,
                                             imaging_mags[star],
                                             imaging_filters[star])
        normflux.append(normalizedflux)

    # Now write the normalized flux for all best models to a file
    normflux = np.array(normflux)
    data = {}
    data['LOGG'] = linear_coefficients.dot(logg)
    data['TEFF'] = linear_coefficients.dot(teff)
    data['FEH'] = linear_coefficients.dot(feh)
    data['CHI2DOF'] = chi2dof
    data['REDSHIFT'] = redshift
    data['COEFF'] = linear_coefficients
    data['DATA_%s' % args.color] = star_colors_array
    data['MODEL_%s' % args.color] = model_colors_array
    norm_model_file = args.outfile
    io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
Ejemplo n.º 14
0
def main(args):
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
             (args.color, args.delta_color))
    log.info('multiprocess parallelizing with {} processes'.format(args.ncpu))

    # READ DATA
    ############################################
    # First loop through and group by exposure and spectrograph
    frames_by_expid = {}
    for filename in args.frames:
        log.info("reading %s" % filename)
        frame = io.read_frame(filename)
        expid = safe_read_key(frame.meta, "EXPID")
        camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
        spec = camera[1]
        uniq_key = (expid, spec)
        if uniq_key in frames_by_expid.keys():
            frames_by_expid[uniq_key][camera] = frame
        else:
            frames_by_expid[uniq_key] = {camera: frame}

    frames = {}
    flats = {}
    skies = {}

    spectrograph = None
    starfibers = None
    starindices = None
    fibermap = None

    # For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
    # cameras and then proceed with extracting the frame information
    # once we modify the fibermap FIBERSTATUS
    for (expid, spec), camdict in frames_by_expid.items():

        fiberstatus = None
        for frame in camdict.values():
            if fiberstatus is None:
                fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
            else:
                fiberstatus |= frame.fibermap['FIBERSTATUS']

        for camera, frame in camdict.items():
            frame.fibermap['FIBERSTATUS'] |= fiberstatus
            # Set fibermask flagged spectra to have 0 flux and variance
            frame = get_fiberbitmasked_frame(frame,
                                             bitmask='stdstars',
                                             ivar_framemask=True)
            frame_fibermap = frame.fibermap
            frame_starindices = np.where(isStdStar(frame_fibermap))[0]

            #- Confirm that all fluxes have entries but trust targeting bits
            #- to get basic magnitude range correct
            keep = np.ones(len(frame_starindices), dtype=bool)

            for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']:  #- and W1 and W2?
                keep &= frame_fibermap[colname][frame_starindices] > 10**(
                    (22.5 - 30) / 2.5)
                keep &= frame_fibermap[colname][frame_starindices] < 10**(
                    (22.5 - 0) / 2.5)

            frame_starindices = frame_starindices[keep]

            if spectrograph is None:
                spectrograph = frame.spectrograph
                fibermap = frame_fibermap
                starindices = frame_starindices
                starfibers = fibermap["FIBER"][starindices]

            elif spectrograph != frame.spectrograph:
                log.error("incompatible spectrographs %d != %d" %
                          (spectrograph, frame.spectrograph))
                raise ValueError("incompatible spectrographs %d != %d" %
                                 (spectrograph, frame.spectrograph))
            elif starindices.size != frame_starindices.size or np.sum(
                    starindices != frame_starindices) > 0:
                log.error("incompatible fibermap")
                raise ValueError("incompatible fibermap")

            if not camera in frames:
                frames[camera] = []

            frames[camera].append(frame)

    # possibly cleanup memory
    del frames_by_expid

    for filename in args.skymodels:
        log.info("reading %s" % filename)
        sky = io.read_sky(filename)
        camera = safe_read_key(sky.header, "CAMERA").strip().lower()
        if not camera in skies:
            skies[camera] = []
        skies[camera].append(sky)

    for filename in args.fiberflats:
        log.info("reading %s" % filename)
        flat = io.read_fiberflat(filename)
        camera = safe_read_key(flat.header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if camera in flats:
            log.warning(
                "cannot handle several flats of same camera (%s), will use only the first one"
                % camera)
            #raise ValueError("cannot handle several flats of same camera (%s)"%camera)
        else:
            flats[camera] = flat

    if starindices.size == 0:
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")

    log.info("found %d STD stars" % starindices.size)

    log.warning("Not using flux errors for Standard Star fits!")

    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    # since poping dict, we need to copy keys to iterate over to avoid
    # RuntimeError due to changing dict
    frame_cams = list(frames.keys())
    for cam in frame_cams:

        if not cam in skies:
            log.warning("Missing sky for %s" % cam)
            frames.pop(cam)
            continue
        if not cam in flats:
            log.warning("Missing flat for %s" % cam)
            frames.pop(cam)
            continue

        flat = flats[cam]
        for frame, sky in zip(frames[cam], skies[cam]):
            frame.flux = frame.flux[starindices]
            frame.ivar = frame.ivar[starindices]
            frame.ivar *= (frame.mask[starindices] == 0)
            frame.ivar *= (sky.ivar[starindices] != 0)
            frame.ivar *= (sky.mask[starindices] == 0)
            frame.ivar *= (flat.ivar[starindices] != 0)
            frame.ivar *= (flat.mask[starindices] == 0)
            frame.flux *= (frame.ivar > 0)  # just for clean plots
            for star in range(frame.flux.shape[0]):
                ok = np.where((frame.ivar[star] > 0)
                              & (flat.fiberflat[star] != 0))[0]
                if ok.size > 0:
                    frame.flux[star] = frame.flux[star] / flat.fiberflat[
                        star] - sky.flux[star]
            frame.resolution_data = frame.resolution_data[starindices]

    # CHECK S/N
    ############################################
    # for each band in 'brz', record quadratic sum of median S/N across wavelength
    snr = dict()
    for band in ['b', 'r', 'z']:
        snr[band] = np.zeros(starindices.size)
    for cam in frames:
        band = cam[0].lower()
        for frame in frames[cam]:
            msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
                             np.sqrt(np.gradient(frame.wave)),
                             axis=1)  # median SNR per sqrt(A.)
            msnr *= (msnr > 0)
            snr[band] = np.sqrt(snr[band]**2 + msnr**2)
    log.info("SNR(B) = {}".format(snr['b']))

    ###############################
    max_number_of_stars = 50
    min_blue_snr = 4.
    ###############################
    indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]

    validstars = np.where(snr['b'][indices] > min_blue_snr)[0]

    #- TODO: later we filter on models based upon color, thus throwing
    #- away very blue stars for which we don't have good models.

    log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
             format(min_blue_snr, validstars.size))
    if validstars.size == 0:
        log.error("No valid star")
        sys.exit(12)

    validstars = indices[validstars]

    for band in ['b', 'r', 'z']:
        snr[band] = snr[band][validstars]

    log.info("BLUE SNR of selected stars={}".format(snr['b']))

    for cam in frames:
        for frame in frames[cam]:
            frame.flux = frame.flux[validstars]
            frame.ivar = frame.ivar[validstars]
            frame.resolution_data = frame.resolution_data[validstars]
    starindices = starindices[validstars]
    starfibers = starfibers[validstars]
    nstars = starindices.size
    fibermap = Table(fibermap[starindices])

    # MASK OUT THROUGHPUT DIP REGION
    ############################################
    mask_throughput_dip_region = True
    if mask_throughput_dip_region:
        wmin = 4300.
        wmax = 4500.
        log.warning(
            "Masking out the wavelength region [{},{}]A in the standard star fit"
            .format(wmin, wmax))
    for cam in frames:
        for frame in frames[cam]:
            ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
            if ii.size > 0:
                frame.ivar[:, ii] = 0

    # READ MODELS
    ############################################
    log.info("reading star models in %s" % args.starmodels)
    stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
        args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################

    #- Support older fibermaps
    if 'PHOTSYS' not in fibermap.colnames:
        log.warning('Old fibermap format; using defaults for missing columns')
        log.warning("    PHOTSYS = 'S'")
        log.warning("    MW_TRANSMISSION_G/R/Z = 1.0")
        log.warning("    EBV = 0.0")
        fibermap['PHOTSYS'] = 'S'
        fibermap['MW_TRANSMISSION_G'] = 1.0
        fibermap['MW_TRANSMISSION_R'] = 1.0
        fibermap['MW_TRANSMISSION_Z'] = 1.0
        fibermap['EBV'] = 0.0

    model_filters = dict()
    for band in ["G", "R", "Z"]:
        for photsys in np.unique(fibermap['PHOTSYS']):
            model_filters[band + photsys] = load_legacy_survey_filter(
                band=band, photsys=photsys)

    log.info("computing model mags for %s" % sorted(model_filters.keys()))
    model_mags = dict()
    fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
    for filter_name, filter_response in model_filters.items():
        model_mags[filter_name] = filter_response.get_ab_magnitude(
            stdflux * fluxunits, stdwave)
    log.info("done computing model mags")

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
    chi2dof = np.zeros((nstars))
    redshift = np.zeros((nstars))
    normflux = []

    star_mags = dict()
    star_unextincted_mags = dict()
    for band in ['G', 'R', 'Z']:
        star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
        star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
            fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])

    star_colors = dict()
    star_colors['G-R'] = star_mags['G'] - star_mags['R']
    star_colors['R-Z'] = star_mags['R'] - star_mags['Z']

    star_unextincted_colors = dict()
    star_unextincted_colors[
        'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
    star_unextincted_colors[
        'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']

    fitted_model_colors = np.zeros(nstars)

    for star in range(nstars):

        log.info("finding best model for observed star #%d" % star)

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames:
            for i, frame in enumerate(frames[camera]):
                identifier = "%s-%d" % (camera, i)
                wave[identifier] = frame.wave
                flux[identifier] = frame.flux[star]
                ivar[identifier] = frame.ivar[star]
                resolution_data[identifier] = frame.resolution_data[star]

        # preselect models based on magnitudes
        photsys = fibermap['PHOTSYS'][star]
        if not args.color in ['G-R', 'R-Z']:
            raise ValueError('Unknown color {}'.format(args.color))
        bands = args.color.split("-")
        model_colors = model_mags[bands[0] + photsys] - model_mags[bands[1] +
                                                                   photsys]

        color_diff = model_colors - star_unextincted_colors[args.color][star]
        selection = np.abs(color_diff) < args.delta_color
        if np.sum(selection) == 0:
            log.warning("no model in the selected color range for this star")
            continue

        # smallest cube in parameter space including this selection (needed for interpolation)
        new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
            teff[selection]))
        new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
            logg[selection]))
        new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
            feh[selection]))
        selection = np.where(new_selection)[0]

        log.info(
            "star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
            % (star, starfibers[star], args.color,
               star_unextincted_colors[args.color][star], selection.size,
               stdflux.shape[0]))

        # Match unextincted standard stars to data
        coefficients, redshift[star], chi2dof[star] = match_templates(
            wave,
            flux,
            ivar,
            resolution_data,
            stdwave,
            stdflux[selection],
            teff[selection],
            logg[selection],
            feh[selection],
            ncpu=args.ncpu,
            z_max=args.z_max,
            z_res=args.z_res,
            template_error=args.template_error)

        linear_coefficients[star, selection] = coefficients

        log.info(
            'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
            .format(starfibers[star], np.inner(teff,
                                               linear_coefficients[star]),
                    np.inner(logg, linear_coefficients[star]),
                    np.inner(feh, linear_coefficients[star]), redshift[star],
                    chi2dof[star]))

        # Apply redshift to original spectrum at full resolution
        model = np.zeros(stdwave.size)
        redshifted_stdwave = stdwave * (1 + redshift[star])
        for i, c in enumerate(linear_coefficients[star]):
            if c != 0:
                model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])

        # Apply dust extinction to the model
        log.info("Applying MW dust extinction to star {} with EBV = {}".format(
            star, fibermap['EBV'][star]))
        model *= dust_transmission(stdwave, fibermap['EBV'][star])

        # Compute final color of dust-extincted model
        photsys = fibermap['PHOTSYS'][star]
        if not args.color in ['G-R', 'R-Z']:
            raise ValueError('Unknown color {}'.format(args.color))
        bands = args.color.split("-")
        model_mag1 = model_filters[bands[0] + photsys].get_ab_magnitude(
            model * fluxunits, stdwave)
        model_mag2 = model_filters[bands[1] + photsys].get_ab_magnitude(
            model * fluxunits, stdwave)
        fitted_model_colors[star] = model_mag1 - model_mag2
        if bands[0] == "R":
            model_magr = model_mag1
        elif bands[1] == "R":
            model_magr = model_mag2

        #- TODO: move this back into normalize_templates, at the cost of
        #- recalculating a model magnitude?

        # Normalize the best model using reported magnitude
        scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)

        log.info('scaling R mag {:.3f} to {:.3f} using scale {}'.format(
            model_magr, star_mags['R'][star], scalefac))
        normflux.append(model * scalefac)

    # Now write the normalized flux for all best models to a file
    normflux = np.array(normflux)

    fitted_stars = np.where(chi2dof != 0)[0]
    if fitted_stars.size == 0:
        log.error("No star has been fit.")
        sys.exit(12)

    data = {}
    data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
    data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
    data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
    data['CHI2DOF'] = chi2dof[fitted_stars]
    data['REDSHIFT'] = redshift[fitted_stars]
    data['COEFF'] = linear_coefficients[fitted_stars, :]
    data['DATA_%s' % args.color] = star_colors[args.color][fitted_stars]
    data['MODEL_%s' % args.color] = fitted_model_colors[fitted_stars]
    data['BLUE_SNR'] = snr['b'][fitted_stars]
    data['RED_SNR'] = snr['r'][fitted_stars]
    data['NIR_SNR'] = snr['z'][fitted_stars]
    io.write_stdstar_models(args.outfile, normflux, stdwave,
                            starfibers[fitted_stars], data)
Ejemplo n.º 15
0
def main(args):

    log = get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    if args.chi2cut > 0:
        ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0]
        if ok.size == 0:
            log.error("chi2cut has discarded all stars")
            sys.exit(12)
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning("discarding %d star(s) out of %d because of chi2cut" %
                        (nbad, nstars))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    if args.delta_color_cut > 0:
        ok = np.where(
            np.abs(model_metadata["MODEL_G-R"] -
                   model_metadata["DATA_G-R"]) < args.delta_color_cut)[0]
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning(
                "discarding %d star(s) out of %d because |delta_color|>%f" %
                (nbad, nstars, args.delta_color_cut))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    # automatically reject stars that ar chi2 outliers
    if args.chi2cut_nsig > 0:
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0]
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning(
                "discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"
                % (nbad, nstars, args.chi2cut_nsig, maxchi2))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    w = np.where(fibermap["OBJTYPE"][model_fibers % 500] != 'STD')[0]

    if len(w) > 0:
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
                (i, fibermap["OBJTYPE"][i]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux,
                                         model_fibers % 500)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 16
0
def main(args):

    log = get_logger()

    cmd = [
        'desi_compute_fluxcalibration',
    ]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--' + key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    # Set fibermask flagged spectra to have 0 flux and variance
    frame = get_fiberbitmasked_frame(frame,
                                     bitmask='flux',
                                     ivar_framemask=True)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    ok = np.ones(len(model_metadata), dtype=bool)

    if args.chi2cut > 0:
        log.info("apply cut CHI2DOF<{}".format(args.chi2cut))
        good = (model_metadata["CHI2DOF"] < args.chi2cut)
        bad = ~good
        ok &= good
        if np.any(bad):
            log.info(" discard {} stars with CHI2DOF= {}".format(
                np.sum(bad), list(model_metadata["CHI2DOF"][bad])))

    legacy_filters = ('G-R', 'R-Z')
    gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP')
    model_column_list = model_metadata.columns.names
    if args.color is None:
        if 'MODEL_G-R' in model_column_list:
            color = 'G-R'
        elif 'MODEL_GAIA-BP-RP' in model_column_list:
            log.info('Using Gaia filters')
            color = 'GAIA-BP-RP'
        else:
            log.error(
                "Can't find either G-R or BP-RP color in the model file.")
            sys.exit(15)
    else:
        if args.color not in legacy_filters and args.color not in gaia_filters:
            log.error(
                'Color name {} is not allowed, must be one of {} {}'.format(
                    args.color, legacy_filters, gaia_filters))
            sys.exit(14)
        color = args.color
        if color not in model_column_list:
            # This should't happen
            log.error(
                'The color {} was not computed in the models'.format(color))
            sys.exit(16)

    if args.delta_color_cut > 0:
        log.info("apply cut |delta color|<{}".format(args.delta_color_cut))
        good = (np.abs(model_metadata["MODEL_" + color] -
                       model_metadata["DATA_" + color]) < args.delta_color_cut)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            vals = model_metadata["MODEL_" +
                                  color][bad] - model_metadata["DATA_" +
                                                               color][bad]
            log.info(" discard {} stars with dcolor= {}".format(
                np.sum(bad), list(vals)))

    if args.min_color is not None:
        log.info("apply cut DATA_{}>{}".format(color, args.min_color))
        good = (model_metadata["DATA_{}".format(color)] > args.min_color)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            vals = model_metadata["DATA_{}".format(color)][bad]
            log.info(" discard {} stars with {}= {}".format(
                np.sum(bad), color, list(vals)))

    if args.chi2cut_nsig > 0:
        # automatically reject stars that ar chi2 outliers
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
            maxchi2, args.chi2cut_nsig))
        good = (model_metadata["CHI2DOF"] <= maxchi2)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            log.info(" discard {} stars with CHI2DOF={}".format(
                np.sum(bad), list(model_metadata["CHI2DOF"][bad])))

    ok = np.where(ok)[0]
    if ok.size == 0:
        log.error("selection cuts discarded all stars")
        sys.exit(12)
    nstars = model_flux.shape[0]
    nbad = nstars - ok.size
    if nbad > 0:
        log.warning("discarding %d star(s) out of %d because of cuts" %
                    (nbad, nstars))
        model_flux = model_flux[ok]
        model_fibers = model_fibers[ok]
        model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap))[0]
    if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
        target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
        colname = target_colnames[0]
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
                .format(i, fibermap["OBJTYPE"][i], colname,
                        fibermap[colname][i]))
        sys.exit(12)

    # Make sure the fibers of interest aren't entirely masked.
    if np.sum(
            np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
            frame.nwave) == len(model_fibers):
        log.warning('All standard-star spectra are masked!')
        return

    fluxcalib = compute_flux_calibration(
        frame,
        model_wave,
        model_flux,
        model_fibers % 500,
        highest_throughput_nstars=args.highest_throughput,
        exposure_seeing_fwhm=args.seeing_fwhm)

    # QA
    if (args.qafile is not None):

        from desispec.io import write_qa_frame
        from desispec.io.qa import load_qa_frame
        from desispec.qa import qa_plots

        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame_meta=frame.meta,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # record inputs
    frame.meta['IN_FRAME'] = shorten_filename(args.infile)
    frame.meta['IN_SKY'] = shorten_filename(args.sky)
    frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
    frame.meta['STDMODEL'] = shorten_filename(args.models)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 17
0
def main(args):
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
             (args.color, args.delta_color))

    frames = {}
    flats = {}
    skies = {}

    spectrograph = None
    starfibers = None
    starindices = None
    fibermap = None

    # READ DATA
    ############################################

    for filename in args.frames:

        log.info("reading %s" % filename)
        frame = io.read_frame(filename)
        header = fits.getheader(filename, 0)
        frame_fibermap = frame.fibermap
        frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]
        camera = safe_read_key(header, "CAMERA").strip().lower()

        if spectrograph is None:
            spectrograph = frame.spectrograph
            fibermap = frame_fibermap
            starindices = frame_starindices
            starfibers = fibermap["FIBER"][starindices]

        elif spectrograph != frame.spectrograph:
            log.error("incompatible spectrographs %d != %d" %
                      (spectrograph, frame.spectrograph))
            raise ValueError("incompatible spectrographs %d != %d" %
                             (spectrograph, frame.spectrograph))
        elif starindices.size != frame_starindices.size or np.sum(
                starindices != frame_starindices) > 0:
            log.error("incompatible fibermap")
            raise ValueError("incompatible fibermap")

        if frames.has_key(camera):
            log.error(
                "cannot handle for now several frame of same camera (%s)" %
                camera)
            raise ValueError(
                "cannot handle for now several frame of same camera (%s)" %
                camera)

        frames[camera] = frame

    for filename in args.skymodels:
        log.info("reading %s" % filename)
        sky = io.read_sky(filename)
        header = fits.getheader(filename, 0)
        camera = safe_read_key(header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if skies.has_key(camera):
            log.error("cannot handle several skymodels of same camera (%s)" %
                      camera)
            raise ValueError(
                "cannot handle several skymodels of same camera (%s)" % camera)

        skies[camera] = sky

    for filename in args.fiberflats:
        log.info("reading %s" % filename)
        header = fits.getheader(filename, 0)
        flat = io.read_fiberflat(filename)
        camera = safe_read_key(header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if flats.has_key(camera):
            log.error("cannot handle several flats of same camera (%s)" %
                      camera)
            raise ValueError(
                "cannot handle several flats of same camera (%s)" % camera)
        flats[camera] = flat

    if starindices.size == 0:
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")

    log.info("found %d STD stars" % starindices.size)

    imaging_filters = fibermap["FILTER"][starindices]
    imaging_mags = fibermap["MAG"][starindices]

    log.warning(
        "NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")
    log.warning(
        "NO EXTINCTION VALUES IN FIBERMAP, I AM IGNORING THIS FOR NOW !!")

    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    for cam in frames:

        if not skies.has_key(cam):
            log.warning("Missing sky for %s" % cam)
            frames.pop(cam)
            continue
        if not flats.has_key(cam):
            log.warning("Missing flat for %s" % cam)
            frames.pop(cam)
            continue

        frames[cam].flux = frames[cam].flux[starindices]
        frames[cam].ivar = frames[cam].ivar[starindices]

        frames[cam].ivar *= (frames[cam].mask[starindices] == 0)
        frames[cam].ivar *= (skies[cam].ivar[starindices] != 0)
        frames[cam].ivar *= (skies[cam].mask[starindices] == 0)
        frames[cam].ivar *= (flats[cam].ivar[starindices] != 0)
        frames[cam].ivar *= (flats[cam].mask[starindices] == 0)
        frames[cam].flux *= (frames[cam].ivar > 0)  # just for clean plots
        for star in range(frames[cam].flux.shape[0]):
            ok = np.where((frames[cam].ivar[star] > 0)
                          & (flats[cam].fiberflat[star] != 0))[0]
            if ok.size > 0:
                frames[cam].flux[star] = frames[cam].flux[star] / flats[
                    cam].fiberflat[star] - skies[cam].flux[star]
    nstars = starindices.size
    starindices = None  # we don't need this anymore

    # READ MODELS
    ############################################
    log.info("reading star models in %s" % args.starmodels)
    stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
        args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################
    model_filters = []
    for tmp in np.unique(imaging_filters):
        if len(tmp) > 0:  # can be one empty entry
            model_filters.append(tmp)

    log.info("computing model mags %s" % model_filters)
    model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
    fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
    for index in range(len(model_filters)):
        filter_response = load_filter(model_filters[index])
        for m in range(stdflux.shape[0]):
            model_mags[m, index] = filter_response.get_ab_magnitude(
                stdflux[m] * fluxunits, stdwave)
    log.info("done computing model mags")

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    bestModelIndex = np.arange(nstars)
    templateID = np.arange(nstars)
    chi2dof = np.zeros((nstars))
    redshift = np.zeros((nstars))
    normflux = []

    for star in range(nstars):

        log.info("finding best model for observed star #%d" % star)

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames:
            band = camera[0]
            wave[band] = frames[camera].wave
            flux[band] = frames[camera].flux[star]
            ivar[band] = frames[camera].ivar[star]
            resolution_data[band] = frames[camera].resolution_data[star]

        # preselec models based on magnitudes

        # compute star color
        index1, index2 = get_color_filter_indices(imaging_filters[star],
                                                  args.color)
        if index1 < 0 or index2 < 0:
            log.error("cannot compute '%s' color from %s" %
                      (color_name, filters))
        filter1 = imaging_filters[star][index1]
        filter2 = imaging_filters[star][index2]
        star_color = imaging_mags[star][index1] - imaging_mags[star][index2]

        # compute models color
        model_index1 = -1
        model_index2 = -1
        for i, fname in enumerate(model_filters):
            if fname == filter1:
                model_index1 = i
            elif fname == filter2:
                model_index2 = i

        if model_index1 < 0 or model_index2 < 0:
            log.error("cannot compute '%s' model color from %s" %
                      (color_name, filters))
        model_colors = model_mags[:, model_index1] - model_mags[:,
                                                                model_index2]

        # selection
        selection = np.where(
            np.abs(model_colors - star_color) < args.delta_color)[0]

        log.info(
            "star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
            % (star, starfibers[star], args.color, filter1, filter2,
               star_color, selection.size, stdflux.shape[0]))

        index_in_selection, redshift[star], chi2dof[star] = match_templates(
            wave,
            flux,
            ivar,
            resolution_data,
            stdwave,
            stdflux[selection],
            teff[selection],
            logg[selection],
            feh[selection],
            ncpu=args.ncpu,
            z_max=args.z_max,
            z_res=args.z_res)

        bestModelIndex[star] = selection[index_in_selection]

        log.info(
            'Star Fiber: {0}; TemplateID: {1}; Redshift: {2}; Chisq/dof: {3}'.
            format(starfibers[star], bestModelIndex[star], redshift[star],
                   chi2dof[star]))
        # Apply redshift to original spectrum at full resolution
        tmp = np.interp(stdwave, stdwave / (1 + redshift[star]),
                        stdflux[bestModelIndex[star]])
        # Normalize the best model using reported magnitude
        normalizedflux = normalize_templates(stdwave, tmp, imaging_mags[star],
                                             imaging_filters[star])
        normflux.append(normalizedflux)

    # Now write the normalized flux for all best models to a file
    normflux = np.array(normflux)
    data = {}
    data['BESTMODEL'] = bestModelIndex
    data['TEMPLATEID'] = bestModelIndex  # IS THAT IT?
    data['CHI2DOF'] = chi2dof
    data['REDSHIFT'] = redshift
    norm_model_file = args.outfile
    io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
Ejemplo n.º 18
0
def main(args, comm=None):
    """ finds the best models of all standard stars in the frame
    and normlize the model flux. Output is written to a file and will be called for calibration.
    """

    log = get_logger()

    log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
             (args.color, args.delta_color))

    if args.mpi or comm is not None:
        from mpi4py import MPI
        if comm is None:
            comm = MPI.COMM_WORLD
        size = comm.Get_size()
        rank = comm.Get_rank()
        if rank == 0:
            log.info('mpi parallelizing with {} ranks'.format(size))
    else:
        comm = None
        rank = 0
        size = 1

    # disable multiprocess by forcing ncpu = 1 when using MPI
    if comm is not None:
        ncpu = 1
        if rank == 0:
            log.info('disabling multiprocess (forcing ncpu = 1)')
    else:
        ncpu = args.ncpu

    if ncpu > 1:
        if rank == 0:
            log.info(
                'multiprocess parallelizing with {} processes'.format(ncpu))

    if args.ignore_gpu and desispec.fluxcalibration.use_gpu:
        # Opt-out of GPU usage
        desispec.fluxcalibration.use_gpu = False
        if rank == 0:
            log.info('ignoring GPU')
    elif desispec.fluxcalibration.use_gpu:
        # Nothing to do here, GPU is used by default if available
        if rank == 0:
            log.info('using GPU')
    else:
        if rank == 0:
            log.info('GPU not available')

    std_targetids = None
    if args.std_targetids is not None:
        std_targetids = args.std_targetids

    # READ DATA
    ############################################
    # First loop through and group by exposure and spectrograph
    frames_by_expid = {}
    rows = list()
    for filename in args.frames:
        log.info("reading %s" % filename)
        frame = io.read_frame(filename)
        night = safe_read_key(frame.meta, "NIGHT")
        expid = safe_read_key(frame.meta, "EXPID")
        camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
        rows.append((night, expid, camera))
        spec = camera[1]
        uniq_key = (expid, spec)
        if uniq_key in frames_by_expid.keys():
            frames_by_expid[uniq_key][camera] = frame
        else:
            frames_by_expid[uniq_key] = {camera: frame}

    input_frames_table = Table(rows=rows, names=('NIGHT', 'EXPID', 'TILEID'))

    frames = {}
    flats = {}
    skies = {}

    spectrograph = None
    starfibers = None
    starindices = None
    fibermap = None
    # For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
    # cameras and then proceed with extracting the frame information
    # once we modify the fibermap FIBERSTATUS
    for (expid, spec), camdict in frames_by_expid.items():

        fiberstatus = None
        for frame in camdict.values():
            if fiberstatus is None:
                fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
            else:
                fiberstatus |= frame.fibermap['FIBERSTATUS']

        for camera, frame in camdict.items():
            frame.fibermap['FIBERSTATUS'] |= fiberstatus
            # Set fibermask flagged spectra to have 0 flux and variance
            frame = get_fiberbitmasked_frame(frame,
                                             bitmask='stdstars',
                                             ivar_framemask=True)
            frame_fibermap = frame.fibermap
            if std_targetids is None:
                frame_starindices = np.where(isStdStar(frame_fibermap))[0]
            else:
                frame_starindices = np.nonzero(
                    np.isin(frame_fibermap['TARGETID'], std_targetids))[0]

            #- Confirm that all fluxes have entries but trust targeting bits
            #- to get basic magnitude range correct
            keep_legacy = np.ones(len(frame_starindices), dtype=bool)

            for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']:  #- and W1 and W2?
                keep_legacy &= frame_fibermap[colname][
                    frame_starindices] > 10**((22.5 - 30) / 2.5)
                keep_legacy &= frame_fibermap[colname][
                    frame_starindices] < 10**((22.5 - 0) / 2.5)
            keep_gaia = np.ones(len(frame_starindices), dtype=bool)

            for colname in ['G', 'BP', 'RP']:  #- and W1 and W2?
                keep_gaia &= frame_fibermap[
                    'GAIA_PHOT_' + colname +
                    '_MEAN_MAG'][frame_starindices] > 10
                keep_gaia &= frame_fibermap[
                    'GAIA_PHOT_' + colname +
                    '_MEAN_MAG'][frame_starindices] < 20
            n_legacy_std = keep_legacy.sum()
            n_gaia_std = keep_gaia.sum()
            keep = keep_legacy | keep_gaia
            # accept both types of standards for the time being

            # keep the indices for gaia/legacy subsets
            gaia_indices = keep_gaia[keep]
            legacy_indices = keep_legacy[keep]

            frame_starindices = frame_starindices[keep]

            if spectrograph is None:
                spectrograph = frame.spectrograph
                fibermap = frame_fibermap
                starindices = frame_starindices
                starfibers = fibermap["FIBER"][starindices]

            elif spectrograph != frame.spectrograph:
                log.error("incompatible spectrographs {} != {}".format(
                    spectrograph, frame.spectrograph))
                raise ValueError("incompatible spectrographs {} != {}".format(
                    spectrograph, frame.spectrograph))
            elif starindices.size != frame_starindices.size or np.sum(
                    starindices != frame_starindices) > 0:
                log.error("incompatible fibermap")
                raise ValueError("incompatible fibermap")

            if not camera in frames:
                frames[camera] = []

            frames[camera].append(frame)

    # possibly cleanup memory
    del frames_by_expid

    for filename in args.skymodels:
        log.info("reading %s" % filename)
        sky = io.read_sky(filename)
        camera = safe_read_key(sky.header, "CAMERA").strip().lower()
        if not camera in skies:
            skies[camera] = []
        skies[camera].append(sky)

    for filename in args.fiberflats:
        log.info("reading %s" % filename)
        flat = io.read_fiberflat(filename)
        camera = safe_read_key(flat.header, "CAMERA").strip().lower()

        # NEED TO ADD MORE CHECKS
        if camera in flats:
            log.warning(
                "cannot handle several flats of same camera (%s), will use only the first one"
                % camera)
            #raise ValueError("cannot handle several flats of same camera (%s)"%camera)
        else:
            flats[camera] = flat

    # if color is not specified we decide on the fly
    color = args.color
    if color is not None:
        if color[:4] == 'GAIA':
            legacy_color = False
            gaia_color = True
        else:
            legacy_color = True
            gaia_color = False
        if n_legacy_std == 0 and legacy_color:
            raise Exception(
                'Specified Legacy survey color, but no legacy standards')
        if n_gaia_std == 0 and gaia_color:
            raise Exception('Specified gaia color, but no gaia stds')

    if starindices.size == 0:
        log.error("no STD star found in fibermap")
        raise ValueError("no STD star found in fibermap")
    log.info("found %d STD stars" % starindices.size)

    if n_legacy_std == 0:
        gaia_std = True
        if color is None:
            color = 'GAIA-BP-RP'
    else:
        gaia_std = False
        if color is None:
            color = 'G-R'
        if n_gaia_std > 0:
            log.info('Gaia standards found but not used')

    if gaia_std:
        # The name of the reference filter to which we normalize the flux
        ref_mag_name = 'GAIA-G'
        color_band1, color_band2 = ['GAIA-' + _ for _ in color[5:].split('-')]
        log.info(
            "Using Gaia standards with color {} and normalizing to {}".format(
                color, ref_mag_name))
        # select appropriate subset of standards
        starindices = starindices[gaia_indices]
        starfibers = starfibers[gaia_indices]
    else:
        ref_mag_name = 'R'
        color_band1, color_band2 = color.split('-')
        log.info("Using Legacy standards with color {} and normalizing to {}".
                 format(color, ref_mag_name))
        # select appropriate subset of standards
        starindices = starindices[legacy_indices]
        starfibers = starfibers[legacy_indices]

    # excessive check but just in case
    if not color in ['G-R', 'R-Z', 'GAIA-BP-RP', 'GAIA-G-RP']:
        raise ValueError('Unknown color {}'.format(color))

    # log.warning("Not using flux errors for Standard Star fits!")

    # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
    ############################################
    # since poping dict, we need to copy keys to iterate over to avoid
    # RuntimeError due to changing dict
    frame_cams = list(frames.keys())
    for cam in frame_cams:

        if not cam in skies:
            log.warning("Missing sky for %s" % cam)
            frames.pop(cam)
            continue
        if not cam in flats:
            log.warning("Missing flat for %s" % cam)
            frames.pop(cam)
            continue

        flat = flats[cam]
        for frame, sky in zip(frames[cam], skies[cam]):
            frame.flux = frame.flux[starindices]
            frame.ivar = frame.ivar[starindices]
            frame.ivar *= (frame.mask[starindices] == 0)
            frame.ivar *= (sky.ivar[starindices] != 0)
            frame.ivar *= (sky.mask[starindices] == 0)
            frame.ivar *= (flat.ivar[starindices] != 0)
            frame.ivar *= (flat.mask[starindices] == 0)
            frame.flux *= (frame.ivar > 0)  # just for clean plots
            for star in range(frame.flux.shape[0]):
                ok = np.where((frame.ivar[star] > 0)
                              & (flat.fiberflat[star] != 0))[0]
                if ok.size > 0:
                    frame.flux[star] = frame.flux[star] / flat.fiberflat[
                        star] - sky.flux[star]
            frame.resolution_data = frame.resolution_data[starindices]

        nframes = len(frames[cam])
        if nframes > 1:
            # optimal weights for the coaddition = ivar*throughput, not directly ivar,
            # we estimate the relative throughput with median fluxes at this stage
            medflux = np.zeros(nframes)
            for i, frame in enumerate(frames[cam]):
                if np.sum(frame.ivar > 0) == 0:
                    log.error(
                        "ivar=0 for all std star spectra in frame {}-{:08d}".
                        format(cam, frame.meta["EXPID"]))
                else:
                    medflux[i] = np.median(frame.flux[frame.ivar > 0])
            log.debug("medflux = {}".format(medflux))
            medflux *= (medflux > 0)
            if np.sum(medflux > 0) == 0:
                log.error(
                    "mean median flux = 0, for all stars in fibers {}".format(
                        list(frames[cam][0].fibermap["FIBER"][starindices])))
                sys.exit(12)
            mmedflux = np.mean(medflux[medflux > 0])
            weights = medflux / mmedflux
            log.info("coadding {} exposures in cam {}, w={}".format(
                nframes, cam, weights))

            sw = np.zeros(frames[cam][0].flux.shape)
            swf = np.zeros(frames[cam][0].flux.shape)
            swr = np.zeros(frames[cam][0].resolution_data.shape)

            for i, frame in enumerate(frames[cam]):
                sw += weights[i] * frame.ivar
                swf += weights[i] * frame.ivar * frame.flux
                swr += weights[i] * frame.ivar[:,
                                               None, :] * frame.resolution_data
            coadded_frame = frames[cam][0]
            coadded_frame.ivar = sw
            coadded_frame.flux = swf / (sw + (sw == 0))
            coadded_frame.resolution_data = swr / ((sw +
                                                    (sw == 0))[:, None, :])
            frames[cam] = [coadded_frame]

    # CHECK S/N
    ############################################
    # for each band in 'brz', record quadratic sum of median S/N across wavelength
    snr = dict()
    for band in ['b', 'r', 'z']:
        snr[band] = np.zeros(starindices.size)
    for cam in frames:
        band = cam[0].lower()
        for frame in frames[cam]:
            msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
                             np.sqrt(np.gradient(frame.wave)),
                             axis=1)  # median SNR per sqrt(A.)
            msnr *= (msnr > 0)
            snr[band] = np.sqrt(snr[band]**2 + msnr**2)
    log.info("SNR(B) = {}".format(snr['b']))

    ###############################
    max_number_of_stars = 50
    min_blue_snr = 4.
    ###############################
    indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]

    validstars = np.where(snr['b'][indices] > min_blue_snr)[0]

    #- TODO: later we filter on models based upon color, thus throwing
    #- away very blue stars for which we don't have good models.

    log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
             format(min_blue_snr, validstars.size))
    if validstars.size == 0:
        log.error("No valid star")
        sys.exit(12)

    validstars = indices[validstars]

    for band in ['b', 'r', 'z']:
        snr[band] = snr[band][validstars]

    log.info("BLUE SNR of selected stars={}".format(snr['b']))

    for cam in frames:
        for frame in frames[cam]:
            frame.flux = frame.flux[validstars]
            frame.ivar = frame.ivar[validstars]
            frame.resolution_data = frame.resolution_data[validstars]
    starindices = starindices[validstars]
    starfibers = starfibers[validstars]
    nstars = starindices.size
    fibermap = Table(fibermap[starindices])

    # MASK OUT THROUGHPUT DIP REGION
    ############################################
    mask_throughput_dip_region = True
    if mask_throughput_dip_region:
        wmin = 4300.
        wmax = 4500.
        log.warning(
            "Masking out the wavelength region [{},{}]A in the standard star fit"
            .format(wmin, wmax))
    for cam in frames:
        for frame in frames[cam]:
            ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
            if ii.size > 0:
                frame.ivar[:, ii] = 0

    # READ MODELS
    ############################################
    log.info("reading star models in %s" % args.starmodels)
    stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
        args.starmodels)

    # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
    ############################################

    #- Support older fibermaps
    if 'PHOTSYS' not in fibermap.colnames:
        log.warning('Old fibermap format; using defaults for missing columns')
        log.warning("    PHOTSYS = 'S'")
        log.warning("    EBV = 0.0")
        fibermap['PHOTSYS'] = 'S'
        fibermap['EBV'] = 0.0

    if not np.in1d(np.unique(fibermap['PHOTSYS']), ['', 'N', 'S', 'G']).all():
        log.error('Unknown PHOTSYS found')
        raise Exception('Unknown PHOTSYS found')
    # Fetching Filter curves
    model_filters = dict()
    for band in ["G", "R", "Z"]:
        for photsys in np.unique(fibermap['PHOTSYS']):
            if photsys in ['N', 'S']:
                model_filters[band + photsys] = load_legacy_survey_filter(
                    band=band, photsys=photsys)
    if len(model_filters) == 0:
        log.info('No Legacy survey photometry identified in fibermap')

    # I will always load gaia data even if we are fitting LS standards only
    for band in ["G", "BP", "RP"]:
        model_filters["GAIA-" + band] = load_gaia_filter(band=band, dr=2)

    # Compute model mags on rank 0 and bcast result to other ranks
    # This sidesteps an OOM event on Cori Haswell with "-c 2"
    model_mags = None
    if rank == 0:
        log.info("computing model mags for %s" % sorted(model_filters.keys()))
        model_mags = dict()
        for filter_name in model_filters.keys():
            model_mags[filter_name] = get_magnitude(stdwave, stdflux,
                                                    model_filters, filter_name)
        log.info("done computing model mags")
    if comm is not None:
        model_mags = comm.bcast(model_mags, root=0)

    # LOOP ON STARS TO FIND BEST MODEL
    ############################################
    star_mags = dict()
    star_unextincted_mags = dict()

    if gaia_std and (fibermap['EBV'] == 0).all():
        log.info("Using E(B-V) from SFD rather than FIBERMAP")
        # when doing gaia standards, on old tiles the
        # EBV is not set so we fetch from SFD (in original SFD scaling)
        ebv = SFDMap(scaling=1).ebv(
            acoo.SkyCoord(ra=fibermap['TARGET_RA'] * units.deg,
                          dec=fibermap['TARGET_DEC'] * units.deg))
    else:
        ebv = fibermap['EBV']

    photometric_systems = np.unique(fibermap['PHOTSYS'])
    if not gaia_std:
        for band in ['G', 'R', 'Z']:
            star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
            star_unextincted_mags[band] = np.zeros(star_mags[band].shape)
            for photsys in photometric_systems:
                r_band = extinction_total_to_selective_ratio(
                    band, photsys)  # dimensionless
                # r_band = a_band / E(B-V)
                # E(B-V) is a difference of magnitudes (dimensionless)
                # a_band = -2.5*log10(effective dust transmission) , dimensionless
                # effective dust transmission =
                #                  integral( SED(lambda) * filter_transmission(lambda,band) * dust_transmission(lambda,E(B-V)) dlamdba)
                #                / integral( SED(lambda) * filter_transmission(lambda,band) dlamdba)
                selection = (fibermap['PHOTSYS'] == photsys)
                a_band = r_band * ebv[selection]  # dimensionless
                star_unextincted_mags[band][selection] = 22.5 - 2.5 * np.log10(
                    fibermap['FLUX_' + band][selection]) - a_band

    for band in ['G', 'BP', 'RP']:
        star_mags['GAIA-' + band] = fibermap['GAIA_PHOT_' + band + '_MEAN_MAG']

    for band, extval in gaia_extinction(star_mags['GAIA-G'],
                                        star_mags['GAIA-BP'],
                                        star_mags['GAIA-RP'], ebv).items():
        star_unextincted_mags['GAIA-' +
                              band] = star_mags['GAIA-' + band] - extval

    star_colors = dict()
    star_unextincted_colors = dict()

    # compute the colors and define the unextincted colors
    # the unextincted colors are filled later
    if not gaia_std:
        for c1, c2 in ['GR', 'RZ']:
            star_colors[c1 + '-' + c2] = star_mags[c1] - star_mags[c2]
            star_unextincted_colors[c1 + '-' +
                                    c2] = (star_unextincted_mags[c1] -
                                           star_unextincted_mags[c2])
    for c1, c2 in [('BP', 'RP'), ('G', 'RP')]:
        star_colors['GAIA-' + c1 + '-' + c2] = (star_mags['GAIA-' + c1] -
                                                star_mags['GAIA-' + c2])
        star_unextincted_colors['GAIA-' + c1 + '-' +
                                c2] = (star_unextincted_mags['GAIA-' + c1] -
                                       star_unextincted_mags['GAIA-' + c2])

    linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
    chi2dof = np.zeros((nstars))
    redshift = np.zeros((nstars))
    normflux = np.zeros((nstars, stdwave.size))
    fitted_model_colors = np.zeros(nstars)

    local_comm, head_comm = None, None
    if comm is not None:
        # All ranks in local_comm work on the same stars
        local_comm = comm.Split(rank % nstars, rank)
        # The color 1 in head_comm contains all ranks that are have rank 0 in local_comm
        head_comm = comm.Split(rank < nstars, rank)

    for star in range(rank % nstars, nstars, size):

        log.info("rank %d: finding best model for observed star #%d" %
                 (rank, star))

        # np.array of wave,flux,ivar,resol
        wave = {}
        flux = {}
        ivar = {}
        resolution_data = {}
        for camera in frames:
            for i, frame in enumerate(frames[camera]):
                identifier = "%s-%d" % (camera, i)
                wave[identifier] = frame.wave
                flux[identifier] = frame.flux[star]
                ivar[identifier] = frame.ivar[star]
                resolution_data[identifier] = frame.resolution_data[star]

        # preselect models based on magnitudes
        photsys = fibermap['PHOTSYS'][star]

        if gaia_std:
            model_colors = model_mags[color_band1] - model_mags[color_band2]
        else:
            model_colors = model_mags[color_band1 +
                                      photsys] - model_mags[color_band2 +
                                                            photsys]

        color_diff = model_colors - star_unextincted_colors[color][star]
        selection = np.abs(color_diff) < args.delta_color
        if np.sum(selection) == 0:
            log.warning("no model in the selected color range for this star")
            continue

        # smallest cube in parameter space including this selection (needed for interpolation)
        new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
            teff[selection]))
        new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
            logg[selection]))
        new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
            feh[selection]))
        selection = np.where(new_selection)[0]

        log.info(
            "star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
            % (star, starfibers[star], color,
               star_unextincted_colors[color][star], selection.size,
               stdflux.shape[0]))

        # Match unextincted standard stars to data
        match_templates_result = match_templates(
            wave,
            flux,
            ivar,
            resolution_data,
            stdwave,
            stdflux[selection],
            teff[selection],
            logg[selection],
            feh[selection],
            ncpu=ncpu,
            z_max=args.z_max,
            z_res=args.z_res,
            template_error=args.template_error,
            comm=local_comm)

        # Only local rank 0 can perform the remaining work
        if local_comm is not None and local_comm.Get_rank() != 0:
            continue

        coefficients, redshift[star], chi2dof[star] = match_templates_result
        linear_coefficients[star, selection] = coefficients
        log.info(
            'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
            .format(starfibers[star], np.inner(teff,
                                               linear_coefficients[star]),
                    np.inner(logg, linear_coefficients[star]),
                    np.inner(feh, linear_coefficients[star]), redshift[star],
                    chi2dof[star]))

        # Apply redshift to original spectrum at full resolution
        model = np.zeros(stdwave.size)
        redshifted_stdwave = stdwave * (1 + redshift[star])
        for i, c in enumerate(linear_coefficients[star]):
            if c != 0:
                model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])

        # Apply dust extinction to the model
        log.info("Applying MW dust extinction to star {} with EBV = {}".format(
            star, ebv[star]))
        model *= dust_transmission(stdwave, ebv[star])

        # Compute final color of dust-extincted model
        photsys = fibermap['PHOTSYS'][star]

        if not gaia_std:
            model_mag1, model_mag2 = [
                get_magnitude(stdwave, model, model_filters, _ + photsys)
                for _ in [color_band1, color_band2]
            ]
        else:
            model_mag1, model_mag2 = [
                get_magnitude(stdwave, model, model_filters, _)
                for _ in [color_band1, color_band2]
            ]

        if color_band1 == ref_mag_name:
            model_magr = model_mag1
        elif color_band2 == ref_mag_name:
            model_magr = model_mag2
        else:
            # if the reference magnitude is not among colours
            # I'm fetching it separately. This will happen when
            # colour is BP-RP and ref magnitude is G
            if gaia_std:
                model_magr = get_magnitude(stdwave, model, model_filters,
                                           ref_mag_name)
            else:
                model_magr = get_magnitude(stdwave, model, model_filters,
                                           ref_mag_name + photsys)
        fitted_model_colors[star] = model_mag1 - model_mag2

        #- TODO: move this back into normalize_templates, at the cost of
        #- recalculating a model magnitude?

        cur_refmag = star_mags[ref_mag_name][star]

        # Normalize the best model using reported magnitude
        scalefac = 10**((model_magr - cur_refmag) / 2.5)

        log.info('scaling {} mag {:.3f} to {:.3f} using scale {}'.format(
            ref_mag_name, model_magr, cur_refmag, scalefac))
        normflux[star] = model * scalefac

    if head_comm is not None and rank < nstars:  # head_comm color is 1
        linear_coefficients = head_comm.reduce(linear_coefficients,
                                               op=MPI.SUM,
                                               root=0)
        redshift = head_comm.reduce(redshift, op=MPI.SUM, root=0)
        chi2dof = head_comm.reduce(chi2dof, op=MPI.SUM, root=0)
        fitted_model_colors = head_comm.reduce(fitted_model_colors,
                                               op=MPI.SUM,
                                               root=0)
        normflux = head_comm.reduce(normflux, op=MPI.SUM, root=0)

    # Check at least one star was fit. The check is peformed on rank 0 and
    # the result is bcast to other ranks so that all ranks exit together if
    # the check fails.
    atleastonestarfit = False
    if rank == 0:
        fitted_stars = np.where(chi2dof != 0)[0]
        atleastonestarfit = fitted_stars.size > 0
    if comm is not None:
        atleastonestarfit = comm.bcast(atleastonestarfit, root=0)
    if not atleastonestarfit:
        log.error("No star has been fit.")
        sys.exit(12)

    # Now write the normalized flux for all best models to a file
    if rank == 0:

        # get the fibermap from any input frame for the standard stars
        fibermap = Table(frame.fibermap)
        keep = np.isin(fibermap['FIBER'], starfibers[fitted_stars])
        fibermap = fibermap[keep]

        # drop fibermap columns specific to exposures instead of targets
        for col in [
                'DELTA_X', 'DELTA_Y', 'EXPTIME', 'NUM_ITER', 'FIBER_RA',
                'FIBER_DEC', 'FIBER_X', 'FIBER_Y'
        ]:
            if col in fibermap.colnames:
                fibermap.remove_column(col)

        data = {}
        data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
        data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
        data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
        data['CHI2DOF'] = chi2dof[fitted_stars]
        data['REDSHIFT'] = redshift[fitted_stars]
        data['COEFF'] = linear_coefficients[fitted_stars, :]
        data['DATA_%s' % color] = star_colors[color][fitted_stars]
        data['MODEL_%s' % color] = fitted_model_colors[fitted_stars]
        data['BLUE_SNR'] = snr['b'][fitted_stars]
        data['RED_SNR'] = snr['r'][fitted_stars]
        data['NIR_SNR'] = snr['z'][fitted_stars]
        io.write_stdstar_models(args.outfile, normflux, stdwave,
                                starfibers[fitted_stars], data, fibermap,
                                input_frames_table)
Ejemplo n.º 19
0
def main(args):

    log = get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers = read_stdstar_models(args.models)
    model_tuple = model_flux, model_wave, model_fibers

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap
    model_fibers = model_fibers % 500
    if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
        for i in model_fibers:
            log.error(
                "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
                (i, fibermap["OBJTYPE"][i]))
        sys.exit(12)

    #fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux)
    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame,
                                flavor=frame.meta['FLAVOR'])
        # Run
        qaframe.run_qa('FLUXCALIB',
                       (frame, fluxcalib, model_tuple))  #, indiv_stars))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib,
                                     model_tuple)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 20
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame, suffix="RAW")

    if args.cosmics_nsig > 0 and args.sky == None:  # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame, args.cosmics_nsig)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame, suffix="FFLAT")

    if args.sky != None:

        # read sky
        skymodel = read_sky(args.sky)

        if args.cosmics_nsig > 0:

            # first subtract sky without throughput correction
            subtract_sky(frame, skymodel, throughput_correction=False)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(frame, args.cosmics_nsig)

            if args.sky_throughput_correction:
                # and (re-)subtract sky, but just the correction term
                subtract_sky(frame,
                             skymodel,
                             throughput_correction=True,
                             default_throughput_correction=0.)

        else:
            # subtract sky
            subtract_sky(frame,
                         skymodel,
                         throughput_correction=args.sky_throughput_correction)

        compute_and_append_frame_scores(frame, suffix="SKYSUB")

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)
        compute_and_append_frame_scores(frame, suffix="CALIB")

    # save output
    write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 Angstrom)')

    log.info("successfully wrote %s" % args.outfile)
Ejemplo n.º 21
0
def main(args) :

    log=get_logger()

    cmd = ['desi_compute_fluxcalibration',]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--'+key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models)

    if args.chi2cut > 0 :
        ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0]
        if ok.size == 0 :
            log.error("chi2cut has discarded all stars")
            sys.exit(12)
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    
    if args.delta_color_cut > 0 :
        ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0]
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    

    # automatically reject stars that ar chi2 outliers
    if args.chi2cut_nsig > 0 :
        mchi2=np.median(model_metadata["CHI2DOF"])
        rmschi2=np.std(model_metadata["CHI2DOF"])
        maxchi2=mchi2+args.chi2cut_nsig*rmschi2
        ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0]
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    
    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0]
    if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)):
        for i in model_fibers%500:
            log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format(
                (i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i])))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s"%args.outfile)
Ejemplo n.º 22
0
def main(args):
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    if (not args.no_tsnr) and (args.calib is None):
        log.critical(
            'need --fiberflat --sky and --calib to compute template SNR')
        sys.exit(12)

    frame = read_frame(args.infile)

    if not args.no_tsnr:
        # tsnr alpha calc. requires uncalibrated + no substraction rame.
        uncalibrated_frame = copy.deepcopy(frame)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame, suffix="RAW")

    if args.cosmics_nsig > 0 and args.sky == None:  # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame, args.cosmics_nsig)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame, suffix="FFLAT")
    else:
        fiberflat = None

    if args.no_xtalk:
        zero_ivar = (not args.no_zero_ivar)
    else:
        zero_ivar = False

    if args.sky != None:

        # read sky
        skymodel = read_sky(args.sky)

        if args.cosmics_nsig > 0:

            # use a copy the frame (not elegant but robust)
            copied_frame = copy.deepcopy(frame)

            # first subtract sky without throughput correction
            subtract_sky(copied_frame,
                         skymodel,
                         apply_throughput_correction=False,
                         zero_ivar=zero_ivar)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(copied_frame, args.cosmics_nsig)

            # copy mask
            frame.mask = copied_frame.mask

            # and (re-)subtract sky, but just the correction term
            subtract_sky(frame,
                         skymodel,
                         apply_throughput_correction=(
                             not args.no_sky_throughput_correction),
                         zero_ivar=zero_ivar)

        else:
            # subtract sky
            subtract_sky(frame,
                         skymodel,
                         apply_throughput_correction=(
                             not args.no_sky_throughput_correction),
                         zero_ivar=zero_ivar)

        compute_and_append_frame_scores(frame, suffix="SKYSUB")

    if not args.no_xtalk:
        log.info("fiber crosstalk correction")
        correct_fiber_crosstalk(frame, fiberflat)

        if not args.no_zero_ivar:
            frame.ivar *= (frame.mask == 0)

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

        # Ensure that ivars are set to 0 for all values if any designated
        # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
        frame = get_fiberbitmasked_frame(
            frame, bitmask="flux", ivar_framemask=(not args.no_zero_ivar))
        compute_and_append_frame_scores(frame, suffix="CALIB")

    if not args.no_tsnr:
        log.info("calculating tsnr")
        results, alpha = calc_tsnr2(uncalibrated_frame,
                                    fiberflat=fiberflat,
                                    skymodel=skymodel,
                                    fluxcalib=fluxcalib,
                                    alpha_only=args.alpha_only)

        frame.meta['TSNRALPH'] = alpha

        comments = {k: "from calc_frame_tsnr" for k in results.keys()}
        append_frame_scores(frame, results, comments, overwrite=True)

    # record inputs
    frame.meta['IN_FRAME'] = shorten_filename(args.infile)
    frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
    frame.meta['IN_SKY'] = shorten_filename(args.sky)
    frame.meta['IN_CALIB'] = shorten_filename(args.calib)

    # save output
    write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
    log.info("successfully wrote %s" % args.outfile)